mirror of
https://github.com/carbon-language/carbon-lang.git
synced 2026-10-05 22:02:55 +01:00
Refactor compile-related tests to share construction. (#4396)
Note in particular that this fixes an issue where SharedValueStore had been shared across files, when they should be per-file. This is only visible when doing multiple compilations in a single test, which was rare before. This also moves these tests into the Testing namespace. My memory of the various namespacing changes is that we'd generally agreed to have tests in Testing so that we'd see SemIR:: and similar, same as we would in a lot of the implementation.
This commit is contained in:
Jon Ross-Perkins
@@ -17,6 +17,7 @@
|
||||
#include "toolchain/diagnostics/mocks.h"
|
||||
#include "toolchain/lex/lex.h"
|
||||
#include "toolchain/lex/tokenized_buffer_test_helpers.h"
|
||||
#include "toolchain/testing/compile_helper.h"
|
||||
#include "toolchain/testing/yaml_test_helpers.h"
|
||||
|
||||
namespace Carbon::Lex {
|
||||
@@ -35,29 +36,11 @@ namespace Yaml = ::Carbon::Testing::Yaml;
|
||||
|
||||
class LexerTest : public ::testing::Test {
|
||||
protected:
|
||||
auto GetSourceBuffer(llvm::StringRef text) -> SourceBuffer& {
|
||||
std::string filename = llvm::formatv("test{0}.carbon", ++file_index_);
|
||||
CARBON_CHECK(fs_.addFile(filename, /*ModificationTime=*/0,
|
||||
llvm::MemoryBuffer::getMemBuffer(text)));
|
||||
source_storage_.push_front(std::move(*SourceBuffer::MakeFromFile(
|
||||
fs_, filename, ConsoleDiagnosticConsumer())));
|
||||
return source_storage_.front();
|
||||
}
|
||||
|
||||
auto Lex(llvm::StringRef text,
|
||||
DiagnosticConsumer& consumer = ConsoleDiagnosticConsumer())
|
||||
-> TokenizedBuffer {
|
||||
return Lex::Lex(value_stores_, GetSourceBuffer(text), consumer);
|
||||
}
|
||||
|
||||
SharedValueStores value_stores_;
|
||||
llvm::vfs::InMemoryFileSystem fs_;
|
||||
int file_index_ = 0;
|
||||
std::forward_list<SourceBuffer> source_storage_;
|
||||
Testing::CompileHelper compile_helper_;
|
||||
};
|
||||
|
||||
TEST_F(LexerTest, HandlesEmptyBuffer) {
|
||||
auto buffer = Lex("");
|
||||
auto& buffer = compile_helper_.GetTokenizedBuffer("");
|
||||
EXPECT_FALSE(buffer.has_errors());
|
||||
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
@@ -65,7 +48,8 @@ TEST_F(LexerTest, HandlesEmptyBuffer) {
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, TracksLinesAndColumns) {
|
||||
auto buffer = Lex("\n ;;\n ;;;\n x\"foo\" '''baz\n a\n ''' y");
|
||||
auto& buffer = compile_helper_.GetTokenizedBuffer(
|
||||
"\n ;;\n ;;;\n x\"foo\" '''baz\n a\n ''' y");
|
||||
EXPECT_FALSE(buffer.has_errors());
|
||||
EXPECT_THAT(
|
||||
buffer,
|
||||
@@ -102,8 +86,8 @@ TEST_F(LexerTest, TracksLinesAndColumns) {
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, TracksLinesAndColumnsCRLF) {
|
||||
auto buffer =
|
||||
Lex("\r\n ;;\r\n ;;;\r\n x\"foo\" '''baz\r\n a\r\n ''' y");
|
||||
auto& buffer = compile_helper_.GetTokenizedBuffer(
|
||||
"\r\n ;;\r\n ;;;\r\n x\"foo\" '''baz\r\n a\r\n ''' y");
|
||||
EXPECT_FALSE(buffer.has_errors());
|
||||
EXPECT_THAT(
|
||||
buffer,
|
||||
@@ -140,7 +124,7 @@ TEST_F(LexerTest, TracksLinesAndColumnsCRLF) {
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, InvalidCR) {
|
||||
auto buffer = Lex("\n ;;\r ;\n x");
|
||||
auto& buffer = compile_helper_.GetTokenizedBuffer("\n ;;\r ;\n x");
|
||||
EXPECT_TRUE(buffer.has_errors());
|
||||
EXPECT_THAT(
|
||||
buffer,
|
||||
@@ -162,7 +146,7 @@ TEST_F(LexerTest, InvalidCR) {
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, InvalidLFCR) {
|
||||
auto buffer = Lex("\n ;;\n\r ;\n x");
|
||||
auto& buffer = compile_helper_.GetTokenizedBuffer("\n ;;\n\r ;\n x");
|
||||
EXPECT_TRUE(buffer.has_errors());
|
||||
EXPECT_THAT(
|
||||
buffer,
|
||||
@@ -184,7 +168,9 @@ TEST_F(LexerTest, InvalidLFCR) {
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, HandlesNumericLiteral) {
|
||||
auto buffer = Lex("12-578\n 1 2\n0x12_3ABC\n0b10_10_11\n1_234_567\n1.5e9");
|
||||
auto [buffer, value_stores] =
|
||||
compile_helper_.GetTokenizedBufferWithSharedValueStore(
|
||||
"12-578\n 1 2\n0x12_3ABC\n0b10_10_11\n1_234_567\n1.5e9");
|
||||
EXPECT_FALSE(buffer.has_errors());
|
||||
ASSERT_THAT(buffer,
|
||||
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
@@ -237,32 +223,33 @@ TEST_F(LexerTest, HandlesNumericLiteral) {
|
||||
}));
|
||||
auto token_start = buffer.tokens().begin();
|
||||
auto token_12 = token_start + 1;
|
||||
EXPECT_EQ(value_stores_.ints().Get(buffer.GetIntLiteral(*token_12)), 12);
|
||||
EXPECT_EQ(value_stores.ints().Get(buffer.GetIntLiteral(*token_12)), 12);
|
||||
auto token_578 = token_12 + 2;
|
||||
EXPECT_EQ(value_stores_.ints().Get(buffer.GetIntLiteral(*token_578)), 578);
|
||||
EXPECT_EQ(value_stores.ints().Get(buffer.GetIntLiteral(*token_578)), 578);
|
||||
auto token_1 = token_578 + 1;
|
||||
EXPECT_EQ(value_stores_.ints().Get(buffer.GetIntLiteral(*token_1)), 1);
|
||||
EXPECT_EQ(value_stores.ints().Get(buffer.GetIntLiteral(*token_1)), 1);
|
||||
auto token_2 = token_1 + 1;
|
||||
EXPECT_EQ(value_stores_.ints().Get(buffer.GetIntLiteral(*token_2)), 2);
|
||||
EXPECT_EQ(value_stores.ints().Get(buffer.GetIntLiteral(*token_2)), 2);
|
||||
auto token_0x12_3abc = token_2 + 1;
|
||||
EXPECT_EQ(value_stores_.ints().Get(buffer.GetIntLiteral(*token_0x12_3abc)),
|
||||
EXPECT_EQ(value_stores.ints().Get(buffer.GetIntLiteral(*token_0x12_3abc)),
|
||||
0x12'3abc);
|
||||
auto token_0b10_10_11 = token_0x12_3abc + 1;
|
||||
EXPECT_EQ(value_stores_.ints().Get(buffer.GetIntLiteral(*token_0b10_10_11)),
|
||||
EXPECT_EQ(value_stores.ints().Get(buffer.GetIntLiteral(*token_0b10_10_11)),
|
||||
0b10'10'11);
|
||||
auto token_1_234_567 = token_0b10_10_11 + 1;
|
||||
EXPECT_EQ(value_stores_.ints().Get(buffer.GetIntLiteral(*token_1_234_567)),
|
||||
EXPECT_EQ(value_stores.ints().Get(buffer.GetIntLiteral(*token_1_234_567)),
|
||||
1'234'567);
|
||||
auto token_1_5e9 = token_1_234_567 + 1;
|
||||
auto value_1_5e9 =
|
||||
value_stores_.reals().Get(buffer.GetRealLiteral(*token_1_5e9));
|
||||
value_stores.reals().Get(buffer.GetRealLiteral(*token_1_5e9));
|
||||
EXPECT_EQ(value_1_5e9.mantissa.getZExtValue(), 15);
|
||||
EXPECT_EQ(value_1_5e9.exponent.getSExtValue(), 8);
|
||||
EXPECT_EQ(value_1_5e9.is_decimal, true);
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, HandlesInvalidNumericLiterals) {
|
||||
auto buffer = Lex("14x 15_49 0x3.5q 0x3_4.5_6 0ops");
|
||||
auto& buffer =
|
||||
compile_helper_.GetTokenizedBuffer("14x 15_49 0x3.5q 0x3_4.5_6 0ops");
|
||||
EXPECT_TRUE(buffer.has_errors());
|
||||
ASSERT_THAT(buffer,
|
||||
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
@@ -312,7 +299,7 @@ TEST_F(LexerTest, SplitsNumericLiteralsProperly) {
|
||||
12e+1
|
||||
13._
|
||||
)";
|
||||
auto buffer = Lex(source_text);
|
||||
auto& buffer = compile_helper_.GetTokenizedBuffer(source_text);
|
||||
EXPECT_TRUE(buffer.has_errors());
|
||||
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
@@ -370,7 +357,8 @@ TEST_F(LexerTest, SplitsNumericLiteralsProperly) {
|
||||
|
||||
TEST_F(LexerTest, HandlesGarbageCharacters) {
|
||||
constexpr char GarbageText[] = "$$💩-$\n$\0$12$\n\\\"\\\n\"x";
|
||||
auto buffer = Lex(llvm::StringRef(GarbageText, sizeof(GarbageText) - 1));
|
||||
auto& buffer = compile_helper_.GetTokenizedBuffer(
|
||||
llvm::StringRef(GarbageText, sizeof(GarbageText) - 1));
|
||||
EXPECT_TRUE(buffer.has_errors());
|
||||
EXPECT_THAT(
|
||||
buffer,
|
||||
@@ -403,104 +391,104 @@ TEST_F(LexerTest, Symbols) {
|
||||
// We don't need to exhaustively test symbols here as they're handled with
|
||||
// common code, but we want to check specific patterns to verify things like
|
||||
// max-munch rule and handling of interesting symbols.
|
||||
auto buffer = Lex("<<<");
|
||||
EXPECT_FALSE(buffer.has_errors());
|
||||
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::LessLess},
|
||||
{.kind = TokenKind::Less},
|
||||
{.kind = TokenKind::FileEnd},
|
||||
}));
|
||||
auto& buffer1 = compile_helper_.GetTokenizedBuffer("<<<");
|
||||
EXPECT_FALSE(buffer1.has_errors());
|
||||
EXPECT_THAT(buffer1, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::LessLess},
|
||||
{.kind = TokenKind::Less},
|
||||
{.kind = TokenKind::FileEnd},
|
||||
}));
|
||||
|
||||
buffer = Lex("<<=>>");
|
||||
EXPECT_FALSE(buffer.has_errors());
|
||||
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::LessLessEqual},
|
||||
{.kind = TokenKind::GreaterGreater},
|
||||
{.kind = TokenKind::FileEnd},
|
||||
}));
|
||||
auto& buffer2 = compile_helper_.GetTokenizedBuffer("<<=>>");
|
||||
EXPECT_FALSE(buffer2.has_errors());
|
||||
EXPECT_THAT(buffer2, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::LessLessEqual},
|
||||
{.kind = TokenKind::GreaterGreater},
|
||||
{.kind = TokenKind::FileEnd},
|
||||
}));
|
||||
|
||||
buffer = Lex("< <=> >");
|
||||
EXPECT_FALSE(buffer.has_errors());
|
||||
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::Less},
|
||||
{.kind = TokenKind::LessEqualGreater},
|
||||
{.kind = TokenKind::Greater},
|
||||
{.kind = TokenKind::FileEnd},
|
||||
}));
|
||||
auto& buffer3 = compile_helper_.GetTokenizedBuffer("< <=> >");
|
||||
EXPECT_FALSE(buffer3.has_errors());
|
||||
EXPECT_THAT(buffer3, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::Less},
|
||||
{.kind = TokenKind::LessEqualGreater},
|
||||
{.kind = TokenKind::Greater},
|
||||
{.kind = TokenKind::FileEnd},
|
||||
}));
|
||||
|
||||
buffer = Lex("\\/?@&^!");
|
||||
EXPECT_FALSE(buffer.has_errors());
|
||||
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::Backslash},
|
||||
{.kind = TokenKind::Slash},
|
||||
{.kind = TokenKind::Question},
|
||||
{.kind = TokenKind::At},
|
||||
{.kind = TokenKind::Amp},
|
||||
{.kind = TokenKind::Caret},
|
||||
{.kind = TokenKind::Exclaim},
|
||||
{.kind = TokenKind::FileEnd},
|
||||
}));
|
||||
auto& buffer4 = compile_helper_.GetTokenizedBuffer("\\/?@&^!");
|
||||
EXPECT_FALSE(buffer4.has_errors());
|
||||
EXPECT_THAT(buffer4, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::Backslash},
|
||||
{.kind = TokenKind::Slash},
|
||||
{.kind = TokenKind::Question},
|
||||
{.kind = TokenKind::At},
|
||||
{.kind = TokenKind::Amp},
|
||||
{.kind = TokenKind::Caret},
|
||||
{.kind = TokenKind::Exclaim},
|
||||
{.kind = TokenKind::FileEnd},
|
||||
}));
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, Parens) {
|
||||
auto buffer = Lex("()");
|
||||
EXPECT_FALSE(buffer.has_errors());
|
||||
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::OpenParen},
|
||||
{.kind = TokenKind::CloseParen},
|
||||
{.kind = TokenKind::FileEnd},
|
||||
}));
|
||||
auto& buffer1 = compile_helper_.GetTokenizedBuffer("()");
|
||||
EXPECT_FALSE(buffer1.has_errors());
|
||||
EXPECT_THAT(buffer1, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::OpenParen},
|
||||
{.kind = TokenKind::CloseParen},
|
||||
{.kind = TokenKind::FileEnd},
|
||||
}));
|
||||
|
||||
buffer = Lex("((()()))");
|
||||
EXPECT_FALSE(buffer.has_errors());
|
||||
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::OpenParen},
|
||||
{.kind = TokenKind::OpenParen},
|
||||
{.kind = TokenKind::OpenParen},
|
||||
{.kind = TokenKind::CloseParen},
|
||||
{.kind = TokenKind::OpenParen},
|
||||
{.kind = TokenKind::CloseParen},
|
||||
{.kind = TokenKind::CloseParen},
|
||||
{.kind = TokenKind::CloseParen},
|
||||
{.kind = TokenKind::FileEnd},
|
||||
}));
|
||||
auto& buffer2 = compile_helper_.GetTokenizedBuffer("((()()))");
|
||||
EXPECT_FALSE(buffer2.has_errors());
|
||||
EXPECT_THAT(buffer2, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::OpenParen},
|
||||
{.kind = TokenKind::OpenParen},
|
||||
{.kind = TokenKind::OpenParen},
|
||||
{.kind = TokenKind::CloseParen},
|
||||
{.kind = TokenKind::OpenParen},
|
||||
{.kind = TokenKind::CloseParen},
|
||||
{.kind = TokenKind::CloseParen},
|
||||
{.kind = TokenKind::CloseParen},
|
||||
{.kind = TokenKind::FileEnd},
|
||||
}));
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, CurlyBraces) {
|
||||
auto buffer = Lex("{}");
|
||||
EXPECT_FALSE(buffer.has_errors());
|
||||
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::OpenCurlyBrace},
|
||||
{.kind = TokenKind::CloseCurlyBrace},
|
||||
{.kind = TokenKind::FileEnd},
|
||||
}));
|
||||
auto& buffer1 = compile_helper_.GetTokenizedBuffer("{}");
|
||||
EXPECT_FALSE(buffer1.has_errors());
|
||||
EXPECT_THAT(buffer1, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::OpenCurlyBrace},
|
||||
{.kind = TokenKind::CloseCurlyBrace},
|
||||
{.kind = TokenKind::FileEnd},
|
||||
}));
|
||||
|
||||
buffer = Lex("{{{}{}}}");
|
||||
EXPECT_FALSE(buffer.has_errors());
|
||||
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::OpenCurlyBrace},
|
||||
{.kind = TokenKind::OpenCurlyBrace},
|
||||
{.kind = TokenKind::OpenCurlyBrace},
|
||||
{.kind = TokenKind::CloseCurlyBrace},
|
||||
{.kind = TokenKind::OpenCurlyBrace},
|
||||
{.kind = TokenKind::CloseCurlyBrace},
|
||||
{.kind = TokenKind::CloseCurlyBrace},
|
||||
{.kind = TokenKind::CloseCurlyBrace},
|
||||
{.kind = TokenKind::FileEnd},
|
||||
}));
|
||||
auto& buffer2 = compile_helper_.GetTokenizedBuffer("{{{}{}}}");
|
||||
EXPECT_FALSE(buffer2.has_errors());
|
||||
EXPECT_THAT(buffer2, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::OpenCurlyBrace},
|
||||
{.kind = TokenKind::OpenCurlyBrace},
|
||||
{.kind = TokenKind::OpenCurlyBrace},
|
||||
{.kind = TokenKind::CloseCurlyBrace},
|
||||
{.kind = TokenKind::OpenCurlyBrace},
|
||||
{.kind = TokenKind::CloseCurlyBrace},
|
||||
{.kind = TokenKind::CloseCurlyBrace},
|
||||
{.kind = TokenKind::CloseCurlyBrace},
|
||||
{.kind = TokenKind::FileEnd},
|
||||
}));
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, MatchingGroups) {
|
||||
{
|
||||
TokenizedBuffer buffer = Lex("(){}");
|
||||
auto& buffer = compile_helper_.GetTokenizedBuffer("(){}");
|
||||
ASSERT_FALSE(buffer.has_errors());
|
||||
auto it = ++buffer.tokens().begin();
|
||||
auto open_paren_token = *it++;
|
||||
@@ -521,14 +509,15 @@ TEST_F(LexerTest, MatchingGroups) {
|
||||
}
|
||||
|
||||
{
|
||||
TokenizedBuffer buffer = Lex("({x}){(y)} {{((z))}}");
|
||||
auto [buffer, value_stores] =
|
||||
compile_helper_.GetTokenizedBufferWithSharedValueStore(
|
||||
"({x}){(y)} {{((z))}}");
|
||||
ASSERT_FALSE(buffer.has_errors());
|
||||
auto it = ++buffer.tokens().begin();
|
||||
auto open_paren_token = *it++;
|
||||
auto open_curly_token = *it++;
|
||||
|
||||
ASSERT_EQ("x",
|
||||
value_stores_.identifiers().Get(buffer.GetIdentifier(*it++)));
|
||||
ASSERT_EQ("x", value_stores.identifiers().Get(buffer.GetIdentifier(*it++)));
|
||||
auto close_curly_token = *it++;
|
||||
auto close_paren_token = *it++;
|
||||
EXPECT_EQ(close_paren_token,
|
||||
@@ -542,8 +531,7 @@ TEST_F(LexerTest, MatchingGroups) {
|
||||
|
||||
open_curly_token = *it++;
|
||||
open_paren_token = *it++;
|
||||
ASSERT_EQ("y",
|
||||
value_stores_.identifiers().Get(buffer.GetIdentifier(*it++)));
|
||||
ASSERT_EQ("y", value_stores.identifiers().Get(buffer.GetIdentifier(*it++)));
|
||||
close_paren_token = *it++;
|
||||
close_curly_token = *it++;
|
||||
EXPECT_EQ(close_curly_token,
|
||||
@@ -559,8 +547,7 @@ TEST_F(LexerTest, MatchingGroups) {
|
||||
auto inner_open_curly_token = *it++;
|
||||
open_paren_token = *it++;
|
||||
auto inner_open_paren_token = *it++;
|
||||
ASSERT_EQ("z",
|
||||
value_stores_.identifiers().Get(buffer.GetIdentifier(*it++)));
|
||||
ASSERT_EQ("z", value_stores.identifiers().Get(buffer.GetIdentifier(*it++)));
|
||||
auto inner_close_paren_token = *it++;
|
||||
close_paren_token = *it++;
|
||||
auto inner_close_curly_token = *it++;
|
||||
@@ -589,26 +576,26 @@ TEST_F(LexerTest, MatchingGroups) {
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, MismatchedGroups) {
|
||||
auto buffer = Lex("{");
|
||||
EXPECT_TRUE(buffer.has_errors());
|
||||
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::Error, .text = "{"},
|
||||
{.kind = TokenKind::FileEnd},
|
||||
}));
|
||||
auto& buffer1 = compile_helper_.GetTokenizedBuffer("{");
|
||||
EXPECT_TRUE(buffer1.has_errors());
|
||||
EXPECT_THAT(buffer1, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::Error, .text = "{"},
|
||||
{.kind = TokenKind::FileEnd},
|
||||
}));
|
||||
|
||||
buffer = Lex("}");
|
||||
EXPECT_TRUE(buffer.has_errors());
|
||||
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::Error, .text = "}"},
|
||||
{.kind = TokenKind::FileEnd},
|
||||
}));
|
||||
auto& buffer2 = compile_helper_.GetTokenizedBuffer("}");
|
||||
EXPECT_TRUE(buffer2.has_errors());
|
||||
EXPECT_THAT(buffer2, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::Error, .text = "}"},
|
||||
{.kind = TokenKind::FileEnd},
|
||||
}));
|
||||
|
||||
buffer = Lex("{(}");
|
||||
EXPECT_TRUE(buffer.has_errors());
|
||||
auto& buffer3 = compile_helper_.GetTokenizedBuffer("{(}");
|
||||
EXPECT_TRUE(buffer3.has_errors());
|
||||
EXPECT_THAT(
|
||||
buffer,
|
||||
buffer3,
|
||||
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::OpenCurlyBrace, .column = 1},
|
||||
@@ -618,10 +605,10 @@ TEST_F(LexerTest, MismatchedGroups) {
|
||||
{.kind = TokenKind::FileEnd},
|
||||
}));
|
||||
|
||||
buffer = Lex(")({)");
|
||||
EXPECT_TRUE(buffer.has_errors());
|
||||
auto& buffer4 = compile_helper_.GetTokenizedBuffer(")({)");
|
||||
EXPECT_TRUE(buffer4.has_errors());
|
||||
EXPECT_THAT(
|
||||
buffer,
|
||||
buffer4,
|
||||
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::Error, .column = 1, .text = ")"},
|
||||
@@ -634,7 +621,7 @@ TEST_F(LexerTest, MismatchedGroups) {
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, Whitespace) {
|
||||
auto buffer = Lex("{( } {(");
|
||||
auto& buffer = compile_helper_.GetTokenizedBuffer("{( } {(");
|
||||
|
||||
// Whether there should be whitespace before/after each token.
|
||||
bool space[] = {false,
|
||||
@@ -675,7 +662,7 @@ TEST_F(LexerTest, Keywords) {
|
||||
#include "toolchain/lex/token_kind.def"
|
||||
};
|
||||
for (const auto& keyword : keywords) {
|
||||
auto buffer = Lex(keyword.fixed_spelling());
|
||||
auto& buffer = compile_helper_.GetTokenizedBuffer(keyword.fixed_spelling());
|
||||
EXPECT_FALSE(buffer.has_errors());
|
||||
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
@@ -686,10 +673,10 @@ TEST_F(LexerTest, Keywords) {
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, Comments) {
|
||||
auto buffer = Lex(" ;\n // foo\n ;\n");
|
||||
EXPECT_FALSE(buffer.has_errors());
|
||||
auto& buffer1 = compile_helper_.GetTokenizedBuffer(" ;\n // foo\n ;\n");
|
||||
EXPECT_FALSE(buffer1.has_errors());
|
||||
EXPECT_THAT(
|
||||
buffer,
|
||||
buffer1,
|
||||
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
|
||||
{.kind = TokenKind::Semi, .line = 1, .column = 2, .indent_column = 2},
|
||||
@@ -697,25 +684,26 @@ TEST_F(LexerTest, Comments) {
|
||||
{.kind = TokenKind::FileEnd, .line = 3, .column = 4},
|
||||
}));
|
||||
|
||||
buffer = Lex("// foo\n//\n// bar");
|
||||
EXPECT_FALSE(buffer.has_errors());
|
||||
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::FileEnd}}));
|
||||
auto& buffer2 = compile_helper_.GetTokenizedBuffer("// foo\n//\n// bar");
|
||||
EXPECT_FALSE(buffer2.has_errors());
|
||||
EXPECT_THAT(buffer2, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::FileEnd}}));
|
||||
|
||||
// Make sure weird characters aren't a problem.
|
||||
buffer = Lex(" // foo#$!^?@-_💩🍫⃠ [̲̅$̲̅(̲̅ ͡° ͜ʖ ͡°̲̅)̲̅$̲̅]");
|
||||
EXPECT_FALSE(buffer.has_errors());
|
||||
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::FileEnd}}));
|
||||
auto& buffer3 =
|
||||
compile_helper_.GetTokenizedBuffer(" // foo#$!^?@-_💩🍫⃠ [̲̅$̲̅(̲̅ ͡° ͜ʖ ͡°̲̅)̲̅$̲̅]");
|
||||
EXPECT_FALSE(buffer3.has_errors());
|
||||
EXPECT_THAT(buffer3, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::FileEnd}}));
|
||||
|
||||
// Make sure we can lex a comment at the end of the input.
|
||||
buffer = Lex("//");
|
||||
EXPECT_FALSE(buffer.has_errors());
|
||||
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::FileEnd}}));
|
||||
auto& buffer4 = compile_helper_.GetTokenizedBuffer("//");
|
||||
EXPECT_FALSE(buffer4.has_errors());
|
||||
EXPECT_THAT(buffer4, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::FileEnd}}));
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, InvalidComments) {
|
||||
@@ -726,53 +714,54 @@ TEST_F(LexerTest, InvalidComments) {
|
||||
" //world",
|
||||
};
|
||||
for (llvm::StringLiteral testcase : testcases) {
|
||||
auto buffer = Lex(testcase);
|
||||
auto& buffer = compile_helper_.GetTokenizedBuffer(testcase);
|
||||
EXPECT_TRUE(buffer.has_errors());
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, Identifiers) {
|
||||
auto buffer = Lex(" foobar");
|
||||
EXPECT_FALSE(buffer.has_errors());
|
||||
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::Identifier,
|
||||
.column = 4,
|
||||
.indent_column = 4,
|
||||
.text = "foobar"},
|
||||
{.kind = TokenKind::FileEnd},
|
||||
}));
|
||||
auto& buffer1 = compile_helper_.GetTokenizedBuffer(" foobar");
|
||||
EXPECT_FALSE(buffer1.has_errors());
|
||||
EXPECT_THAT(buffer1, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::Identifier,
|
||||
.column = 4,
|
||||
.indent_column = 4,
|
||||
.text = "foobar"},
|
||||
{.kind = TokenKind::FileEnd},
|
||||
}));
|
||||
|
||||
// Check different kinds of identifier character sequences.
|
||||
buffer = Lex("_foo_bar");
|
||||
EXPECT_FALSE(buffer.has_errors());
|
||||
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::Identifier, .text = "_foo_bar"},
|
||||
{.kind = TokenKind::FileEnd},
|
||||
}));
|
||||
auto& buffer2 = compile_helper_.GetTokenizedBuffer("_foo_bar");
|
||||
EXPECT_FALSE(buffer2.has_errors());
|
||||
EXPECT_THAT(buffer2, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::Identifier, .text = "_foo_bar"},
|
||||
{.kind = TokenKind::FileEnd},
|
||||
}));
|
||||
|
||||
buffer = Lex("foo2bar00");
|
||||
EXPECT_FALSE(buffer.has_errors());
|
||||
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::Identifier, .text = "foo2bar00"},
|
||||
{.kind = TokenKind::FileEnd},
|
||||
}));
|
||||
auto& buffer3 = compile_helper_.GetTokenizedBuffer("foo2bar00");
|
||||
EXPECT_FALSE(buffer3.has_errors());
|
||||
EXPECT_THAT(buffer3, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::Identifier, .text = "foo2bar00"},
|
||||
{.kind = TokenKind::FileEnd},
|
||||
}));
|
||||
|
||||
// Check that we can parse identifiers that start with a keyword.
|
||||
buffer = Lex("fnord");
|
||||
EXPECT_FALSE(buffer.has_errors());
|
||||
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::Identifier, .text = "fnord"},
|
||||
{.kind = TokenKind::FileEnd},
|
||||
}));
|
||||
auto& buffer4 = compile_helper_.GetTokenizedBuffer("fnord");
|
||||
EXPECT_FALSE(buffer4.has_errors());
|
||||
EXPECT_THAT(buffer4, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart},
|
||||
{.kind = TokenKind::Identifier, .text = "fnord"},
|
||||
{.kind = TokenKind::FileEnd},
|
||||
}));
|
||||
|
||||
// Check multiple identifiers with indent and interning.
|
||||
buffer = Lex(" foo;bar\nbar \n foo\tfoo");
|
||||
EXPECT_FALSE(buffer.has_errors());
|
||||
EXPECT_THAT(buffer,
|
||||
auto& buffer5 =
|
||||
compile_helper_.GetTokenizedBuffer(" foo;bar\nbar \n foo\tfoo");
|
||||
EXPECT_FALSE(buffer5.has_errors());
|
||||
EXPECT_THAT(buffer5,
|
||||
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
|
||||
{.kind = TokenKind::Identifier,
|
||||
@@ -823,7 +812,8 @@ TEST_F(LexerTest, StringLiterals) {
|
||||
"""x"""
|
||||
)";
|
||||
|
||||
auto buffer = Lex(testcase);
|
||||
auto [buffer, value_stores] =
|
||||
compile_helper_.GetTokenizedBufferWithSharedValueStore(testcase);
|
||||
EXPECT_FALSE(buffer.has_errors());
|
||||
EXPECT_THAT(buffer,
|
||||
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
@@ -832,13 +822,13 @@ TEST_F(LexerTest, StringLiterals) {
|
||||
.line = 2,
|
||||
.column = 5,
|
||||
.indent_column = 5,
|
||||
.value_stores = &value_stores_,
|
||||
.value_stores = &value_stores,
|
||||
.string_contents = {"hello world\n"}},
|
||||
{.kind = TokenKind::StringLiteral,
|
||||
.line = 4,
|
||||
.column = 5,
|
||||
.indent_column = 5,
|
||||
.value_stores = &value_stores_,
|
||||
.value_stores = &value_stores,
|
||||
.string_contents = {" test \xAB\n"}},
|
||||
{.kind = TokenKind::Identifier,
|
||||
.line = 7,
|
||||
@@ -849,19 +839,19 @@ TEST_F(LexerTest, StringLiterals) {
|
||||
.line = 9,
|
||||
.column = 7,
|
||||
.indent_column = 7,
|
||||
.value_stores = &value_stores_,
|
||||
.value_stores = &value_stores,
|
||||
.string_contents = {"\""}},
|
||||
{.kind = TokenKind::StringLiteral,
|
||||
.line = 11,
|
||||
.column = 5,
|
||||
.indent_column = 5,
|
||||
.value_stores = &value_stores_,
|
||||
.value_stores = &value_stores,
|
||||
.string_contents = llvm::StringLiteral::withInnerNUL("\0")},
|
||||
{.kind = TokenKind::StringLiteral,
|
||||
.line = 13,
|
||||
.column = 5,
|
||||
.indent_column = 5,
|
||||
.value_stores = &value_stores_,
|
||||
.value_stores = &value_stores,
|
||||
.string_contents = {"\\0\"foo\"\\1"}},
|
||||
|
||||
// """x""" is three string literals, not one invalid
|
||||
@@ -870,19 +860,19 @@ TEST_F(LexerTest, StringLiterals) {
|
||||
.line = 15,
|
||||
.column = 5,
|
||||
.indent_column = 5,
|
||||
.value_stores = &value_stores_,
|
||||
.value_stores = &value_stores,
|
||||
.string_contents = {""}},
|
||||
{.kind = TokenKind::StringLiteral,
|
||||
.line = 15,
|
||||
.column = 7,
|
||||
.indent_column = 5,
|
||||
.value_stores = &value_stores_,
|
||||
.value_stores = &value_stores,
|
||||
.string_contents = {"x"}},
|
||||
{.kind = TokenKind::StringLiteral,
|
||||
.line = 15,
|
||||
.column = 10,
|
||||
.indent_column = 5,
|
||||
.value_stores = &value_stores_,
|
||||
.value_stores = &value_stores,
|
||||
.string_contents = {""}},
|
||||
{.kind = TokenKind::FileEnd, .line = 16, .column = 3},
|
||||
}));
|
||||
@@ -909,7 +899,7 @@ TEST_F(LexerTest, InvalidStringLiterals) {
|
||||
|
||||
for (llvm::StringLiteral test : invalid) {
|
||||
SCOPED_TRACE(test);
|
||||
auto buffer = Lex(test);
|
||||
auto& buffer = compile_helper_.GetTokenizedBuffer(test);
|
||||
EXPECT_TRUE(buffer.has_errors());
|
||||
|
||||
// We should have formed at least one error token.
|
||||
@@ -932,7 +922,8 @@ TEST_F(LexerTest, TypeLiterals) {
|
||||
s1
|
||||
)";
|
||||
|
||||
auto buffer = Lex(testcase);
|
||||
auto [buffer, value_stores] =
|
||||
compile_helper_.GetTokenizedBufferWithSharedValueStore(testcase);
|
||||
EXPECT_FALSE(buffer.has_errors());
|
||||
ASSERT_THAT(buffer,
|
||||
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
@@ -1017,7 +1008,7 @@ TEST_F(LexerTest, TypeLiterals) {
|
||||
|
||||
auto type_size = [&](int token_index) {
|
||||
auto token = buffer.tokens().begin()[token_index];
|
||||
return value_stores_.ints().Get(buffer.GetTypeLiteralSize(token));
|
||||
return value_stores.ints().Get(buffer.GetTypeLiteralSize(token));
|
||||
};
|
||||
|
||||
EXPECT_EQ(type_size(2), 1);
|
||||
@@ -1040,7 +1031,7 @@ TEST_F(LexerTest, TypeLiteralTooManyDigits) {
|
||||
HandleDiagnostic(IsSingleDiagnostic(
|
||||
DiagnosticKind::TooManyDigits, DiagnosticLevel::Error, 1, 2,
|
||||
HasSubstr(llvm::formatv(" {0} ", Count)))));
|
||||
auto buffer = Lex(code, consumer);
|
||||
auto& buffer = compile_helper_.GetTokenizedBuffer(code, &consumer);
|
||||
EXPECT_TRUE(buffer.has_errors());
|
||||
ASSERT_THAT(buffer,
|
||||
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
@@ -1064,7 +1055,7 @@ TEST_F(LexerTest, DiagnosticTrailingComment) {
|
||||
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
|
||||
DiagnosticKind::TrailingComment,
|
||||
DiagnosticLevel::Error, 3, 19, _)));
|
||||
Lex(testcase, consumer);
|
||||
compile_helper_.GetTokenizedBuffer(testcase, &consumer);
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, DiagnosticWhitespace) {
|
||||
@@ -1072,7 +1063,7 @@ TEST_F(LexerTest, DiagnosticWhitespace) {
|
||||
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
|
||||
DiagnosticKind::NoWhitespaceAfterCommentIntroducer,
|
||||
DiagnosticLevel::Error, 1, 3, _)));
|
||||
Lex("//no space after comment", consumer);
|
||||
compile_helper_.GetTokenizedBuffer("//no space after comment", &consumer);
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, DiagnosticUnrecognizedEscape) {
|
||||
@@ -1080,7 +1071,7 @@ TEST_F(LexerTest, DiagnosticUnrecognizedEscape) {
|
||||
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
|
||||
DiagnosticKind::UnknownEscapeSequence,
|
||||
DiagnosticLevel::Error, 1, 8, HasSubstr("`b`"))));
|
||||
Lex(R"("hello\bworld")", consumer);
|
||||
compile_helper_.GetTokenizedBuffer(R"("hello\bworld")", &consumer);
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, DiagnosticBadHex) {
|
||||
@@ -1088,7 +1079,7 @@ TEST_F(LexerTest, DiagnosticBadHex) {
|
||||
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
|
||||
DiagnosticKind::HexadecimalEscapeMissingDigits,
|
||||
DiagnosticLevel::Error, 1, 9, _)));
|
||||
Lex(R"("hello\xabworld")", consumer);
|
||||
compile_helper_.GetTokenizedBuffer(R"("hello\xabworld")", &consumer);
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, DiagnosticInvalidDigit) {
|
||||
@@ -1096,7 +1087,7 @@ TEST_F(LexerTest, DiagnosticInvalidDigit) {
|
||||
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
|
||||
DiagnosticKind::InvalidDigit,
|
||||
DiagnosticLevel::Error, 1, 6, HasSubstr("'a'"))));
|
||||
Lex("0x123abc", consumer);
|
||||
compile_helper_.GetTokenizedBuffer("0x123abc", &consumer);
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, DiagnosticMissingTerminator) {
|
||||
@@ -1104,7 +1095,7 @@ TEST_F(LexerTest, DiagnosticMissingTerminator) {
|
||||
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
|
||||
DiagnosticKind::UnterminatedString,
|
||||
DiagnosticLevel::Error, 1, 1, _)));
|
||||
Lex(R"(#" ")", consumer);
|
||||
compile_helper_.GetTokenizedBuffer(R"(#" ")", &consumer);
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, DiagnosticUnrecognizedChar) {
|
||||
@@ -1112,12 +1103,13 @@ TEST_F(LexerTest, DiagnosticUnrecognizedChar) {
|
||||
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
|
||||
DiagnosticKind::UnrecognizedCharacters,
|
||||
DiagnosticLevel::Error, 1, 1, _)));
|
||||
Lex("\b", consumer);
|
||||
compile_helper_.GetTokenizedBuffer("\b", &consumer);
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, PrintingOutputYaml) {
|
||||
// Test that we can parse this into YAML and verify line and indent data.
|
||||
auto buffer = Lex("\n ;\n\n\n; ;\n\n\n\n\n\n\n\n\n\n\n");
|
||||
auto& buffer =
|
||||
compile_helper_.GetTokenizedBuffer("\n ;\n\n\n; ;\n\n\n\n\n\n\n\n\n\n\n");
|
||||
ASSERT_FALSE(buffer.has_errors());
|
||||
TestRawOstream print_stream;
|
||||
buffer.Print(print_stream);
|
||||
@@ -1125,7 +1117,7 @@ TEST_F(LexerTest, PrintingOutputYaml) {
|
||||
EXPECT_THAT(
|
||||
Yaml::Value::FromText(print_stream.TakeStr()),
|
||||
IsYaml(ElementsAre(Yaml::Sequence(ElementsAre(Yaml::Mapping(ElementsAre(
|
||||
Pair("filename", source_storage_.front().filename().str()),
|
||||
Pair("filename", buffer.source().filename().str()),
|
||||
Pair("tokens", Yaml::Sequence(ElementsAre(
|
||||
Yaml::Mapping(ElementsAre(
|
||||
Pair("index", "0"), Pair("kind", "FileStart"),
|
||||
|
||||
Reference in New Issue
Block a user