carbon-language/carbon-lang
1
0
Fork 0
mirror of https://github.com/carbon-language/carbon-lang.git synced 2026-10-04 16:51:05 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Align on FileStart/FileEnd for naming. (#3428)

Browse Source 
The lexer has been using EndOfFile form (stemming from EOF), parser went
to FileEnd form. This consolidates on FileEnd form.
This commit is contained in:
Jon Ross-Perkins
2023-11-29 16:36:57 +00:00
committed by GitHub
parent 239f8030a4
commit 3f208e27f9
28 changed files with 147 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files
toolchain/lex/tokenized_buffer_test.cpp
+72 -73
View File
@@ -60,7 +60,7 @@ TEST_F(LexerTest, HandlesEmptyBuffer) {
auto buffer = Lex("");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile}, {TokenKind::EndOfFile}}));
{TokenKind::FileStart}, {TokenKind::FileEnd}}));
}
TEST_F(LexerTest, TracksLinesAndColumns) {
@@ -69,7 +69,7 @@ TEST_F(LexerTest, TracksLinesAndColumns) {
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::StartOfFile,
{.kind = TokenKind::FileStart,
.line = 1,
.column = 1,
.indent_column = 1},
@@ -96,7 +96,7 @@ TEST_F(LexerTest, TracksLinesAndColumns) {
.column = 6,
.indent_column = 11,
.text = "y"},
{.kind = TokenKind::EndOfFile, .line = 6, .column = 7},
{.kind = TokenKind::FileEnd, .line = 6, .column = 7},
}));
}
@@ -105,7 +105,7 @@ TEST_F(LexerTest, HandlesNumericLiteral) {
EXPECT_FALSE(buffer.has_errors());
ASSERT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::StartOfFile, .line = 1, .column = 1},
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::IntegerLiteral,
.line = 1,
.column = 1,
@@ -150,7 +150,7 @@ TEST_F(LexerTest, HandlesNumericLiteral) {
.column = 1,
.indent_column = 1,
.text = "1.5e9"},
{.kind = TokenKind::EndOfFile, .line = 6, .column = 6},
{.kind = TokenKind::FileEnd, .line = 6, .column = 6},
}));
auto token_start = buffer.tokens().begin();
auto token_12 = token_start + 1;
@@ -190,7 +190,7 @@ TEST_F(LexerTest, HandlesInvalidNumericLiterals) {
EXPECT_TRUE(buffer.has_errors());
ASSERT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::StartOfFile, .line = 1, .column = 1},
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::Error,
.line = 1,
.column = 1,
@@ -216,7 +216,7 @@ TEST_F(LexerTest, HandlesInvalidNumericLiterals) {
.column = 28,
.indent_column = 1,
.text = "0ops"},
{.kind = TokenKind::EndOfFile, .line = 1, .column = 32},
{.kind = TokenKind::FileEnd, .line = 1, .column = 32},
}));
}
@@ -239,7 +239,7 @@ TEST_F(LexerTest, SplitsNumericLiteralsProperly) {
auto buffer = Lex(source_text);
EXPECT_TRUE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::StartOfFile},
{.kind = TokenKind::FileStart},
{.kind = TokenKind::IntegerLiteral, .text = "1"},
{.kind = TokenKind::Period},
// newline
@@ -288,7 +288,7 @@ TEST_F(LexerTest, SplitsNumericLiteralsProperly) {
{.kind = TokenKind::Period},
{.kind = TokenKind::Underscore},
// newline
{.kind = TokenKind::EndOfFile},
{.kind = TokenKind::FileEnd},
}));
}
@@ -299,7 +299,7 @@ TEST_F(LexerTest, HandlesGarbageCharacters) {
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::StartOfFile, .line = 1, .column = 1},
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::Error,
.line = 1,
.column = 1,
@@ -322,7 +322,7 @@ TEST_F(LexerTest, HandlesGarbageCharacters) {
{.kind = TokenKind::Error, .line = 3, .column = 2, .text = "\"\\"},
// newline
{.kind = TokenKind::Error, .line = 4, .column = 1, .text = "\"x"},
{.kind = TokenKind::EndOfFile, .line = 4, .column = 3},
{.kind = TokenKind::FileEnd, .line = 4, .column = 3},
}));
}
@@ -333,35 +333,35 @@ TEST_F(LexerTest, Symbols) {
auto buffer = Lex("<<<");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{TokenKind::LessLess},
{TokenKind::Less},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
buffer = Lex("<<=>>");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{TokenKind::LessLessEqual},
{TokenKind::GreaterGreater},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
buffer = Lex("< <=> >");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{TokenKind::Less},
{TokenKind::LessEqualGreater},
{TokenKind::Greater},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
buffer = Lex("\\/?@&^!");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{TokenKind::Backslash},
{TokenKind::Slash},
{TokenKind::Question},
@@ -369,7 +369,7 @@ TEST_F(LexerTest, Symbols) {
{TokenKind::Amp},
{TokenKind::Caret},
{TokenKind::Exclaim},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
}
@@ -377,16 +377,16 @@ TEST_F(LexerTest, Parens) {
auto buffer = Lex("()");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{TokenKind::OpenParen},
{TokenKind::CloseParen},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
buffer = Lex("((()()))");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{TokenKind::OpenParen},
{TokenKind::OpenParen},
{TokenKind::OpenParen},
@@ -395,7 +395,7 @@ TEST_F(LexerTest, Parens) {
{TokenKind::CloseParen},
{TokenKind::CloseParen},
{TokenKind::CloseParen},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
}
@@ -403,16 +403,16 @@ TEST_F(LexerTest, CurlyBraces) {
auto buffer = Lex("{}");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{TokenKind::OpenCurlyBrace},
{TokenKind::CloseCurlyBrace},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
buffer = Lex("{{{}{}}}");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{TokenKind::OpenCurlyBrace},
{TokenKind::OpenCurlyBrace},
{TokenKind::OpenCurlyBrace},
@@ -421,7 +421,7 @@ TEST_F(LexerTest, CurlyBraces) {
{TokenKind::CloseCurlyBrace},
{TokenKind::CloseCurlyBrace},
{TokenKind::CloseCurlyBrace},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
}
@@ -443,7 +443,7 @@ TEST_F(LexerTest, MatchingGroups) {
EXPECT_EQ(open_curly_token,
buffer.GetMatchedOpeningToken(close_curly_token));
auto eof_token = *it++;
EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::EndOfFile);
EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::FileEnd);
EXPECT_EQ(buffer.tokens().end(), it);
}
@@ -510,7 +510,7 @@ TEST_F(LexerTest, MatchingGroups) {
buffer.GetMatchedOpeningToken(inner_close_paren_token));
auto eof_token = *it++;
EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::EndOfFile);
EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::FileEnd);
EXPECT_EQ(buffer.tokens().end(), it);
}
}
@@ -520,18 +520,18 @@ TEST_F(LexerTest, MismatchedGroups) {
EXPECT_TRUE(buffer.has_errors());
EXPECT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{TokenKind::OpenCurlyBrace},
{.kind = TokenKind::CloseCurlyBrace, .recovery = true},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
buffer = Lex("}");
EXPECT_TRUE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{.kind = TokenKind::Error, .text = "}"},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
buffer = Lex("{(}");
@@ -539,12 +539,12 @@ TEST_F(LexerTest, MismatchedGroups) {
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{.kind = TokenKind::OpenCurlyBrace, .column = 1},
{.kind = TokenKind::OpenParen, .column = 2},
{.kind = TokenKind::CloseParen, .column = 3, .recovery = true},
{.kind = TokenKind::CloseCurlyBrace, .column = 3},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
buffer = Lex(")({)");
@@ -552,13 +552,13 @@ TEST_F(LexerTest, MismatchedGroups) {
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{.kind = TokenKind::Error, .column = 1, .text = ")"},
{.kind = TokenKind::OpenParen, .column = 2},
{.kind = TokenKind::OpenCurlyBrace, .column = 3},
{.kind = TokenKind::CloseCurlyBrace, .column = 4, .recovery = true},
{.kind = TokenKind::CloseParen, .column = 4},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
}
@@ -606,15 +606,15 @@ TEST_F(LexerTest, Keywords) {
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{.kind = TokenKind::Fn, .column = 4, .indent_column = 4},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
buffer = Lex("and or not if else for return var break continue _");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{TokenKind::And},
{TokenKind::Or},
{TokenKind::Not},
@@ -626,7 +626,7 @@ TEST_F(LexerTest, Keywords) {
{TokenKind::Break},
{TokenKind::Continue},
{TokenKind::Underscore},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
}
@@ -636,28 +636,28 @@ TEST_F(LexerTest, Comments) {
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::StartOfFile, .line = 1, .column = 1},
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::Semi, .line = 1, .column = 2, .indent_column = 2},
{.kind = TokenKind::Semi, .line = 3, .column = 3, .indent_column = 3},
{.kind = TokenKind::EndOfFile, .line = 3, .column = 4},
{.kind = TokenKind::FileEnd, .line = 3, .column = 4},
}));
buffer = Lex("// foo\n//\n// bar");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile}, {TokenKind::EndOfFile}}));
{TokenKind::FileStart}, {TokenKind::FileEnd}}));
// Make sure weird characters aren't a problem.
buffer = Lex(" // foo#$!^?@-_💩🍫⃠ [̲̅$̲̅(̲̅ ͡° ͜ʖ ͡°̲̅)̲̅$̲̅]");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile}, {TokenKind::EndOfFile}}));
{TokenKind::FileStart}, {TokenKind::FileEnd}}));
// Make sure we can lex a comment at the end of the input.
buffer = Lex("//");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile}, {TokenKind::EndOfFile}}));
{TokenKind::FileStart}, {TokenKind::FileEnd}}));
}
TEST_F(LexerTest, InvalidComments) {
@@ -677,38 +677,38 @@ TEST_F(LexerTest, Identifiers) {
auto buffer = Lex(" foobar");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{.kind = TokenKind::Identifier,
.column = 4,
.indent_column = 4,
.text = "foobar"},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
// Check different kinds of identifier character sequences.
buffer = Lex("_foo_bar");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{.kind = TokenKind::Identifier, .text = "_foo_bar"},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
buffer = Lex("foo2bar00");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{.kind = TokenKind::Identifier, .text = "foo2bar00"},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
// Check that we can parse identifiers that start with a keyword.
buffer = Lex("fnord");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{.kind = TokenKind::Identifier, .text = "fnord"},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
// Check multiple identifiers with indent and interning.
@@ -716,7 +716,7 @@ TEST_F(LexerTest, Identifiers) {
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::StartOfFile, .line = 1, .column = 1},
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::Identifier,
.line = 1,
.column = 4,
@@ -743,7 +743,7 @@ TEST_F(LexerTest, Identifiers) {
.column = 7,
.indent_column = 3,
.text = "foo"},
{.kind = TokenKind::EndOfFile, .line = 3, .column = 10},
{.kind = TokenKind::FileEnd, .line = 3, .column = 10},
}));
}
@@ -769,7 +769,7 @@ TEST_F(LexerTest, StringLiterals) {
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::StartOfFile, .line = 1, .column = 1},
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::StringLiteral,
.line = 2,
.column = 5,
@@ -826,7 +826,7 @@ TEST_F(LexerTest, StringLiterals) {
.indent_column = 5,
.value_stores = &value_stores_,
.string_contents = {""}},
{.kind = TokenKind::EndOfFile, .line = 16, .column = 3},
{.kind = TokenKind::FileEnd, .line = 16, .column = 3},
}));
}
@@ -878,7 +878,7 @@ TEST_F(LexerTest, TypeLiterals) {
EXPECT_FALSE(buffer.has_errors());
ASSERT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::StartOfFile, .line = 1, .column = 1},
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::Identifier,
.line = 2,
@@ -954,7 +954,7 @@ TEST_F(LexerTest, TypeLiterals) {
.indent_column = 5,
.text = {"s1"}},
{.kind = TokenKind::EndOfFile, .line = 6, .column = 3},
{.kind = TokenKind::FileEnd, .line = 6, .column = 3},
}));
auto token_i1 = buffer.tokens().begin() + 2;
@@ -987,17 +987,16 @@ TEST_F(LexerTest, TypeLiteralTooManyDigits) {
HasSubstr(llvm::formatv(" {0} ", Count)))));
auto buffer = Lex(code, consumer);
EXPECT_TRUE(buffer.has_errors());
ASSERT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::StartOfFile, .line = 1, .column = 1},
{.kind = TokenKind::Error,
.line = 1,
.column = 1,
.indent_column = 1,
.text = {code}},
{.kind = TokenKind::EndOfFile, .line = 1, .column = Count + 2},
}));
ASSERT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::Error,
.line = 1,
.column = 1,
.indent_column = 1,
.text = {code}},
{.kind = TokenKind::FileEnd, .line = 1, .column = Count + 2},
}));
}
TEST_F(LexerTest, DiagnosticTrailingComment) {
@@ -1075,7 +1074,7 @@ TEST_F(LexerTest, PrintingOutputYaml) {
Pair("tokens",
Yaml::Sequence(ElementsAre(
Yaml::Mapping(ElementsAre(
Pair("index", "0"), Pair("kind", "StartOfFile"),
Pair("index", "0"), Pair("kind", "FileStart"),
Pair("line", "1"), Pair("column", "1"),
Pair("indent", "1"), Pair("spelling", ""),
Pair("has_trailing_space", "true"))),
@@ -1095,7 +1094,7 @@ TEST_F(LexerTest, PrintingOutputYaml) {
Pair("indent", "1"), Pair("spelling", ";"),
Pair("has_trailing_space", "true"))),
Yaml::Mapping(ElementsAre(
Pair("index", "4"), Pair("kind", "EndOfFile"),
Pair("index", "4"), Pair("kind", "FileEnd"),
Pair("line", "15"), Pair("column", "1"),
Pair("indent", "1"), Pair("spelling", "")))))))))))));
}