mirror of
https://github.com/carbon-language/carbon-lang.git
synced 2026-09-27 11:40:10 +01:00
Add sentinel Eof token at the end of the tokenized buffer. (#388)
This has two goals: 1) It allows us to simplify and remove special cases from the parser: when we expect a particular token next, we can just check for it without needing a special case for end-of-file. 2) It gives us a token to use as a position when emitting diagnostics at the end of the file. Centralize all updating of `position` to `Consume` and `SkipTo`, so that we can in a single place ensure that we never go past the EOF token.
This commit is contained in:
Richard Smith
GitHub
+100
-60
@@ -48,7 +48,7 @@ TEST_F(ParseTreeTest, Empty) {
|
||||
TokenizedBuffer tokens = GetTokenizedBuffer("");
|
||||
ParseTree tree = ParseTree::Parse(tokens, consumer);
|
||||
EXPECT_FALSE(tree.HasErrors());
|
||||
EXPECT_THAT(tree.Postorder().begin(), Eq(tree.Postorder().end()));
|
||||
EXPECT_THAT(tree, MatchParseTreeNodes({{.kind = ParseNodeKind::FileEnd()}}));
|
||||
}
|
||||
|
||||
TEST_F(ParseTreeTest, EmptyDeclaration) {
|
||||
@@ -59,20 +59,28 @@ TEST_F(ParseTreeTest, EmptyDeclaration) {
|
||||
auto end = tree.Postorder().end();
|
||||
ASSERT_THAT(it, Ne(end));
|
||||
ParseTree::Node n = *it++;
|
||||
ASSERT_THAT(it, Ne(end));
|
||||
ParseTree::Node eof = *it++;
|
||||
EXPECT_THAT(it, Eq(end));
|
||||
|
||||
// Directly test the main API so that we get easier to understand errors in
|
||||
// simple cases than what the custom matcher will produce.
|
||||
EXPECT_FALSE(tree.HasErrorInNode(n));
|
||||
EXPECT_FALSE(tree.HasErrorInNode(eof));
|
||||
EXPECT_THAT(tree.GetNodeKind(n), Eq(ParseNodeKind::EmptyDeclaration()));
|
||||
EXPECT_THAT(tree.GetNodeKind(eof), Eq(ParseNodeKind::FileEnd()));
|
||||
|
||||
auto t = tree.GetNodeToken(n);
|
||||
ASSERT_THAT(tokens.Tokens().begin(), Ne(tokens.Tokens().end()));
|
||||
EXPECT_THAT(t, Eq(*tokens.Tokens().begin()));
|
||||
EXPECT_THAT(tokens.GetTokenText(t), Eq(";"));
|
||||
|
||||
EXPECT_THAT(tree.Postorder(n).begin(), Eq(tree.Postorder().begin()));
|
||||
EXPECT_THAT(tree.Postorder(n).end(), Eq(tree.Postorder().end()));
|
||||
EXPECT_THAT(tree.Children(n).begin(), Eq(tree.Children(n).end()));
|
||||
EXPECT_THAT(tree.Children(eof).begin(), Eq(tree.Children(eof).end()));
|
||||
|
||||
EXPECT_THAT(tree.Postorder().begin(), Eq(tree.Postorder(n).begin()));
|
||||
EXPECT_THAT(tree.Postorder(n).end(), Eq(tree.Postorder(eof).begin()));
|
||||
EXPECT_THAT(tree.Postorder(eof).end(), Eq(tree.Postorder().end()));
|
||||
}
|
||||
|
||||
TEST_F(ParseTreeTest, BasicFunctionDeclaration) {
|
||||
@@ -83,19 +91,20 @@ TEST_F(ParseTreeTest, BasicFunctionDeclaration) {
|
||||
tree, MatchParseTreeNodes(
|
||||
{{.kind = ParseNodeKind::FunctionDeclaration(),
|
||||
.text = "fn",
|
||||
.children = {
|
||||
{ParseNodeKind::Identifier(), "F"},
|
||||
{.kind = ParseNodeKind::ParameterList(),
|
||||
.text = "(",
|
||||
.children = {{ParseNodeKind::ParameterListEnd(), ")"}}},
|
||||
{ParseNodeKind::DeclarationEnd(), ";"}}}}));
|
||||
.children = {{ParseNodeKind::Identifier(), "F"},
|
||||
{.kind = ParseNodeKind::ParameterList(),
|
||||
.text = "(",
|
||||
.children = {{ParseNodeKind::ParameterListEnd(),
|
||||
")"}}},
|
||||
{ParseNodeKind::DeclarationEnd(), ";"}}},
|
||||
{.kind = ParseNodeKind::FileEnd()}}));
|
||||
}
|
||||
|
||||
TEST_F(ParseTreeTest, NoDeclarationIntroducerOrSemi) {
|
||||
TokenizedBuffer tokens = GetTokenizedBuffer("foo bar baz");
|
||||
ParseTree tree = ParseTree::Parse(tokens, consumer);
|
||||
EXPECT_TRUE(tree.HasErrors());
|
||||
EXPECT_THAT(tree.Postorder().begin(), Eq(tree.Postorder().end()));
|
||||
EXPECT_THAT(tree, MatchParseTreeNodes({{.kind = ParseNodeKind::FileEnd()}}));
|
||||
}
|
||||
|
||||
TEST_F(ParseTreeTest, NoDeclarationIntroducerWithSemi) {
|
||||
@@ -105,7 +114,8 @@ TEST_F(ParseTreeTest, NoDeclarationIntroducerWithSemi) {
|
||||
EXPECT_THAT(tree,
|
||||
MatchParseTreeNodes({{.kind = ParseNodeKind::EmptyDeclaration(),
|
||||
.text = ";",
|
||||
.has_error = true}}));
|
||||
.has_error = true},
|
||||
{.kind = ParseNodeKind::FileEnd()}}));
|
||||
}
|
||||
|
||||
TEST_F(ParseTreeTest, JustFunctionIntroducerAndSemi) {
|
||||
@@ -115,7 +125,8 @@ TEST_F(ParseTreeTest, JustFunctionIntroducerAndSemi) {
|
||||
EXPECT_THAT(tree, MatchParseTreeNodes(
|
||||
{{.kind = ParseNodeKind::FunctionDeclaration(),
|
||||
.has_error = true,
|
||||
.children = {{ParseNodeKind::DeclarationEnd()}}}}));
|
||||
.children = {{ParseNodeKind::DeclarationEnd()}}},
|
||||
{.kind = ParseNodeKind::FileEnd()}}));
|
||||
}
|
||||
|
||||
TEST_F(ParseTreeTest, RepeatedFunctionIntroducerAndSemi) {
|
||||
@@ -125,18 +136,19 @@ TEST_F(ParseTreeTest, RepeatedFunctionIntroducerAndSemi) {
|
||||
EXPECT_THAT(tree, MatchParseTreeNodes(
|
||||
{{.kind = ParseNodeKind::FunctionDeclaration(),
|
||||
.has_error = true,
|
||||
.children = {{ParseNodeKind::DeclarationEnd()}}}}));
|
||||
.children = {{ParseNodeKind::DeclarationEnd()}}},
|
||||
{.kind = ParseNodeKind::FileEnd()}}));
|
||||
}
|
||||
|
||||
TEST_F(ParseTreeTest, FunctionDeclarationWithNoSignatureOrSemi) {
|
||||
TokenizedBuffer tokens = GetTokenizedBuffer("fn foo");
|
||||
ParseTree tree = ParseTree::Parse(tokens, consumer);
|
||||
EXPECT_TRUE(tree.HasErrors());
|
||||
EXPECT_THAT(tree,
|
||||
MatchParseTreeNodes(
|
||||
{{.kind = ParseNodeKind::FunctionDeclaration(),
|
||||
.has_error = true,
|
||||
.children = {{ParseNodeKind::Identifier(), "foo"}}}}));
|
||||
EXPECT_THAT(tree, MatchParseTreeNodes(
|
||||
{{.kind = ParseNodeKind::FunctionDeclaration(),
|
||||
.has_error = true,
|
||||
.children = {{ParseNodeKind::Identifier(), "foo"}}},
|
||||
{.kind = ParseNodeKind::FileEnd()}}));
|
||||
}
|
||||
|
||||
TEST_F(ParseTreeTest,
|
||||
@@ -148,7 +160,8 @@ TEST_F(ParseTreeTest,
|
||||
{{.kind = ParseNodeKind::FunctionDeclaration(),
|
||||
.has_error = true,
|
||||
.children = {{ParseNodeKind::Identifier(), "foo"},
|
||||
{ParseNodeKind::DeclarationEnd()}}}}));
|
||||
{ParseNodeKind::DeclarationEnd()}}},
|
||||
{.kind = ParseNodeKind::FileEnd()}}));
|
||||
}
|
||||
|
||||
TEST_F(ParseTreeTest, FunctionDeclarationWithSingleIdentifierParameterList) {
|
||||
@@ -166,7 +179,8 @@ TEST_F(ParseTreeTest, FunctionDeclarationWithSingleIdentifierParameterList) {
|
||||
{.kind = ParseNodeKind::ParameterList(),
|
||||
.has_error = true,
|
||||
.children = {{ParseNodeKind::ParameterListEnd()}}},
|
||||
{ParseNodeKind::DeclarationEnd()}}}}));
|
||||
{ParseNodeKind::DeclarationEnd()}}},
|
||||
{.kind = ParseNodeKind::FileEnd()}}));
|
||||
}
|
||||
|
||||
TEST_F(ParseTreeTest, FunctionDeclarationWithoutName) {
|
||||
@@ -176,7 +190,8 @@ TEST_F(ParseTreeTest, FunctionDeclarationWithoutName) {
|
||||
EXPECT_THAT(tree, MatchParseTreeNodes(
|
||||
{{.kind = ParseNodeKind::FunctionDeclaration(),
|
||||
.has_error = true,
|
||||
.children = {{ParseNodeKind::DeclarationEnd()}}}}));
|
||||
.children = {{ParseNodeKind::DeclarationEnd()}}},
|
||||
{.kind = ParseNodeKind::FileEnd()}}));
|
||||
}
|
||||
|
||||
TEST_F(ParseTreeTest,
|
||||
@@ -188,7 +203,8 @@ TEST_F(ParseTreeTest,
|
||||
EXPECT_THAT(tree, MatchParseTreeNodes(
|
||||
{{.kind = ParseNodeKind::FunctionDeclaration(),
|
||||
.has_error = true,
|
||||
.children = {{ParseNodeKind::DeclarationEnd()}}}}));
|
||||
.children = {{ParseNodeKind::DeclarationEnd()}}},
|
||||
{.kind = ParseNodeKind::FileEnd()}}));
|
||||
}
|
||||
|
||||
TEST_F(ParseTreeTest, FunctionDeclarationSkipToNewlineWithoutSemi) {
|
||||
@@ -205,7 +221,8 @@ TEST_F(ParseTreeTest, FunctionDeclarationSkipToNewlineWithoutSemi) {
|
||||
.children = {{ParseNodeKind::Identifier(), "F"},
|
||||
{.kind = ParseNodeKind::ParameterList(),
|
||||
.children = {{ParseNodeKind::ParameterListEnd()}}},
|
||||
{ParseNodeKind::DeclarationEnd()}}}}));
|
||||
{ParseNodeKind::DeclarationEnd()}}},
|
||||
{.kind = ParseNodeKind::FileEnd()}}));
|
||||
}
|
||||
|
||||
TEST_F(ParseTreeTest, FunctionDeclarationSkipIndentedNewlineWithSemi) {
|
||||
@@ -226,7 +243,8 @@ TEST_F(ParseTreeTest, FunctionDeclarationSkipIndentedNewlineWithSemi) {
|
||||
.children = {{ParseNodeKind::Identifier(), "F"},
|
||||
{.kind = ParseNodeKind::ParameterList(),
|
||||
.children = {{ParseNodeKind::ParameterListEnd()}}},
|
||||
{ParseNodeKind::DeclarationEnd()}}}}));
|
||||
{ParseNodeKind::DeclarationEnd()}}},
|
||||
{.kind = ParseNodeKind::FileEnd()}}));
|
||||
}
|
||||
|
||||
TEST_F(ParseTreeTest, FunctionDeclarationSkipIndentedNewlineWithoutSemi) {
|
||||
@@ -245,7 +263,8 @@ TEST_F(ParseTreeTest, FunctionDeclarationSkipIndentedNewlineWithoutSemi) {
|
||||
.children = {{ParseNodeKind::Identifier(), "F"},
|
||||
{.kind = ParseNodeKind::ParameterList(),
|
||||
.children = {{ParseNodeKind::ParameterListEnd()}}},
|
||||
{ParseNodeKind::DeclarationEnd()}}}}));
|
||||
{ParseNodeKind::DeclarationEnd()}}},
|
||||
{.kind = ParseNodeKind::FileEnd()}}));
|
||||
}
|
||||
|
||||
TEST_F(ParseTreeTest, FunctionDeclarationSkipIndentedNewlineUntilOutdent) {
|
||||
@@ -264,7 +283,8 @@ TEST_F(ParseTreeTest, FunctionDeclarationSkipIndentedNewlineUntilOutdent) {
|
||||
.children = {{ParseNodeKind::Identifier(), "F"},
|
||||
{.kind = ParseNodeKind::ParameterList(),
|
||||
.children = {{ParseNodeKind::ParameterListEnd()}}},
|
||||
{ParseNodeKind::DeclarationEnd()}}}}));
|
||||
{ParseNodeKind::DeclarationEnd()}}},
|
||||
{.kind = ParseNodeKind::FileEnd()}}));
|
||||
}
|
||||
|
||||
TEST_F(ParseTreeTest, FunctionDeclarationSkipWithoutSemiToCurly) {
|
||||
@@ -287,15 +307,16 @@ TEST_F(ParseTreeTest, BasicFunctionDefinition) {
|
||||
ParseTree tree = ParseTree::Parse(tokens, consumer);
|
||||
EXPECT_FALSE(tree.HasErrors());
|
||||
EXPECT_THAT(
|
||||
tree, MatchParseTreeNodes(
|
||||
{{.kind = ParseNodeKind::FunctionDeclaration(),
|
||||
.children = {
|
||||
{ParseNodeKind::Identifier(), "F"},
|
||||
{.kind = ParseNodeKind::ParameterList(),
|
||||
.children = {{ParseNodeKind::ParameterListEnd()}}},
|
||||
{.kind = ParseNodeKind::CodeBlock(),
|
||||
.text = "{",
|
||||
.children = {{ParseNodeKind::CodeBlockEnd(), "}"}}}}}}));
|
||||
tree,
|
||||
MatchParseTreeNodes(
|
||||
{{.kind = ParseNodeKind::FunctionDeclaration(),
|
||||
.children = {{ParseNodeKind::Identifier(), "F"},
|
||||
{.kind = ParseNodeKind::ParameterList(),
|
||||
.children = {{ParseNodeKind::ParameterListEnd()}}},
|
||||
{.kind = ParseNodeKind::CodeBlock(),
|
||||
.text = "{",
|
||||
.children = {{ParseNodeKind::CodeBlockEnd(), "}"}}}}},
|
||||
{.kind = ParseNodeKind::FileEnd()}}));
|
||||
}
|
||||
|
||||
TEST_F(ParseTreeTest, FunctionDefinitionWithNestedBlocks) {
|
||||
@@ -311,21 +332,22 @@ TEST_F(ParseTreeTest, FunctionDefinitionWithNestedBlocks) {
|
||||
tree,
|
||||
MatchParseTreeNodes(
|
||||
{{.kind = ParseNodeKind::FunctionDeclaration(),
|
||||
.children = {
|
||||
{ParseNodeKind::Identifier(), "F"},
|
||||
{.kind = ParseNodeKind::ParameterList(),
|
||||
.children = {{ParseNodeKind::ParameterListEnd()}}},
|
||||
{.kind = ParseNodeKind::CodeBlock(),
|
||||
.children = {
|
||||
{.kind = ParseNodeKind::CodeBlock(),
|
||||
.children = {{.kind = ParseNodeKind::CodeBlock(),
|
||||
.children =
|
||||
{{.kind = ParseNodeKind::CodeBlock(),
|
||||
.children = {{ParseNodeKind::
|
||||
CodeBlockEnd()}}},
|
||||
{ParseNodeKind::CodeBlockEnd()}}},
|
||||
{ParseNodeKind::CodeBlockEnd()}}},
|
||||
{ParseNodeKind::CodeBlockEnd()}}}}}}));
|
||||
.children =
|
||||
{{ParseNodeKind::Identifier(), "F"},
|
||||
{.kind = ParseNodeKind::ParameterList(),
|
||||
.children = {{ParseNodeKind::ParameterListEnd()}}},
|
||||
{.kind = ParseNodeKind::CodeBlock(),
|
||||
.children =
|
||||
{{.kind = ParseNodeKind::CodeBlock(),
|
||||
.children =
|
||||
{{.kind = ParseNodeKind::CodeBlock(),
|
||||
.children = {{.kind = ParseNodeKind::CodeBlock(),
|
||||
.children = {{ParseNodeKind::
|
||||
CodeBlockEnd()}}},
|
||||
{ParseNodeKind::CodeBlockEnd()}}},
|
||||
{ParseNodeKind::CodeBlockEnd()}}},
|
||||
{ParseNodeKind::CodeBlockEnd()}}}}},
|
||||
{.kind = ParseNodeKind::FileEnd()}}));
|
||||
}
|
||||
|
||||
TEST_F(ParseTreeTest, FunctionDefinitionWithIdenifierInStatements) {
|
||||
@@ -346,7 +368,8 @@ TEST_F(ParseTreeTest, FunctionDefinitionWithIdenifierInStatements) {
|
||||
.children = {{ParseNodeKind::ParameterListEnd()}}},
|
||||
{.kind = ParseNodeKind::CodeBlock(),
|
||||
.has_error = true,
|
||||
.children = {{ParseNodeKind::CodeBlockEnd()}}}}}}));
|
||||
.children = {{ParseNodeKind::CodeBlockEnd()}}}}},
|
||||
{.kind = ParseNodeKind::FileEnd()}}));
|
||||
}
|
||||
|
||||
TEST_F(ParseTreeTest, FunctionDefinitionWithIdenifierInNestedBlock) {
|
||||
@@ -362,15 +385,16 @@ TEST_F(ParseTreeTest, FunctionDefinitionWithIdenifierInNestedBlock) {
|
||||
tree,
|
||||
MatchParseTreeNodes(
|
||||
{{.kind = ParseNodeKind::FunctionDeclaration(),
|
||||
.children = {
|
||||
{ParseNodeKind::Identifier(), "F"},
|
||||
{.kind = ParseNodeKind::ParameterList(),
|
||||
.children = {{ParseNodeKind::ParameterListEnd()}}},
|
||||
{.kind = ParseNodeKind::CodeBlock(),
|
||||
.children = {{.kind = ParseNodeKind::CodeBlock(),
|
||||
.has_error = true,
|
||||
.children = {{ParseNodeKind::CodeBlockEnd()}}},
|
||||
{ParseNodeKind::CodeBlockEnd()}}}}}}));
|
||||
.children =
|
||||
{{ParseNodeKind::Identifier(), "F"},
|
||||
{.kind = ParseNodeKind::ParameterList(),
|
||||
.children = {{ParseNodeKind::ParameterListEnd()}}},
|
||||
{.kind = ParseNodeKind::CodeBlock(),
|
||||
.children = {{.kind = ParseNodeKind::CodeBlock(),
|
||||
.has_error = true,
|
||||
.children = {{ParseNodeKind::CodeBlockEnd()}}},
|
||||
{ParseNodeKind::CodeBlockEnd()}}}}},
|
||||
{.kind = ParseNodeKind::FileEnd()}}));
|
||||
}
|
||||
|
||||
auto GetAndDropLine(llvm::StringRef& s) -> std::string {
|
||||
@@ -408,6 +432,8 @@ TEST_F(ParseTreeTest, Printing) {
|
||||
"text: ')'}]},"));
|
||||
EXPECT_THAT(GetAndDropLine(print),
|
||||
StrEq(" {node_index: 3, kind: 'DeclarationEnd', text: ';'}]},"));
|
||||
EXPECT_THAT(GetAndDropLine(print),
|
||||
StrEq("{node_index: 5, kind: 'FileEnd', text: ''},"));
|
||||
EXPECT_THAT(GetAndDropLine(print), StrEq("]"));
|
||||
EXPECT_TRUE(print.empty()) << print;
|
||||
}
|
||||
@@ -526,6 +552,20 @@ TEST_F(ParseTreeTest, PrintingAsYAML) {
|
||||
|
||||
++nkvi;
|
||||
EXPECT_THAT(nkvi, Eq(nkve));
|
||||
|
||||
++ni;
|
||||
ASSERT_THAT(ni, Ne(ne));
|
||||
node = llvm::dyn_cast<llvm::yaml::MappingNode>(&*ni);
|
||||
ASSERT_THAT(node, NotNull());
|
||||
nkvi = node->begin();
|
||||
EXPECT_THAT(&*nkvi, IsKeyValueScalars("node_index", "5"));
|
||||
++nkvi;
|
||||
EXPECT_THAT(&*nkvi, IsKeyValueScalars("kind", "FileEnd"));
|
||||
++nkvi;
|
||||
EXPECT_THAT(&*nkvi, IsKeyValueScalars("text", ""));
|
||||
++nkvi;
|
||||
EXPECT_THAT(nkvi, Eq(node->end()));
|
||||
|
||||
++ni;
|
||||
EXPECT_THAT(ni, Eq(ne));
|
||||
++di;
|
||||
|
||||
Reference in New Issue
Block a user