carbon-language/carbon-lang
1
0
Fork 0
mirror of https://github.com/carbon-language/carbon-lang.git synced 2026-10-06 08:44:41 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Add sentinel Eof token at the end of the tokenized buffer. (#388)

Browse Source 
This has two goals:

1) It allows us to simplify and remove special cases from the parser:
   when we expect a particular token next, we can just check for it
   without needing a special case for end-of-file.

2) It gives us a token to use as a position when emitting diagnostics at
   the end of the file.

Centralize all updating of `position` to `Consume` and `SkipTo`, so that we can in a single place ensure that we never go past the EOF token.
This commit is contained in:
Richard Smith
2021-03-18 21:39:10 -07:00
committed by GitHub
parent d1757d9979
commit 1e6c7e3963
7 changed files with 415 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files
parser/parse_node_kind.def
+1
View File
@@ -21,5 +21,6 @@ CARBON_PARSE_NODE_KIND(FunctionDeclaration)
CARBON_PARSE_NODE_KIND(Identifier)
CARBON_PARSE_NODE_KIND(ParameterListEnd)
CARBON_PARSE_NODE_KIND(ParameterList)
CARBON_PARSE_NODE_KIND(FileEnd)
#undef CARBON_PARSE_NODE_KIND
parser/parse_tree_test.cpp
+100 -60
View File
@@ -48,7 +48,7 @@ TEST_F(ParseTreeTest, Empty) {
TokenizedBuffer tokens = GetTokenizedBuffer("");
ParseTree tree = ParseTree::Parse(tokens, consumer);
EXPECT_FALSE(tree.HasErrors());
EXPECT_THAT(tree.Postorder().begin(), Eq(tree.Postorder().end()));
EXPECT_THAT(tree, MatchParseTreeNodes({{.kind = ParseNodeKind::FileEnd()}}));
}
TEST_F(ParseTreeTest, EmptyDeclaration) {
@@ -59,20 +59,28 @@ TEST_F(ParseTreeTest, EmptyDeclaration) {
auto end = tree.Postorder().end();
ASSERT_THAT(it, Ne(end));
ParseTree::Node n = *it++;
ASSERT_THAT(it, Ne(end));
ParseTree::Node eof = *it++;
EXPECT_THAT(it, Eq(end));
// Directly test the main API so that we get easier to understand errors in
// simple cases than what the custom matcher will produce.
EXPECT_FALSE(tree.HasErrorInNode(n));
EXPECT_FALSE(tree.HasErrorInNode(eof));
EXPECT_THAT(tree.GetNodeKind(n), Eq(ParseNodeKind::EmptyDeclaration()));
EXPECT_THAT(tree.GetNodeKind(eof), Eq(ParseNodeKind::FileEnd()));
auto t = tree.GetNodeToken(n);
ASSERT_THAT(tokens.Tokens().begin(), Ne(tokens.Tokens().end()));
EXPECT_THAT(t, Eq(*tokens.Tokens().begin()));
EXPECT_THAT(tokens.GetTokenText(t), Eq(";"));
EXPECT_THAT(tree.Postorder(n).begin(), Eq(tree.Postorder().begin()));
EXPECT_THAT(tree.Postorder(n).end(), Eq(tree.Postorder().end()));
EXPECT_THAT(tree.Children(n).begin(), Eq(tree.Children(n).end()));
EXPECT_THAT(tree.Children(eof).begin(), Eq(tree.Children(eof).end()));
EXPECT_THAT(tree.Postorder().begin(), Eq(tree.Postorder(n).begin()));
EXPECT_THAT(tree.Postorder(n).end(), Eq(tree.Postorder(eof).begin()));
EXPECT_THAT(tree.Postorder(eof).end(), Eq(tree.Postorder().end()));
}
TEST_F(ParseTreeTest, BasicFunctionDeclaration) {
@@ -83,19 +91,20 @@ TEST_F(ParseTreeTest, BasicFunctionDeclaration) {
tree, MatchParseTreeNodes(
{{.kind = ParseNodeKind::FunctionDeclaration(),
.text = "fn",
.children = {
{ParseNodeKind::Identifier(), "F"},
{.kind = ParseNodeKind::ParameterList(),
.text = "(",
.children = {{ParseNodeKind::ParameterListEnd(), ")"}}},
{ParseNodeKind::DeclarationEnd(), ";"}}}}));
.children = {{ParseNodeKind::Identifier(), "F"},
{.kind = ParseNodeKind::ParameterList(),
.text = "(",
.children = {{ParseNodeKind::ParameterListEnd(),
")"}}},
{ParseNodeKind::DeclarationEnd(), ";"}}},
{.kind = ParseNodeKind::FileEnd()}}));
}
TEST_F(ParseTreeTest, NoDeclarationIntroducerOrSemi) {
TokenizedBuffer tokens = GetTokenizedBuffer("foo bar baz");
ParseTree tree = ParseTree::Parse(tokens, consumer);
EXPECT_TRUE(tree.HasErrors());
EXPECT_THAT(tree.Postorder().begin(), Eq(tree.Postorder().end()));
EXPECT_THAT(tree, MatchParseTreeNodes({{.kind = ParseNodeKind::FileEnd()}}));
}
TEST_F(ParseTreeTest, NoDeclarationIntroducerWithSemi) {
@@ -105,7 +114,8 @@ TEST_F(ParseTreeTest, NoDeclarationIntroducerWithSemi) {
EXPECT_THAT(tree,
MatchParseTreeNodes({{.kind = ParseNodeKind::EmptyDeclaration(),
.text = ";",
.has_error = true}}));
.has_error = true},
{.kind = ParseNodeKind::FileEnd()}}));
}
TEST_F(ParseTreeTest, JustFunctionIntroducerAndSemi) {
@@ -115,7 +125,8 @@ TEST_F(ParseTreeTest, JustFunctionIntroducerAndSemi) {
EXPECT_THAT(tree, MatchParseTreeNodes(
{{.kind = ParseNodeKind::FunctionDeclaration(),
.has_error = true,
.children = {{ParseNodeKind::DeclarationEnd()}}}}));
.children = {{ParseNodeKind::DeclarationEnd()}}},
{.kind = ParseNodeKind::FileEnd()}}));
}
TEST_F(ParseTreeTest, RepeatedFunctionIntroducerAndSemi) {
@@ -125,18 +136,19 @@ TEST_F(ParseTreeTest, RepeatedFunctionIntroducerAndSemi) {
EXPECT_THAT(tree, MatchParseTreeNodes(
{{.kind = ParseNodeKind::FunctionDeclaration(),
.has_error = true,
.children = {{ParseNodeKind::DeclarationEnd()}}}}));
.children = {{ParseNodeKind::DeclarationEnd()}}},
{.kind = ParseNodeKind::FileEnd()}}));
}
TEST_F(ParseTreeTest, FunctionDeclarationWithNoSignatureOrSemi) {
TokenizedBuffer tokens = GetTokenizedBuffer("fn foo");
ParseTree tree = ParseTree::Parse(tokens, consumer);
EXPECT_TRUE(tree.HasErrors());
EXPECT_THAT(tree,
MatchParseTreeNodes(
{{.kind = ParseNodeKind::FunctionDeclaration(),
.has_error = true,
.children = {{ParseNodeKind::Identifier(), "foo"}}}}));
EXPECT_THAT(tree, MatchParseTreeNodes(
{{.kind = ParseNodeKind::FunctionDeclaration(),
.has_error = true,
.children = {{ParseNodeKind::Identifier(), "foo"}}},
{.kind = ParseNodeKind::FileEnd()}}));
}
TEST_F(ParseTreeTest,
@@ -148,7 +160,8 @@ TEST_F(ParseTreeTest,
{{.kind = ParseNodeKind::FunctionDeclaration(),
.has_error = true,
.children = {{ParseNodeKind::Identifier(), "foo"},
{ParseNodeKind::DeclarationEnd()}}}}));
{ParseNodeKind::DeclarationEnd()}}},
{.kind = ParseNodeKind::FileEnd()}}));
}
TEST_F(ParseTreeTest, FunctionDeclarationWithSingleIdentifierParameterList) {
@@ -166,7 +179,8 @@ TEST_F(ParseTreeTest, FunctionDeclarationWithSingleIdentifierParameterList) {
{.kind = ParseNodeKind::ParameterList(),
.has_error = true,
.children = {{ParseNodeKind::ParameterListEnd()}}},
{ParseNodeKind::DeclarationEnd()}}}}));
{ParseNodeKind::DeclarationEnd()}}},
{.kind = ParseNodeKind::FileEnd()}}));
}
TEST_F(ParseTreeTest, FunctionDeclarationWithoutName) {
@@ -176,7 +190,8 @@ TEST_F(ParseTreeTest, FunctionDeclarationWithoutName) {
EXPECT_THAT(tree, MatchParseTreeNodes(
{{.kind = ParseNodeKind::FunctionDeclaration(),
.has_error = true,
.children = {{ParseNodeKind::DeclarationEnd()}}}}));
.children = {{ParseNodeKind::DeclarationEnd()}}},
{.kind = ParseNodeKind::FileEnd()}}));
}
TEST_F(ParseTreeTest,
@@ -188,7 +203,8 @@ TEST_F(ParseTreeTest,
EXPECT_THAT(tree, MatchParseTreeNodes(
{{.kind = ParseNodeKind::FunctionDeclaration(),
.has_error = true,
.children = {{ParseNodeKind::DeclarationEnd()}}}}));
.children = {{ParseNodeKind::DeclarationEnd()}}},
{.kind = ParseNodeKind::FileEnd()}}));
}
TEST_F(ParseTreeTest, FunctionDeclarationSkipToNewlineWithoutSemi) {
@@ -205,7 +221,8 @@ TEST_F(ParseTreeTest, FunctionDeclarationSkipToNewlineWithoutSemi) {
.children = {{ParseNodeKind::Identifier(), "F"},
{.kind = ParseNodeKind::ParameterList(),
.children = {{ParseNodeKind::ParameterListEnd()}}},
{ParseNodeKind::DeclarationEnd()}}}}));
{ParseNodeKind::DeclarationEnd()}}},
{.kind = ParseNodeKind::FileEnd()}}));
}
TEST_F(ParseTreeTest, FunctionDeclarationSkipIndentedNewlineWithSemi) {
@@ -226,7 +243,8 @@ TEST_F(ParseTreeTest, FunctionDeclarationSkipIndentedNewlineWithSemi) {
.children = {{ParseNodeKind::Identifier(), "F"},
{.kind = ParseNodeKind::ParameterList(),
.children = {{ParseNodeKind::ParameterListEnd()}}},
{ParseNodeKind::DeclarationEnd()}}}}));
{ParseNodeKind::DeclarationEnd()}}},
{.kind = ParseNodeKind::FileEnd()}}));
}
TEST_F(ParseTreeTest, FunctionDeclarationSkipIndentedNewlineWithoutSemi) {
@@ -245,7 +263,8 @@ TEST_F(ParseTreeTest, FunctionDeclarationSkipIndentedNewlineWithoutSemi) {
.children = {{ParseNodeKind::Identifier(), "F"},
{.kind = ParseNodeKind::ParameterList(),
.children = {{ParseNodeKind::ParameterListEnd()}}},
{ParseNodeKind::DeclarationEnd()}}}}));
{ParseNodeKind::DeclarationEnd()}}},
{.kind = ParseNodeKind::FileEnd()}}));
}
TEST_F(ParseTreeTest, FunctionDeclarationSkipIndentedNewlineUntilOutdent) {
@@ -264,7 +283,8 @@ TEST_F(ParseTreeTest, FunctionDeclarationSkipIndentedNewlineUntilOutdent) {
.children = {{ParseNodeKind::Identifier(), "F"},
{.kind = ParseNodeKind::ParameterList(),
.children = {{ParseNodeKind::ParameterListEnd()}}},
{ParseNodeKind::DeclarationEnd()}}}}));
{ParseNodeKind::DeclarationEnd()}}},
{.kind = ParseNodeKind::FileEnd()}}));
}
TEST_F(ParseTreeTest, FunctionDeclarationSkipWithoutSemiToCurly) {
@@ -287,15 +307,16 @@ TEST_F(ParseTreeTest, BasicFunctionDefinition) {
ParseTree tree = ParseTree::Parse(tokens, consumer);
EXPECT_FALSE(tree.HasErrors());
EXPECT_THAT(
tree, MatchParseTreeNodes(
{{.kind = ParseNodeKind::FunctionDeclaration(),
.children = {
{ParseNodeKind::Identifier(), "F"},
{.kind = ParseNodeKind::ParameterList(),
.children = {{ParseNodeKind::ParameterListEnd()}}},
{.kind = ParseNodeKind::CodeBlock(),
.text = "{",
.children = {{ParseNodeKind::CodeBlockEnd(), "}"}}}}}}));
tree,
MatchParseTreeNodes(
{{.kind = ParseNodeKind::FunctionDeclaration(),
.children = {{ParseNodeKind::Identifier(), "F"},
{.kind = ParseNodeKind::ParameterList(),
.children = {{ParseNodeKind::ParameterListEnd()}}},
{.kind = ParseNodeKind::CodeBlock(),
.text = "{",
.children = {{ParseNodeKind::CodeBlockEnd(), "}"}}}}},
{.kind = ParseNodeKind::FileEnd()}}));
}
TEST_F(ParseTreeTest, FunctionDefinitionWithNestedBlocks) {
@@ -311,21 +332,22 @@ TEST_F(ParseTreeTest, FunctionDefinitionWithNestedBlocks) {
tree,
MatchParseTreeNodes(
{{.kind = ParseNodeKind::FunctionDeclaration(),
.children = {
{ParseNodeKind::Identifier(), "F"},
{.kind = ParseNodeKind::ParameterList(),
.children = {{ParseNodeKind::ParameterListEnd()}}},
{.kind = ParseNodeKind::CodeBlock(),
.children = {
{.kind = ParseNodeKind::CodeBlock(),
.children = {{.kind = ParseNodeKind::CodeBlock(),
.children =
{{.kind = ParseNodeKind::CodeBlock(),
.children = {{ParseNodeKind::
CodeBlockEnd()}}},
{ParseNodeKind::CodeBlockEnd()}}},
{ParseNodeKind::CodeBlockEnd()}}},
{ParseNodeKind::CodeBlockEnd()}}}}}}));
.children =
{{ParseNodeKind::Identifier(), "F"},
{.kind = ParseNodeKind::ParameterList(),
.children = {{ParseNodeKind::ParameterListEnd()}}},
{.kind = ParseNodeKind::CodeBlock(),
.children =
{{.kind = ParseNodeKind::CodeBlock(),
.children =
{{.kind = ParseNodeKind::CodeBlock(),
.children = {{.kind = ParseNodeKind::CodeBlock(),
.children = {{ParseNodeKind::
CodeBlockEnd()}}},
{ParseNodeKind::CodeBlockEnd()}}},
{ParseNodeKind::CodeBlockEnd()}}},
{ParseNodeKind::CodeBlockEnd()}}}}},
{.kind = ParseNodeKind::FileEnd()}}));
}
TEST_F(ParseTreeTest, FunctionDefinitionWithIdenifierInStatements) {
@@ -346,7 +368,8 @@ TEST_F(ParseTreeTest, FunctionDefinitionWithIdenifierInStatements) {
.children = {{ParseNodeKind::ParameterListEnd()}}},
{.kind = ParseNodeKind::CodeBlock(),
.has_error = true,
.children = {{ParseNodeKind::CodeBlockEnd()}}}}}}));
.children = {{ParseNodeKind::CodeBlockEnd()}}}}},
{.kind = ParseNodeKind::FileEnd()}}));
}
TEST_F(ParseTreeTest, FunctionDefinitionWithIdenifierInNestedBlock) {
@@ -362,15 +385,16 @@ TEST_F(ParseTreeTest, FunctionDefinitionWithIdenifierInNestedBlock) {
tree,
MatchParseTreeNodes(
{{.kind = ParseNodeKind::FunctionDeclaration(),
.children = {
{ParseNodeKind::Identifier(), "F"},
{.kind = ParseNodeKind::ParameterList(),
.children = {{ParseNodeKind::ParameterListEnd()}}},
{.kind = ParseNodeKind::CodeBlock(),
.children = {{.kind = ParseNodeKind::CodeBlock(),
.has_error = true,
.children = {{ParseNodeKind::CodeBlockEnd()}}},
{ParseNodeKind::CodeBlockEnd()}}}}}}));
.children =
{{ParseNodeKind::Identifier(), "F"},
{.kind = ParseNodeKind::ParameterList(),
.children = {{ParseNodeKind::ParameterListEnd()}}},
{.kind = ParseNodeKind::CodeBlock(),
.children = {{.kind = ParseNodeKind::CodeBlock(),
.has_error = true,
.children = {{ParseNodeKind::CodeBlockEnd()}}},
{ParseNodeKind::CodeBlockEnd()}}}}},
{.kind = ParseNodeKind::FileEnd()}}));
}
auto GetAndDropLine(llvm::StringRef& s) -> std::string {
@@ -408,6 +432,8 @@ TEST_F(ParseTreeTest, Printing) {
"text: ')'}]},"));
EXPECT_THAT(GetAndDropLine(print),
StrEq(" {node_index: 3, kind: 'DeclarationEnd', text: ';'}]},"));
EXPECT_THAT(GetAndDropLine(print),
StrEq("{node_index: 5, kind: 'FileEnd', text: ''},"));
EXPECT_THAT(GetAndDropLine(print), StrEq("]"));
EXPECT_TRUE(print.empty()) << print;
}
@@ -526,6 +552,20 @@ TEST_F(ParseTreeTest, PrintingAsYAML) {
++nkvi;
EXPECT_THAT(nkvi, Eq(nkve));
++ni;
ASSERT_THAT(ni, Ne(ne));
node = llvm::dyn_cast<llvm::yaml::MappingNode>(&*ni);
ASSERT_THAT(node, NotNull());
nkvi = node->begin();
EXPECT_THAT(&*nkvi, IsKeyValueScalars("node_index", "5"));
++nkvi;
EXPECT_THAT(&*nkvi, IsKeyValueScalars("kind", "FileEnd"));
++nkvi;
EXPECT_THAT(&*nkvi, IsKeyValueScalars("text", ""));
++nkvi;
EXPECT_THAT(nkvi, Eq(node->end()));
++ni;
EXPECT_THAT(ni, Eq(ne));
++di;
parser/parser_impl.cpp
+45 -53
View File
@@ -25,18 +25,34 @@ auto ParseTree::Parser::Parse(TokenizedBuffer& tokens,
tree.node_impls.reserve(tokens.Size());
Parser parser(tree, tokens);
while (parser.position != parser.end) {
while (!parser.AtEndOfFile()) {
parser.ParseDeclaration();
}
parser.AddLeafNode(ParseNodeKind::FileEnd(), *parser.position);
assert(tree.Verify() && "Parse tree built but does not verify!");
return tree;
}
ParseTree::Parser::Parser(ParseTree& tree_arg, TokenizedBuffer& tokens_arg)
: tree(tree_arg),
tokens(tokens_arg),
position(tokens.Tokens().begin()),
end(tokens.Tokens().end()) {
assert(std::find_if(position, end,
[&](TokenizedBuffer::Token t) {
return tokens.GetKind(t) == TokenKind::EndOfFile();
}) != end &&
"No EndOfFileToken in token buffer.");
}
auto ParseTree::Parser::Consume(TokenKind kind) -> TokenizedBuffer::Token {
TokenizedBuffer::Token t = *position;
assert(kind != TokenKind::EndOfFile() && "Cannot consume the EOF token!");
assert(tokens.GetKind(t) == kind && "The current token is the wrong kind!");
++position;
assert(position != end && "Reached end of tokens without finding EOF token.");
return t;
}
@@ -45,8 +61,7 @@ auto ParseTree::Parser::ConsumeIf(TokenKind kind)
if (tokens.GetKind(*position) != kind) {
return {};
}
return *position++;
return Consume(kind);
}
auto ParseTree::Parser::AddLeafNode(ParseNodeKind kind,
@@ -108,22 +123,27 @@ auto ParseTree::Parser::AddNode(ParseNodeKind n_kind, TokenizedBuffer::Token t,
}
auto ParseTree::Parser::SkipMatchingGroup() -> bool {
assert(position != end && "Cannot skip at the end!");
TokenizedBuffer::Token t = *position;
TokenKind t_kind = tokens.GetKind(t);
if (!t_kind.IsOpeningSymbol()) {
return false;
}
position = std::next(
TokenizedBuffer::TokenIterator(tokens.GetMatchedClosingToken(t)));
SkipTo(tokens.GetMatchedClosingToken(t));
Consume(t_kind.GetClosingSymbol());
return true;
}
auto ParseTree::Parser::SkipTo(TokenizedBuffer::Token t) -> void {
assert(t >= *position && "Tried to skip backwards.");
position = TokenizedBuffer::TokenIterator(t);
assert(position != end && "Skipped past EOF.");
}
auto ParseTree::Parser::SkipPastLikelyDeclarationEnd(
TokenizedBuffer::Token skip_root, bool is_inside_declaration)
-> llvm::Optional<Node> {
if (position == end) {
if (AtEndOfFile()) {
return {};
}
@@ -150,17 +170,14 @@ auto ParseTree::Parser::SkipPastLikelyDeclarationEnd(
return {};
}
// If we find a semicolon, we want to parse it to end the declaration.
if (current_kind == TokenKind::Semi()) {
TokenizedBuffer::Token semi = *position++;
// Add a node for the semicolon. If we're inside of a declaration, this
// is a declaration ending semicolon, otherwise it simply forms an empty
// declaration.
return AddLeafNode(is_inside_declaration
? ParseNodeKind::DeclarationEnd()
: ParseNodeKind::EmptyDeclaration(),
semi);
// If we find a semicolon, parse it and add a corresponding node. If we're
// inside of a declaration, this is a declaration ending semicolon,
// otherwise it simply forms an empty declaration.
if (auto end_node = ConsumeAndAddLeafNodeIf(
TokenKind::Semi(), is_inside_declaration
? ParseNodeKind::DeclarationEnd()
: ParseNodeKind::EmptyDeclaration())) {
return end_node;
}
// Skip over any matching group of tokens.
@@ -169,36 +186,31 @@ auto ParseTree::Parser::SkipPastLikelyDeclarationEnd(
}
// Otherwise just step forward one token.
++position;
} while (position != end &&
Consume(current_kind);
} while (!AtEndOfFile() &&
is_same_line_or_indent_greater_than_root(*position));
return {};
}
auto ParseTree::Parser::ParseFunctionSignature() -> Node {
assert(position != end && "Cannot parse past the end!");
TokenizedBuffer::Token open_paren = Consume(TokenKind::OpenParen());
assert(position != end &&
"The lexer ensures we always have a closing paren!");
auto start = StartSubtree();
// FIXME: Add support for parsing parameters.
bool has_errors = false;
auto close_paren = ConsumeIf(TokenKind::CloseParen());
if (!close_paren) {
if (tokens.GetKind(*position) != TokenKind::CloseParen()) {
llvm::errs() << "ERROR: unexpected token before the close of the "
"parameters on line "
<< tokens.GetLineNumber(*position) << "!\n";
has_errors = true;
// We can trivially skip to the actual close parenthesis from here.
close_paren = tokens.GetMatchedClosingToken(open_paren);
position = std::next(TokenizedBuffer::TokenIterator(*close_paren));
SkipTo(tokens.GetMatchedClosingToken(open_paren));
}
AddLeafNode(ParseNodeKind::ParameterListEnd(), *close_paren);
AddLeafNode(ParseNodeKind::ParameterListEnd(),
Consume(TokenKind::CloseParen()));
// FIXME: Implement parsing of a return type.
@@ -206,11 +218,7 @@ auto ParseTree::Parser::ParseFunctionSignature() -> Node {
}
auto ParseTree::Parser::ParseCodeBlock() -> Node {
assert(position != end && "Cannot parse past the end!");
TokenizedBuffer::Token open_curly = Consume(TokenKind::OpenCurlyBrace());
assert(position != end &&
"The lexer ensures we always have a closing curly!");
auto start = StartSubtree();
bool has_errors = false;
@@ -226,8 +234,7 @@ auto ParseTree::Parser::ParseCodeBlock() -> Node {
has_errors = true;
// We can trivially skip to the actual close curly brace from here.
position = TokenizedBuffer::TokenIterator(
tokens.GetMatchedClosingToken(open_curly));
SkipTo(tokens.GetMatchedClosingToken(open_curly));
// Now fall through to the close curly brace handling code.
LLVM_FALLTHROUGH;
@@ -254,21 +261,14 @@ auto ParseTree::Parser::ParseCodeBlock() -> Node {
}
auto ParseTree::Parser::ParseFunctionDeclaration() -> Node {
assert(position != end && "Cannot parse past the end!");
TokenizedBuffer::Token function_intro_token = Consume(TokenKind::FnKeyword());
auto start = StartSubtree();
auto add_error_function_node = [&] {
return AddNode(ParseNodeKind::FunctionDeclaration(), function_intro_token,
start, /*has_error=*/true);
};
if (position == end) {
llvm::errs() << "ERROR: File ended with a function introducer on line "
<< tokens.GetLineNumber(function_intro_token) << "!\n";
return add_error_function_node();
}
auto name_n = ConsumeAndAddLeafNodeIf(TokenKind::Identifier(),
ParseNodeKind::Identifier());
if (!name_n) {
@@ -280,12 +280,6 @@ auto ParseTree::Parser::ParseFunctionDeclaration() -> Node {
SkipPastLikelyDeclarationEnd(function_intro_token);
return add_error_function_node();
}
if (position == end) {
llvm::errs() << "ERROR: File ended after a function introducer and "
"identifier on line "
<< tokens.GetLineNumber(function_intro_token) << "!\n";
return add_error_function_node();
}
TokenizedBuffer::Token open_paren = *position;
if (tokens.GetKind(open_paren) != TokenKind::OpenParen()) {
@@ -296,8 +290,6 @@ auto ParseTree::Parser::ParseFunctionDeclaration() -> Node {
SkipPastLikelyDeclarationEnd(function_intro_token);
return add_error_function_node();
}
assert(std::next(position) != end &&
"Unbalanced parentheses should be rejected by the lexer.");
TokenizedBuffer::Token close_paren =
tokens.GetMatchedClosingToken(open_paren);
@@ -333,19 +325,19 @@ auto ParseTree::Parser::ParseFunctionDeclaration() -> Node {
}
auto ParseTree::Parser::ParseEmptyDeclaration() -> Node {
assert(position != end && "Cannot parse past the end!");
return AddLeafNode(ParseNodeKind::EmptyDeclaration(),
Consume(TokenKind::Semi()));
}
auto ParseTree::Parser::ParseDeclaration() -> llvm::Optional<Node> {
assert(position != end && "Cannot parse past the end!");
TokenizedBuffer::Token t = *position;
switch (tokens.GetKind(t)) {
case TokenKind::FnKeyword():
return ParseFunctionDeclaration();
case TokenKind::Semi():
return ParseEmptyDeclaration();
case TokenKind::EndOfFile():
return llvm::None;
default:
// Errors are handled outside the switch.
break;
parser/parser_impl.h
+11 -5
View File
@@ -25,11 +25,11 @@ class ParseTree::Parser {
private:
struct SubtreeStart;
explicit Parser(ParseTree& tree_arg, TokenizedBuffer& tokens_arg)
: tree(tree_arg),
tokens(tokens_arg),
position(tokens.Tokens().begin()),
end(tokens.Tokens().end()) {}
explicit Parser(ParseTree& tree_arg, TokenizedBuffer& tokens_arg);
auto AtEndOfFile() -> bool {
return tokens.GetKind(*position) == TokenKind::EndOfFile();
}
// Requires (and asserts) that the current position matches the provide
// `Kind`. Returns the current token and advances to the next position.
@@ -75,6 +75,9 @@ class ParseTree::Parser {
// returns false.
auto SkipMatchingGroup() -> bool;
// Skip forward to the token immediately after the given token.
auto SkipTo(TokenizedBuffer::Token t) -> void;
// Skips forward to move past the likely end of a declaration.
//
// Looks forward, skipping over any matched symbol groups, to find the next
@@ -125,7 +128,10 @@ class ParseTree::Parser {
ParseTree& tree;
TokenizedBuffer& tokens;
// The current position within the token buffer. Never equal to `end`.
TokenizedBuffer::TokenIterator position;
// The end position of the token buffer. There will always be an `EndOfFile`
// token between `position` (inclusive) and `end` (exclusive).
TokenizedBuffer::TokenIterator end;
};