From d8c724045ef8ce34f2a5f555673622cdb6e92c1f Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Mon, 19 Apr 2021 12:55:35 -0700 Subject: [PATCH] Implement parsing for variable declarations. (#466) Following the rules proposed in #339. --- parser/parse_node_kind.def | 4 ++- parser/parse_tree_test.cpp | 32 +++++++++++++++++++++ parser/parser_impl.cpp | 59 ++++++++++++++++++++++++++++++++++---- parser/parser_impl.h | 11 +++++-- 4 files changed, 97 insertions(+), 9 deletions(-) diff --git a/parser/parse_node_kind.def b/parser/parse_node_kind.def index 70ef0a4ed249..fd6256877599 100644 --- a/parser/parse_node_kind.def +++ b/parser/parse_node_kind.def @@ -16,10 +16,12 @@ // Declarations. CARBON_PARSE_NODE_KIND(DeclarationEnd) CARBON_PARSE_NODE_KIND(EmptyDeclaration) -CARBON_PARSE_NODE_KIND(FunctionDeclaration) CARBON_PARSE_NODE_KIND(DeclaredName) +CARBON_PARSE_NODE_KIND(FunctionDeclaration) CARBON_PARSE_NODE_KIND(ParameterListEnd) CARBON_PARSE_NODE_KIND(ParameterList) +CARBON_PARSE_NODE_KIND(VariableDeclaration) +CARBON_PARSE_NODE_KIND(VariableInitializer) CARBON_PARSE_NODE_KIND(FileEnd) // Statements. diff --git a/parser/parse_tree_test.cpp b/parser/parse_tree_test.cpp index 4fe01ffec68d..e450b4ea7b99 100644 --- a/parser/parse_tree_test.cpp +++ b/parser/parse_tree_test.cpp @@ -484,6 +484,38 @@ TEST_F(ParseTreeTest, Operators) { MatchFileEnd()})); } +TEST_F(ParseTreeTest, VariableDeclarations) { + TokenizedBuffer tokens = GetTokenizedBuffer( + "var Int v = 0;\n" + "var Int w;\n" + "fn F() {\n" + " var String s = \"hello\";\n" + "}"); + ParseTree tree = ParseTree::Parse(tokens, consumer); + EXPECT_FALSE(tree.HasErrors()); + + EXPECT_THAT( + tree, + MatchParseTreeNodes( + {MatchVariableDeclaration(MatchNameReference("Int"), + MatchDeclaredName("v"), + MatchVariableInitializer(MatchLiteral("0")), + MatchDeclarationEnd()), + MatchVariableDeclaration(MatchNameReference("Int"), + MatchDeclaredName("w"), + MatchDeclarationEnd()), + MatchFunctionDeclaration( + MatchDeclaredName("F"), + MatchParameterList(MatchParameterListEnd()), + MatchCodeBlock( + MatchVariableDeclaration( + MatchNameReference("String"), MatchDeclaredName("s"), + MatchVariableInitializer(MatchLiteral("\"hello\"")), + MatchDeclarationEnd()), + MatchCodeBlockEnd())), + MatchFileEnd()})); +} + auto GetAndDropLine(llvm::StringRef& s) -> std::string { auto newline_offset = s.find_first_of('\n'); llvm::StringRef line = s.slice(0, newline_offset); diff --git a/parser/parser_impl.cpp b/parser/parser_impl.cpp index 1918deb636cb..c2bd043408ae 100644 --- a/parser/parser_impl.cpp +++ b/parser/parser_impl.cpp @@ -48,6 +48,12 @@ struct ExpectedFunctionBodyOrSemi "Expected function definition or `;` after function declaration."; }; +struct ExpectedVariableName : SimpleDiagnostic { + static constexpr llvm::StringLiteral ShortName = "syntax-error"; + static constexpr llvm::StringLiteral Message = + "Expected variable name after type in `var` declaration."; +}; + struct UnrecognizedDeclaration : SimpleDiagnostic { static constexpr llvm::StringLiteral ShortName = "syntax-error"; static constexpr llvm::StringLiteral Message = @@ -217,14 +223,17 @@ auto ParseTree::Parser::SkipTo(TokenizedBuffer::Token t) -> void { assert(position != end && "Skipped past EOF."); } -auto ParseTree::Parser::FindNext(TokenKind desired_kind) +auto ParseTree::Parser::FindNextOf( + std::initializer_list desired_kinds) -> llvm::Optional { auto new_position = position; while (true) { TokenizedBuffer::Token token = *new_position; TokenKind kind = tokens.GetKind(token); - if (kind == desired_kind) { - return token; + for (TokenKind desired_kind : desired_kinds) { + if (kind == desired_kind) { + return token; + } } // Step to the next token at the current bracketing level. @@ -320,6 +329,10 @@ auto ParseTree::Parser::ParseCodeBlock() -> Node { // Loop over all the different possibly nested elements in the code block. for (;;) { switch (tokens.GetKind(*position)) { + case TokenKind::VarKeyword(): + ParseVariableDeclaration(); + continue; + default: // A statement with no introducer token can only be an expression // statement. @@ -420,6 +433,41 @@ auto ParseTree::Parser::ParseFunctionDeclaration() -> Node { start); } +auto ParseTree::Parser::ParseVariableDeclaration() -> Node { + // `var` expression identifier [= expression] `;` + TokenizedBuffer::Token var_token = Consume(TokenKind::VarKeyword()); + auto start = StartSubtree(); + + auto type = ParseExpression(); + + auto name = ConsumeAndAddLeafNodeIf(TokenKind::Identifier(), + ParseNodeKind::DeclaredName()); + if (!name) { + emitter.EmitError(*position); + if (auto after_name = FindNextOf({TokenKind::Equal(), TokenKind::Semi()})) { + SkipTo(*after_name); + } + } + + auto start_init = StartSubtree(); + if (auto equal_token = ConsumeIf(TokenKind::Equal())) { + auto init = ParseExpression(); + AddNode(ParseNodeKind::VariableInitializer(), *equal_token, start_init, + /*has_error=*/!init); + } + + auto semi = ConsumeAndAddLeafNodeIf(TokenKind::Semi(), + ParseNodeKind::DeclarationEnd()); + if (!semi) { + SkipPastLikelyEnd(var_token, [&](TokenizedBuffer::Token semi) { + return AddLeafNode(ParseNodeKind::DeclarationEnd(), semi); + }); + } + + return AddNode(ParseNodeKind::VariableDeclaration(), var_token, start, + /*has_error=*/!type || !name || !semi); +} + auto ParseTree::Parser::ParseEmptyDeclaration() -> Node { return AddLeafNode(ParseNodeKind::EmptyDeclaration(), Consume(TokenKind::Semi())); @@ -430,6 +478,8 @@ auto ParseTree::Parser::ParseDeclaration() -> llvm::Optional { switch (tokens.GetKind(t)) { case TokenKind::FnKeyword(): return ParseFunctionDeclaration(); + case TokenKind::VarKeyword(): + return ParseVariableDeclaration(); case TokenKind::Semi(): return ParseEmptyDeclaration(); case TokenKind::EndOfFile(): @@ -551,13 +601,12 @@ auto ParseTree::Parser::ParseCallExpression(SubtreeStart start, bool has_errors) } has_errors = true; - auto comma_position = FindNext(TokenKind::Comma()); + auto comma_position = FindNextOf({TokenKind::Comma()}); if (!comma_position) { SkipTo(tokens.GetMatchedClosingToken(open_paren)); break; } SkipTo(*comma_position); - break; } AddLeafNode(ParseNodeKind::CallExpressionComma(), diff --git a/parser/parser_impl.h b/parser/parser_impl.h index 9da905dc9810..57fd8d3ba588 100644 --- a/parser/parser_impl.h +++ b/parser/parser_impl.h @@ -76,11 +76,13 @@ class ParseTree::Parser { // returns false. auto SkipMatchingGroup() -> bool; - // Skip forward to the token immediately after the given token. + // Skip forward to the given token. auto SkipTo(TokenizedBuffer::Token t) -> void; - // Find the next token of the given kind at the current bracketing level. - auto FindNext(TokenKind t) -> llvm::Optional; + // Find the next token of any of the given kinds at the current bracketing + // level. + auto FindNextOf(std::initializer_list desired_kinds) + -> llvm::Optional; // Callback used if we find a semicolon when skipping to the end of a // declaration or statement. @@ -123,6 +125,9 @@ class ParseTree::Parser { // function parse node which is based on the `fn` introducer keyword. auto ParseFunctionDeclaration() -> Node; + // Parses a variable declaration with an optional initializer. + auto ParseVariableDeclaration() -> Node; + // Parses and returns an empty declaration node from a single semicolon token. auto ParseEmptyDeclaration() -> Node;