diff --git a/toolchain/parser/parse_tree.cpp b/toolchain/parser/parse_tree.cpp index bbac1c3ec12f..930e68b5d01d 100644 --- a/toolchain/parser/parse_tree.cpp +++ b/toolchain/parser/parse_tree.cpp @@ -28,9 +28,7 @@ auto ParseTree::Parse(TokenizedBuffer& tokens, DiagnosticConsumer& consumer) TokenDiagnosticEmitter emitter(translator, consumer); // Delegate to the parser. - // TODO: Edit this to swap between Parser and Parser2. This is manual in order - // to avoid test duplication. - return Parser::Parse(tokens, emitter); + return Parser2::Parse(tokens, emitter); } auto ParseTree::postorder() const -> llvm::iterator_range { diff --git a/toolchain/parser/parse_tree_test.cpp b/toolchain/parser/parse_tree_test.cpp index 5ee406211493..ec835c8c884f 100644 --- a/toolchain/parser/parse_tree_test.cpp +++ b/toolchain/parser/parse_tree_test.cpp @@ -209,7 +209,7 @@ TEST_F(ParseTreeTest, PrintPreorderAsYAML) { EXPECT_THAT(Yaml::Value::FromText(print_output), ElementsAre(file)); } -TEST_F(ParseTreeTest, RecursionLimit) { +TEST_F(ParseTreeTest, HighRecursion) { std::string code = "fn Foo() { return "; code.append(10000, '('); code.append(10000, ')'); @@ -217,31 +217,8 @@ TEST_F(ParseTreeTest, RecursionLimit) { TokenizedBuffer tokens = GetTokenizedBuffer(code); ASSERT_FALSE(tokens.has_errors()); Testing::MockDiagnosticConsumer consumer; - // Recursion might be exceeded multiple times due to quirks in parse tree - // handling; we only need to be sure it's hit at least once for test - // correctness. - EXPECT_CALL(consumer, HandleDiagnostic(IsDiagnosticMessage( - llvm::formatv("Exceeded recursion limit ({0})", - ParseTree::StackDepthLimit) - .str()))) - .Times(AtLeast(1)); ParseTree tree = ParseTree::Parse(tokens, consumer); - EXPECT_TRUE(tree.has_errors()); -} - -TEST_F(ParseTreeTest, ParsePostfixExpressionRegression) { - // Stack depth errors could cause ParsePostfixExpression to infinitely loop - // when calling children and those children error. Because of the fragility of - // stack depth, this tries a few different values. - for (int n = 0; n <= 10; ++n) { - std::string code = "var x: auto = "; - code.append(ParseTree::StackDepthLimit - n, '*'); - code += "(z);"; - TokenizedBuffer tokens = GetTokenizedBuffer(code); - ASSERT_FALSE(tokens.has_errors()); - ParseTree tree = ParseTree::Parse(tokens, consumer); - EXPECT_TRUE(tree.has_errors()); - } + EXPECT_FALSE(tree.has_errors()); } TEST_F(ParseTreeTest, PackageErrors) { diff --git a/toolchain/parser/parser2.cpp b/toolchain/parser/parser2.cpp index f1c6c79aa927..be4b4c14d186 100644 --- a/toolchain/parser/parser2.cpp +++ b/toolchain/parser/parser2.cpp @@ -22,6 +22,9 @@ CARBON_DIAGNOSTIC( OperatorRequiresParentheses, Error, "Parentheses are required to disambiguate operator precedence."); +CARBON_DIAGNOSTIC(ExpectedSemiAfterExpression, Error, + "Expected `;` after expression."); + // A relative location for characters in errors. enum class RelativeLocation : int8_t { Around, @@ -235,7 +238,9 @@ auto Parser2::SkipTo(TokenizedBuffer::Token t) -> void { CARBON_CHECK(position_ != end_) << "Skipped past EOF."; } -auto Parser2::HandleCodeBlock() -> void { +auto Parser2::HandleCodeBlockState() -> void { + PopAndDiscardState(); + PushState(ParserState::CodeBlockFinish()); if (ConsumeAndAddLeafNodeIf(TokenKind::OpenCurlyBrace(), ParseNodeKind::CodeBlockStart())) { @@ -248,7 +253,7 @@ auto Parser2::HandleCodeBlock() -> void { CARBON_DIAGNOSTIC(ExpectedCodeBlock, Error, "Expected braced code block."); emitter_.Emit(*position_, ExpectedCodeBlock); - HandleStatement(PositionKind()); + PushState(ParserState::Statement()); } } @@ -371,11 +376,41 @@ auto Parser2::DiagnoseOperatorFixity(OperatorFixity fixity) -> void { } } +auto Parser2::ConsumeListToken(ParseNodeKind comma_kind, TokenKind close_kind, + bool already_has_error) -> ListTokenKind { + if (!PositionIs(TokenKind::Comma()) && !PositionIs(close_kind)) { + // Don't error a second time on the same element. + if (!already_has_error) { + CARBON_DIAGNOSTIC(UnexpectedTokenAfterListElement, Error, + "Expected `,` or `{0}`.", TokenKind); + emitter_.Emit(*position_, UnexpectedTokenAfterListElement, close_kind); + ReturnErrorOnState(); + } + + // Recover from the invalid token. + auto end_of_element = FindNextOf({TokenKind::Comma(), close_kind}); + // The lexer guarantees that parentheses are balanced. + CARBON_CHECK(end_of_element) + << "missing matching `" << close_kind.GetOpeningSymbol() << "` for `" + << close_kind << "`"; + + SkipTo(*end_of_element); + } + + if (PositionIs(close_kind)) { + return ListTokenKind::Close; + } else { + AddLeafNode(comma_kind, Consume()); + return PositionIs(close_kind) ? ListTokenKind::CommaClose + : ListTokenKind::Comma; + } +} + auto Parser2::Parse() -> void { // Traces state_stack_. This runs even in opt because it's low overhead. PrettyStackTraceParseState pretty_stack(this); - PushState(ParserState::Declaration()); + PushState(ParserState::DeclarationLoop()); while (!state_stack_.empty()) { switch (state_stack_.back().state) { #define CARBON_PARSER_STATE(Name) \ @@ -389,6 +424,251 @@ auto Parser2::Parse() -> void { AddLeafNode(ParseNodeKind::FileEnd(), *position_); } +auto Parser2::HandleBraceExpressionState() -> void { + auto state = PopState(); + + state.state = ParserState::BraceExpressionFinishAsUnknown(); + PushState(state); + + // TODO: When swapping {} start/end, this should AddLeafNode the open before + // continuing. + + // Advance past the open brace. + CARBON_CHECK(PositionIs(TokenKind::OpenCurlyBrace())); + ++position_; + if (!PositionIs(TokenKind::CloseCurlyBrace())) { + PushState(ParserState::BraceExpressionParameterAsUnknown()); + } +} + +auto Parser2::BraceExpressionKindToParserState(BraceExpressionKind kind, + ParserState type, + ParserState value, + ParserState unknown) + -> ParserState { + switch (kind) { + case BraceExpressionKind::Type: { + return type; + } + case BraceExpressionKind::Value: { + return value; + } + case BraceExpressionKind::Unknown: { + return unknown; + } + } +} + +auto Parser2::HandleBraceExpressionParameterError(StateStackEntry state, + BraceExpressionKind kind) + -> void { + CARBON_DIAGNOSTIC(ExpectedStructLiteralField, Error, "Expected {0}{1}{2}.", + llvm::StringRef, llvm::StringRef, llvm::StringRef); + bool can_be_type = kind != BraceExpressionKind::Value; + bool can_be_value = kind != BraceExpressionKind::Type; + emitter_.Emit(*position_, ExpectedStructLiteralField, + can_be_type ? "`.field: type`" : "", + (can_be_type && can_be_value) ? " or " : "", + can_be_value ? "`.field = value`" : ""); + + state.state = BraceExpressionKindToParserState( + kind, ParserState::BraceExpressionParameterFinishAsType(), + ParserState::BraceExpressionParameterFinishAsValue(), + ParserState::BraceExpressionParameterFinishAsUnknown()); + state.has_error = true; + PushState(state); +} + +auto Parser2::HandleBraceExpressionParameter(BraceExpressionKind kind) -> void { + auto state = PopState(); + + if (!PositionIs(TokenKind::Period())) { + HandleBraceExpressionParameterError(state, kind); + return; + } + + state.state = BraceExpressionKindToParserState( + kind, ParserState::BraceExpressionParameterAfterDesignatorAsType(), + ParserState::BraceExpressionParameterAfterDesignatorAsValue(), + ParserState::BraceExpressionParameterAfterDesignatorAsUnknown()); + PushState(state); + PushState(ParserState::DesignatorAsStruct()); +} + +auto Parser2::HandleBraceExpressionParameterAsTypeState() -> void { + HandleBraceExpressionParameter(BraceExpressionKind::Type); +} + +auto Parser2::HandleBraceExpressionParameterAsValueState() -> void { + HandleBraceExpressionParameter(BraceExpressionKind::Value); +} + +auto Parser2::HandleBraceExpressionParameterAsUnknownState() -> void { + HandleBraceExpressionParameter(BraceExpressionKind::Unknown); +} + +auto Parser2::HandleBraceExpressionParameterAfterDesignator( + BraceExpressionKind kind) -> void { + auto state = PopState(); + + if (state.has_error) { + auto recovery_pos = FindNextOf( + {TokenKind::Equal(), TokenKind::Colon(), TokenKind::Comma()}); + if (!recovery_pos || tokens_.GetKind(*recovery_pos) == TokenKind::Comma()) { + state.state = BraceExpressionKindToParserState( + kind, ParserState::BraceExpressionParameterFinishAsType(), + ParserState::BraceExpressionParameterFinishAsValue(), + ParserState::BraceExpressionParameterFinishAsUnknown()); + PushState(state); + return; + } + SkipTo(*recovery_pos); + } + + // Work out the kind of this element. + auto elem_kind = BraceExpressionKind::Unknown; + if (PositionIs(TokenKind::Colon())) { + elem_kind = BraceExpressionKind::Type; + } else if (PositionIs(TokenKind::Equal())) { + elem_kind = BraceExpressionKind::Value; + } + // Unknown kinds and changes between type and value are errors. + if (elem_kind == BraceExpressionKind::Unknown || + (kind != BraceExpressionKind::Unknown && elem_kind != kind)) { + HandleBraceExpressionParameterError(state, kind); + return; + } + + // If we're setting the kind, update the BraceExpressionFinish state. + if (kind == BraceExpressionKind::Unknown) { + kind = elem_kind; + auto finish_state = PopState(); + CARBON_CHECK(finish_state.state == + ParserState::BraceExpressionFinishAsUnknown()); + finish_state.state = BraceExpressionKindToParserState( + kind, ParserState::BraceExpressionFinishAsType(), + ParserState::BraceExpressionFinishAsValue(), + ParserState::BraceExpressionFinishAsUnknown()); + PushState(finish_state); + } + + state.state = BraceExpressionKindToParserState( + kind, ParserState::BraceExpressionParameterFinishAsType(), + ParserState::BraceExpressionParameterFinishAsValue(), + ParserState::BraceExpressionParameterFinishAsUnknown()); + + state.token = Consume(); + + // Struct type fields and value fields use the same grammar except + // that one has a `:` separator and the other has an `=` separator. + PushState(state); + PushState(ParserState::Expression()); +} + +auto Parser2::HandleBraceExpressionParameterAfterDesignatorAsTypeState() + -> void { + HandleBraceExpressionParameterAfterDesignator(BraceExpressionKind::Type); +} + +auto Parser2::HandleBraceExpressionParameterAfterDesignatorAsValueState() + -> void { + HandleBraceExpressionParameterAfterDesignator(BraceExpressionKind::Value); +} + +auto Parser2::HandleBraceExpressionParameterAfterDesignatorAsUnknownState() + -> void { + HandleBraceExpressionParameterAfterDesignator(BraceExpressionKind::Unknown); +} + +auto Parser2::HandleBraceExpressionParameterFinish(BraceExpressionKind kind) + -> void { + auto state = PopState(); + + AddNode(kind == BraceExpressionKind::Type ? ParseNodeKind::StructFieldType() + : ParseNodeKind::StructFieldValue(), + state.token, state.subtree_start, state.has_error); + + if (ConsumeListToken(ParseNodeKind::StructComma(), + TokenKind::CloseCurlyBrace(), + state.has_error) == ListTokenKind::Comma) { + PushState(BraceExpressionKindToParserState( + kind, ParserState::BraceExpressionParameterAsType(), + ParserState::BraceExpressionParameterAsValue(), + ParserState::BraceExpressionParameterAsUnknown())); + } +} + +auto Parser2::HandleBraceExpressionParameterFinishAsTypeState() -> void { + HandleBraceExpressionParameterFinish(BraceExpressionKind::Type); +} + +auto Parser2::HandleBraceExpressionParameterFinishAsValueState() -> void { + HandleBraceExpressionParameterFinish(BraceExpressionKind::Value); +} + +auto Parser2::HandleBraceExpressionParameterFinishAsUnknownState() -> void { + HandleBraceExpressionParameterFinish(BraceExpressionKind::Unknown); +} + +auto Parser2::HandleBraceExpressionFinish(BraceExpressionKind kind) -> void { + auto state = PopState(); + + AddLeafNode(ParseNodeKind::StructEnd(), Consume()); + AddNode(kind == BraceExpressionKind::Type ? ParseNodeKind::StructTypeLiteral() + : ParseNodeKind::StructLiteral(), + state.token, state.subtree_start, state.has_error); +} + +auto Parser2::HandleBraceExpressionFinishAsTypeState() -> void { + HandleBraceExpressionFinish(BraceExpressionKind::Type); +} + +auto Parser2::HandleBraceExpressionFinishAsValueState() -> void { + HandleBraceExpressionFinish(BraceExpressionKind::Value); +} + +auto Parser2::HandleBraceExpressionFinishAsUnknownState() -> void { + HandleBraceExpressionFinish(BraceExpressionKind::Unknown); +} + +auto Parser2::HandleCallExpressionState() -> void { + auto state = PopState(); + + // TODO: When swapping () start/end, this should AddLeafNode the open before + // continuing. + state.state = ParserState::CallExpressionFinish(); + PushState(state); + // Advance past the open paren. + ++position_; + if (!PositionIs(TokenKind::CloseParen())) { + PushState(ParserState::CallExpressionParameterFinish()); + PushState(ParserState::Expression()); + } +} + +auto Parser2::HandleCallExpressionParameterFinishState() -> void { + auto state = PopState(); + + if (state.has_error) { + ReturnErrorOnState(); + } + + if (ConsumeListToken(ParseNodeKind::CallExpressionComma(), + TokenKind::CloseParen(), + state.has_error) == ListTokenKind::Comma) { + PushState(ParserState::CallExpressionParameterFinish()); + PushState(ParserState::Expression()); + } +} + +auto Parser2::HandleCallExpressionFinishState() -> void { + auto state = PopState(); + + AddLeafNode(ParseNodeKind::CallExpressionEnd(), Consume()); + AddNode(ParseNodeKind::CallExpression(), state.token, state.subtree_start, + state.has_error); +} + auto Parser2::HandleCodeBlockFinishState() -> void { auto state = PopState(); @@ -403,10 +683,9 @@ auto Parser2::HandleCodeBlockFinishState() -> void { } } -auto Parser2::HandleDeclarationState() -> void { +auto Parser2::HandleDeclarationLoopState() -> void { // This maintains the current state unless we're at the end of the file. - // TODO: INCOMPLETE switch (PositionKind()) { case TokenKind::EndOfFile(): { PopAndDiscardState(); @@ -418,11 +697,20 @@ auto Parser2::HandleDeclarationState() -> void { ++position_; break; } + case TokenKind::Package(): { + PushState(ParserState::Package()); + ++position_; + break; + } case TokenKind::Semi(): { AddLeafNode(ParseNodeKind::EmptyDeclaration(), *position_); ++position_; break; } + case TokenKind::Var(): { + PushState(ParserState::VarAsRequireSemicolon()); + break; + } default: { CARBON_DIAGNOSTIC(UnrecognizedDeclaration, Error, "Unrecognized declaration introducer."); @@ -437,7 +725,7 @@ auto Parser2::HandleDeclarationState() -> void { } } -auto Parser2::HandleDesignatorExpressionState() -> void { +auto Parser2::HandleDesignator(bool as_struct) -> void { auto state = PopState(); // `.` identifier @@ -460,8 +748,17 @@ auto Parser2::HandleDesignatorExpressionState() -> void { } } - AddNode(ParseNodeKind::DesignatorExpression(), *dot, state.subtree_start, - state.has_error); + AddNode(as_struct ? ParseNodeKind::StructFieldDesignator() + : ParseNodeKind::DesignatorExpression(), + *dot, state.subtree_start, state.has_error); +} + +auto Parser2::HandleDesignatorAsExpressionState() -> void { + HandleDesignator(/*as_struct=*/false); +} + +auto Parser2::HandleDesignatorAsStructState() -> void { + HandleDesignator(/*as_struct=*/true); } auto Parser2::HandleExpressionState() -> void { @@ -493,36 +790,47 @@ auto Parser2::HandleExpressionState() -> void { } auto Parser2::HandleExpressionInPostfixState() -> void { - PopAndDiscardState(); + auto state = PopState(); - // Regardless of success or failure, we'll continue to the Loop state. - PushState(ParserState::ExpressionInPostfixLoop()); + // Continue to the loop state. + state.state = ParserState::ExpressionInPostfixLoop(); // Parses a primary expression, which is either a terminal portion of an // expression tree, such as an identifier or literal, or a parenthesized // expression. - // TODO: INCOMPLETE: Handle OpenParen and OpenCurlyBrace. switch (PositionKind()) { - case TokenKind::Identifier(): - AddLeafNode(ParseNodeKind::NameReference(), *position_); + case TokenKind::Identifier(): { + AddLeafNode(ParseNodeKind::NameReference(), Consume()); + PushState(state); break; - + } case TokenKind::IntegerLiteral(): case TokenKind::RealLiteral(): case TokenKind::StringLiteral(): case TokenKind::IntegerTypeLiteral(): case TokenKind::UnsignedIntegerTypeLiteral(): - case TokenKind::FloatingPointTypeLiteral(): - AddLeafNode(ParseNodeKind::Literal(), *position_); + case TokenKind::FloatingPointTypeLiteral(): { + AddLeafNode(ParseNodeKind::Literal(), Consume()); + PushState(state); break; - - default: + } + case TokenKind::OpenCurlyBrace(): { + PushState(state); + PushState(ParserState::BraceExpression()); + break; + } + case TokenKind::OpenParen(): { + PushState(state); + PushState(ParserState::ParenExpression()); + break; + } + default: { CARBON_DIAGNOSTIC(ExpectedExpression, Error, "Expected expression."); emitter_.Emit(*position_, ExpectedExpression); ReturnErrorOnState(); - return; + break; + } } - ++position_; } auto Parser2::HandleExpressionInPostfixLoopState() -> void { @@ -530,23 +838,27 @@ auto Parser2::HandleExpressionInPostfixLoopState() -> void { // on. auto state = PopState(); + state.token = *position_; + switch (PositionKind()) { - case TokenKind::Period(): + case TokenKind::Period(): { PushState(state); - state.state = ParserState::DesignatorExpression(); + state.state = ParserState::DesignatorAsExpression(); PushState(state); break; - - case TokenKind::OpenParen(): - // expression = ParseCallExpression(start, !expression); - CARBON_FATAL() << "TODO: INCOMPLETE"; + } + case TokenKind::OpenParen(): { + PushState(state); + state.state = ParserState::CallExpression(); + PushState(state); break; - - default: + } + default: { if (state.has_error) { ReturnErrorOnState(); } break; + } } } @@ -589,9 +901,7 @@ auto Parser2::HandleExpressionLoopState() -> void { : OperatorFixity::Postfix); } - state.token = *position_; - ++position_; - + state.token = Consume(); state.lhs_precedence = operator_precedence; if (is_binary) { @@ -636,8 +946,6 @@ auto Parser2::HandleExpressionStatementFinishState() -> void { } if (!state.has_error) { - CARBON_DIAGNOSTIC(ExpectedSemiAfterExpression, Error, - "Expected `;` after expression."); emitter_.Emit(*position_, ExpectedSemiAfterExpression); } @@ -696,20 +1004,19 @@ auto Parser2::HandleFunctionIntroducerState() -> void { PushState(ParserState::FunctionParameterListFinish()); // Advance past the open paren. ++position_; - if (PositionKind() != TokenKind::CloseParen()) { - PushState(ParserState::PatternForFunctionParameter()); + if (!PositionIs(TokenKind::CloseParen())) { + PushState(ParserState::PatternAsFunctionParameter()); } } auto Parser2::HandleFunctionParameterListFinishState() -> void { auto state = PopState(); - CARBON_CHECK(PositionKind() == TokenKind::CloseParen()) + CARBON_CHECK(ConsumeAndAddLeafNodeIf(TokenKind::CloseParen(), + ParseNodeKind::ParameterListEnd())) << PositionKind().Name(); - AddLeafNode(ParseNodeKind::ParameterListEnd(), *position_); AddNode(ParseNodeKind::ParameterList(), state.token, state.subtree_start, state.has_error); - ++position_; } auto Parser2::HandleFunctionAfterParameterListState() -> void { @@ -740,15 +1047,13 @@ auto Parser2::HandleFunctionSignatureFinishState() -> void { switch (PositionKind()) { case TokenKind::Semi(): { - AddNode(ParseNodeKind::FunctionDeclaration(), *position_, + AddNode(ParseNodeKind::FunctionDeclaration(), Consume(), state.subtree_start, state.has_error); - ++position_; break; } case TokenKind::OpenCurlyBrace(): { - AddNode(ParseNodeKind::FunctionDefinitionStart(), *position_, + AddNode(ParseNodeKind::FunctionDefinitionStart(), Consume(), state.subtree_start, state.has_error); - ++position_; // Any error is recorded on the FunctionDefinitionStart. state.has_error = false; state.state = ParserState::FunctionDefinitionFinish(); @@ -772,55 +1077,87 @@ auto Parser2::HandleFunctionSignatureFinishState() -> void { auto Parser2::HandleFunctionDefinitionFinishState() -> void { auto state = PopState(); - AddNode(ParseNodeKind::FunctionDefinition(), *position_, state.subtree_start, + AddNode(ParseNodeKind::FunctionDefinition(), Consume(), state.subtree_start, state.has_error); - ++position_; } -auto Parser2::HandlePatternStart(PatternKind pattern_kind) -> void { +auto Parser2::HandlePackageState() -> void { auto state = PopState(); - // Ensure the finish state always follows. - switch (pattern_kind) { - case PatternKind::Parameter: { - state.state = ParserState::PatternForFunctionParameterFinish(); - break; + auto exit_on_parse_error = [&]() { + if (auto semi_token = SkipPastLikelyEnd(state.token)) { + AddLeafNode(ParseNodeKind::PackageEnd(), *semi_token); } - case PatternKind::Variable: { - CARBON_FATAL() << "TODO: INCOMPLETE"; - break; - } - } + return AddNode(ParseNodeKind::PackageDirective(), state.token, + state.subtree_start, /*has_error=*/true); + }; - // Handle an invalid pattern introducer. - if (!PositionIs(TokenKind::Identifier()) || - tokens_.GetKind(*(position_ + 1)) != TokenKind::Colon()) { - switch (pattern_kind) { - case PatternKind::Parameter: { - CARBON_DIAGNOSTIC(ExpectedParameterName, Error, - "Expected parameter declaration."); - emitter_.Emit(*position_, ExpectedParameterName); - break; - } - case PatternKind::Variable: { - CARBON_DIAGNOSTIC(ExpectedVariableName, Error, - "Expected pattern in `var` declaration."); - emitter_.Emit(*position_, ExpectedVariableName); - break; - } - } - state.has_error = true; - PushState(state); + if (!ConsumeAndAddLeafNodeIf(TokenKind::Identifier(), + ParseNodeKind::DeclaredName())) { + CARBON_DIAGNOSTIC(ExpectedIdentifierAfterPackage, Error, + "Expected identifier after `package`."); + emitter_.Emit(*position_, ExpectedIdentifierAfterPackage); + exit_on_parse_error(); return; } - // Switch the context token to the colon, so that it'll be used for the root - // node. - state.token = *(position_ + 1); - PushState(state); - PushStateForExpression(PrecedenceGroup::ForType()); - AddLeafNode(ParseNodeKind::DeclaredName(), *position_); - position_ += 2; + bool library_parsed = false; + if (auto library_token = ConsumeIf(TokenKind::Library())) { + auto library_start = tree_.size(); + + if (!ConsumeAndAddLeafNodeIf(TokenKind::StringLiteral(), + ParseNodeKind::Literal())) { + CARBON_DIAGNOSTIC( + ExpectedLibraryName, Error, + "Expected a string literal to specify the library name."); + emitter_.Emit(*position_, ExpectedLibraryName); + exit_on_parse_error(); + return; + } + + AddNode(ParseNodeKind::PackageLibrary(), *library_token, library_start, + /*has_error=*/false); + library_parsed = true; + } + + switch (auto api_or_impl_token = tokens_.GetKind(*(position_))) { + case TokenKind::Api(): { + AddLeafNode(ParseNodeKind::PackageApi(), Consume()); + break; + } + case TokenKind::Impl(): { + AddLeafNode(ParseNodeKind::PackageImpl(), Consume()); + break; + } + default: { + if (!library_parsed && api_or_impl_token == TokenKind::StringLiteral()) { + // If we come acroess a string literal and we didn't parse `library + // "..."` yet, then most probably the user forgot to add `library` + // before the library name. + CARBON_DIAGNOSTIC(MissingLibraryKeyword, Error, + "Missing `library` keyword."); + emitter_.Emit(*position_, MissingLibraryKeyword); + } else { + CARBON_DIAGNOSTIC(ExpectedApiOrImpl, Error, + "Expected a `api` or `impl`."); + emitter_.Emit(*position_, ExpectedApiOrImpl); + } + exit_on_parse_error(); + return; + } + } + + if (!ConsumeAndAddLeafNodeIf(TokenKind::Semi(), + ParseNodeKind::PackageEnd())) { + CARBON_DIAGNOSTIC(ExpectedSemiToEndPackageDirective, Error, + "Expected `;` to end package directive."); + emitter_.Emit(*position_, ExpectedSemiToEndPackageDirective); + exit_on_parse_error(); + return; + } + + AddNode(ParseNodeKind::PackageDirective(), state.token, state.subtree_start, + /*has_error=*/false); } auto Parser2::HandleParenConditionState() -> void { @@ -860,63 +1197,208 @@ auto Parser2::HandleParenConditionFinishState() -> void { /*has_error=*/state.has_error || !close_paren); } -auto Parser2::HandlePatternForFunctionParameterState() -> void { - HandlePatternStart(PatternKind::Parameter); +auto Parser2::HandleParenExpressionState() -> void { + auto state = PopState(); + + // TODO: When swapping () start/end, this should AddLeafNode the open before + // continuing. + + // Advance past the open paren. + CARBON_CHECK(PositionIs(TokenKind::OpenParen())); + ++position_; + if (PositionIs(TokenKind::CloseParen())) { + state.state = ParserState::ParenExpressionFinishAsTuple(); + PushState(state); + } else { + state.state = ParserState::ParenExpressionFinish(); + PushState(state); + PushState(ParserState::ParenExpressionParameterFinishAsUnknown()); + PushState(ParserState::Expression()); + } } -auto Parser2::HandlePatternForFunctionParameterFinishState() -> void { +auto Parser2::HandleParenExpressionParameterFinish(bool as_tuple) -> void { + auto state = PopState(); + + auto list_token_kind = + ConsumeListToken(ParseNodeKind::TupleLiteralComma(), + TokenKind::CloseParen(), state.has_error); + if (list_token_kind == ListTokenKind::Close) { + return; + } + + // If this is the first item and a comma was found, switch to tuple handling. + // Note this could be `(expr,)` so we may not reuse the current state, but + // it's still necessary to switch the parent. + if (!as_tuple) { + state.state = ParserState::ParenExpressionParameterFinishAsTuple(); + + auto finish_state = PopState(); + CARBON_CHECK(finish_state.state == ParserState::ParenExpressionFinish()) + << "Unexpected parent state, found: " << finish_state.state; + finish_state.state = ParserState::ParenExpressionFinishAsTuple(); + PushState(finish_state); + } + + // On a comma, push another expression handler. + if (list_token_kind == ListTokenKind::Comma) { + PushState(state); + PushState(ParserState::Expression()); + } +} + +auto Parser2::HandleParenExpressionParameterFinishAsUnknownState() -> void { + HandleParenExpressionParameterFinish(/*as_tuple=*/false); +} + +auto Parser2::HandleParenExpressionParameterFinishAsTupleState() -> void { + HandleParenExpressionParameterFinish(/*as_tuple=*/true); +} + +auto Parser2::HandleParenExpressionFinishState() -> void { + auto state = PopState(); + + AddLeafNode(ParseNodeKind::ParenExpressionEnd(), Consume()); + AddNode(ParseNodeKind::ParenExpression(), state.token, state.subtree_start, + state.has_error); +} + +auto Parser2::HandleParenExpressionFinishAsTupleState() -> void { + auto state = PopState(); + + AddLeafNode(ParseNodeKind::TupleLiteralEnd(), Consume()); + AddNode(ParseNodeKind::TupleLiteral(), state.token, state.subtree_start, + state.has_error); +} + +auto Parser2::HandlePatternStart(PatternKind pattern_kind) -> void { + auto state = PopState(); + + // Ensure the finish state always follows. + switch (pattern_kind) { + case PatternKind::Parameter: { + state.state = ParserState::PatternAsFunctionParameterFinish(); + break; + } + case PatternKind::Variable: { + state.state = ParserState::PatternAsVariableFinish(); + break; + } + } + + // Handle an invalid pattern introducer. + if (!PositionIs(TokenKind::Identifier()) || + tokens_.GetKind(*(position_ + 1)) != TokenKind::Colon()) { + switch (pattern_kind) { + case PatternKind::Parameter: { + CARBON_DIAGNOSTIC(ExpectedParameterName, Error, + "Expected parameter declaration."); + emitter_.Emit(*position_, ExpectedParameterName); + break; + } + case PatternKind::Variable: { + CARBON_DIAGNOSTIC(ExpectedVariableName, Error, + "Expected pattern in `var` declaration."); + emitter_.Emit(*position_, ExpectedVariableName); + break; + } + } + state.has_error = true; + PushState(state); + return; + } + + // Switch the context token to the colon, so that it'll be used for the root + // node. + state.token = *(position_ + 1); + PushState(state); + PushStateForExpression(PrecedenceGroup::ForType()); + AddLeafNode(ParseNodeKind::DeclaredName(), *position_); + position_ += 2; +} + +auto Parser2::HandlePatternFinish() -> bool { auto state = PopState(); // If an error was encountered, propagate it without adding a node. - bool has_error = state.has_error; - if (has_error) { + if (state.has_error) { ReturnErrorOnState(); - } else { - // TODO: may need to mark has_error if !type. - AddNode(ParseNodeKind::PatternBinding(), state.token, state.subtree_start, - state.has_error); + return true; } - // Handle tokens following a parameter. + // TODO: may need to mark has_error if !type. + AddNode(ParseNodeKind::PatternBinding(), state.token, state.subtree_start, + /*has_error=*/false); + return false; +} + +auto Parser2::HandlePatternAsFunctionParameterState() -> void { + HandlePatternStart(PatternKind::Parameter); +} + +auto Parser2::HandlePatternAsFunctionParameterFinishState() -> void { + bool has_error = HandlePatternFinish(); + + if (ConsumeListToken(ParseNodeKind::ParameterListComma(), + TokenKind::CloseParen(), + has_error) == ListTokenKind::Comma) { + PushState(ParserState::PatternAsFunctionParameter()); + } +} + +auto Parser2::HandlePatternAsVariableState() -> void { + HandlePatternStart(PatternKind::Variable); +} + +auto Parser2::HandlePatternAsVariableFinishState() -> void { + HandlePatternFinish(); +} + +auto Parser2::HandleStatementState() -> void { + PopAndDiscardState(); + switch (PositionKind()) { - case TokenKind::CloseParen(): { - // Done with the parameter list. - return; + case TokenKind::Break(): { + PushState(ParserState::StatementBreakFinish()); + ++position_; + break; } - case TokenKind::Comma(): { - // Comma handling is after the switch. + case TokenKind::Continue(): { + PushState(ParserState::StatementContinueFinish()); + ++position_; + break; + } + case TokenKind::For(): { + // Process the header as a child of the for so that we can get consistent + // starts. + // TODO: When reorganizing components, we can probably make this flatter. + PushState(ParserState::StatementForFinish()); + ++position_; + PushState(ParserState::StatementForHeader()); + break; + } + case TokenKind::If(): { + PushState(ParserState::StatementIf()); + break; + } + case TokenKind::Return(): { + PushState(ParserState::StatementReturn()); + break; + } + case TokenKind::Var(): { + PushState(ParserState::VarAsRequireSemicolon()); + break; + } + case TokenKind::While(): { + PushState(ParserState::StatementWhile()); break; } default: { - // Don't error twice for the same issue. - if (!has_error) { - CARBON_DIAGNOSTIC(UnexpectedTokenAfterListElement, Error, - "Expected `,` or `)`."); - emitter_.Emit(*position_, UnexpectedTokenAfterListElement); - ReturnErrorOnState(); - } - - // Recover from the invalid token. - auto end_of_element = - FindNextOf({TokenKind::Comma(), TokenKind::CloseParen()}); - // The lexer guarantees that parentheses are balanced. - CARBON_CHECK(end_of_element) << "missing matching `)` for `(`"; - SkipTo(*end_of_element); - if (PositionKind() == TokenKind::CloseParen()) { - // Done with the parameter list. - return; - } - // Comma handling is after the switch. + PushState(ParserState::ExpressionStatementFinish()); + PushState(ParserState::Expression()); break; } } - - // We are guaranteed to now be at a comma. - AddLeafNode(ParseNodeKind::ParameterListComma(), *position_); - ++position_; - if (PositionKind() != TokenKind::CloseParen()) { - PushState(ParserState::PatternForFunctionParameter()); - } } auto Parser2::HandleStatementBreakFinishState() -> void { @@ -929,7 +1411,95 @@ auto Parser2::HandleStatementContinueFinishState() -> void { ParseNodeKind::ContinueStatement()); } -auto Parser2::HandleStatementIf() -> void { +auto Parser2::HandleStatementForHeaderState() -> void { + auto state = PopState(); + + auto open_paren = ConsumeIf(TokenKind::OpenParen()); + if (!open_paren) { + CARBON_DIAGNOSTIC(ExpectedParenAfter, Error, + "Expected `(` after `{0}`. Recovering from missing `(` " + "not implemented yet!", + TokenKind); + emitter_.Emit(*position_, ExpectedParenAfter, TokenKind::For()); + // TODO: A proper recovery strategy is needed here. For now, I assume + // that all brackets are properly balanced (i.e. each open bracket has a + // closing one). + // This is temporary until we come to a conclusion regarding the + // recovery tokens strategy. + ReturnErrorOnState(); + PushState(ParserState::CodeBlock()); + return; + } + + state.state = ParserState::StatementForHeaderIn(); + + if (PositionIs(TokenKind::Var())) { + PushState(state); + PushState(ParserState::VarAsNoSemicolon()); + } else { + CARBON_DIAGNOSTIC(ExpectedVariableDeclaration, Error, + "Expected `var` declaration."); + emitter_.Emit(*position_, ExpectedVariableDeclaration); + + if (auto next_in = FindNextOf({TokenKind::In()})) { + SkipTo(*next_in); + } + state.has_error = true; + PushState(state); + } +} + +auto Parser2::HandleStatementForHeaderInState() -> void { + auto state = PopState(); + + state.state = ParserState::StatementForHeaderFinish(); + + if (!ConsumeAndAddLeafNodeIf(TokenKind::In(), ParseNodeKind::ForIn())) { + if (auto colon = ConsumeIf(TokenKind::Colon())) { + CARBON_DIAGNOSTIC(ExpectedIn, Error, "`:` should be replaced by `in`."); + emitter_.Emit(*colon, ExpectedIn); + // TODO: Should probably add a ForIn node for consistency in ParseTree + // structure, but doesn't for consistency with the old implementation. + state.has_error = true; + } else { + CARBON_DIAGNOSTIC(ExpectedIn, Error, + "Expected `in` after loop `var` declaration."); + emitter_.Emit(*position_, ExpectedIn); + SkipTo(tokens_.GetMatchedClosingToken(state.token)); + + state.has_error = true; + PushState(state); + return; + } + } + + PushState(state); + PushState(ParserState::Expression()); +} + +auto Parser2::HandleStatementForHeaderFinishState() -> void { + auto state = PopState(); + + if (!ConsumeAndAddCloseParen(state.token, ParseNodeKind::ForHeaderEnd())) { + state.has_error = true; + } + + AddNode(ParseNodeKind::ForHeader(), state.token, state.subtree_start, + state.has_error); + + PushState(ParserState::CodeBlock()); +} + +auto Parser2::HandleStatementForFinishState() -> void { + auto state = PopState(); + + AddNode(ParseNodeKind::ForStatement(), state.token, state.subtree_start, + state.has_error); +} + +auto Parser2::HandleStatementIfState() -> void { + PopAndDiscardState(); + PushState(ParserState::StatementIfConditionFinish()); PushState(ParserState::ParenCondition()); ++position_; @@ -940,7 +1510,7 @@ auto Parser2::HandleStatementIfConditionFinishState() -> void { state.state = ParserState::StatementIfThenBlockFinish(); PushState(state); - HandleCodeBlock(); + PushState(ParserState::CodeBlock()); } auto Parser2::HandleStatementIfThenBlockFinishState() -> void { @@ -951,11 +1521,8 @@ auto Parser2::HandleStatementIfThenBlockFinishState() -> void { state.state = ParserState::StatementIfElseBlockFinish(); PushState(state); // `else if` is permitted as a special case. - if (PositionIs(TokenKind::If())) { - HandleStatementIf(); - } else { - HandleCodeBlock(); - } + PushState(PositionIs(TokenKind::If()) ? ParserState::StatementIf() + : ParserState::CodeBlock()); } else { AddNode(ParseNodeKind::IfStatement(), state.token, state.subtree_start, state.has_error); @@ -985,75 +1552,22 @@ auto Parser2::HandleStatementKeywordFinish(TokenKind token_kind, AddNode(node_kind, state.token, state.subtree_start, state.has_error); } +auto Parser2::HandleStatementReturnState() -> void { + auto state = PopState(); + + state.state = ParserState::StatementReturnFinish(); + PushState(state); + ++position_; + if (!PositionIs(TokenKind::Semi())) { + PushState(ParserState::Expression()); + } +} + auto Parser2::HandleStatementReturnFinishState() -> void { HandleStatementKeywordFinish(TokenKind::Return(), ParseNodeKind::ReturnStatement()); } -auto Parser2::HandleStatementWhile() -> void { - PushState(ParserState::StatementWhileConditionFinish()); - PushState(ParserState::ParenCondition()); - ++position_; -} - -auto Parser2::HandleStatementWhileConditionFinishState() -> void { - auto state = PopState(); - - state.state = ParserState::StatementWhileBlockFinish(); - PushState(state); - HandleCodeBlock(); -} - -auto Parser2::HandleStatementWhileBlockFinishState() -> void { - auto state = PopState(); - - AddNode(ParseNodeKind::WhileStatement(), state.token, state.subtree_start, - state.has_error); -} - -auto Parser2::HandleStatement(TokenKind token_kind) -> void { - switch (token_kind) { - case TokenKind::If(): { - HandleStatementIf(); - break; - } - case TokenKind::Break(): { - PushState(ParserState::StatementBreakFinish()); - ++position_; - break; - } - case TokenKind::Continue(): { - PushState(ParserState::StatementContinueFinish()); - ++position_; - break; - } - case TokenKind::Return(): { - auto return_token = *position_; - if (tokens_.GetKind(*(position_ + 1)) == TokenKind::Semi()) { - int subtree_start = tree_.size(); - AddLeafNode(ParseNodeKind::StatementEnd(), *(position_ + 1)); - AddNode(ParseNodeKind::ReturnStatement(), return_token, subtree_start, - /*has_error=*/false); - position_ += 2; - } else { - PushState(ParserState::StatementReturnFinish()); - ++position_; - PushState(ParserState::Expression()); - } - break; - } - case TokenKind::While(): { - HandleStatementWhile(); - break; - } - default: { - PushState(ParserState::ExpressionStatementFinish()); - PushState(ParserState::Expression()); - break; - } - } -} - auto Parser2::HandleStatementScopeLoopState() -> void { // This maintains the current state until we're at the end of the scope. @@ -1064,8 +1578,103 @@ auto Parser2::HandleStatementScopeLoopState() -> void { ReturnErrorOnState(); } } else { - HandleStatement(token_kind); + PushState(ParserState::Statement()); } } +auto Parser2::HandleStatementWhileState() -> void { + PopAndDiscardState(); + + PushState(ParserState::StatementWhileConditionFinish()); + PushState(ParserState::ParenCondition()); + ++position_; +} + +auto Parser2::HandleStatementWhileConditionFinishState() -> void { + auto state = PopState(); + + state.state = ParserState::StatementWhileBlockFinish(); + PushState(state); + PushState(ParserState::CodeBlock()); +} + +auto Parser2::HandleStatementWhileBlockFinishState() -> void { + auto state = PopState(); + + AddNode(ParseNodeKind::WhileStatement(), state.token, state.subtree_start, + state.has_error); +} + +auto Parser2::HandleVar(bool require_semicolon) -> void { + PopAndDiscardState(); + + PushState(require_semicolon ? ParserState::VarFinishAsRequireSemicolon() + : ParserState::VarFinishAsNoSemicolon()); + PushState(ParserState::VarAfterPattern()); + ++position_; + PushState(ParserState::PatternAsVariable()); +} + +auto Parser2::HandleVarAsRequireSemicolonState() -> void { + HandleVar(/*require_semicolon=*/true); +} + +auto Parser2::HandleVarAsNoSemicolonState() -> void { + HandleVar(/*require_semicolon=*/false); +} + +auto Parser2::HandleVarAfterPatternState() -> void { + auto state = PopState(); + + if (state.has_error) { + if (auto after_pattern = + FindNextOf({TokenKind::Equal(), TokenKind::Semi()})) { + SkipTo(*after_pattern); + } + } + + if (PositionIs(TokenKind::Equal())) { + PushState(ParserState::VarAfterInitializer()); + ++position_; + PushState(ParserState::Expression()); + return; + } +} + +auto Parser2::HandleVarAfterInitializerState() -> void { + auto state = PopState(); + + AddNode(ParseNodeKind::VariableInitializer(), state.token, + state.subtree_start, state.has_error); +} + +auto Parser2::HandleVarFinish(bool require_semicolon) -> void { + auto state = PopState(); + + if (require_semicolon) { + auto semi = ConsumeAndAddLeafNodeIf(TokenKind::Semi(), + ParseNodeKind::DeclarationEnd()); + if (!semi) { + emitter_.Emit(*position_, ExpectedSemiAfterExpression); + if (auto semi_token = SkipPastLikelyEnd(state.token)) { + AddLeafNode(ParseNodeKind::DeclarationEnd(), *semi_token, + /*has_error=*/true); + } else { + state.has_error = true; + } + } + } + + return AddNode(ParseNodeKind::VariableDeclaration(), state.token, + state.subtree_start, state.has_error); +} + +auto Parser2::HandleVarFinishAsRequireSemicolonState() -> void { + HandleVarFinish(/*require_semicolon=*/true); +} + +auto Parser2::HandleVarFinishAsNoSemicolonState() -> void { + HandleVarFinish(/*require_semicolon=*/false); +} + } // namespace Carbon diff --git a/toolchain/parser/parser2.h b/toolchain/parser/parser2.h index 2e551f2fb715..d47cd5c45944 100644 --- a/toolchain/parser/parser2.h +++ b/toolchain/parser/parser2.h @@ -5,6 +5,7 @@ #ifndef CARBON_TOOLCHAIN_PARSER_PARSER2_H_ #define CARBON_TOOLCHAIN_PARSER_PARSER2_H_ +#include "common/check.h" #include "llvm/ADT/Optional.h" #include "toolchain/lexer/token_kind.h" #include "toolchain/lexer/tokenized_buffer.h" @@ -88,22 +89,32 @@ class Parser2 { static_assert(sizeof(StateStackEntry) == 12, "StateStackEntry has unexpected size!"); + // Possible return values for FindListToken. + enum class ListTokenKind { + Comma, + Close, + CommaClose, + }; + + // The kind of brace expression being evaluated. + enum class BraceExpressionKind { Unknown, Value, Type }; + Parser2(ParseTree& tree, TokenizedBuffer& tokens, TokenDiagnosticEmitter& emitter); auto Parse() -> void; - // Adds a node to the parse tree that is fully parsed, has no children - // ("leaf"), and has a subsequent sibling. - // - // This sets up the next sibling of the node to be the next node in the parse - // tree's preorder sequence. + // Adds a node to the parse tree that has no children (a leaf). auto AddLeafNode(ParseNodeKind kind, TokenizedBuffer::Token token, bool has_error = false) -> void; + // Adds a node to the parse tree that has children. auto AddNode(ParseNodeKind kind, TokenizedBuffer::Token token, int subtree_start, bool has_error) -> void; + // Returns the current position and moves past it. + auto Consume() -> TokenizedBuffer::Token { return *(position_++); } + // Parses a close paren token corresponding to the given open paren token, // possibly skipping forward and diagnosing if necessary. Creates a parse node // of the specified kind if successful. @@ -163,6 +174,15 @@ class Parser2 { // whether there's an error, it's expected that parsing continues. auto DiagnoseOperatorFixity(OperatorFixity fixity) -> void; + // If the current position is a `,`, consumes it, adds the provided token, and + // returns `Comma`. Returns `Close` if the current position is close_token + // (for example, `)`). `CommaClose` indicates it found both (for example, + // `,)`). Handles cases where invalid tokens are present by advancing the + // position, and may emit errors. Pass already_has_error in order to suppress + // duplicate errors. + auto ConsumeListToken(ParseNodeKind comma_kind, TokenKind close_kind, + bool already_has_error) -> ListTokenKind; + // Gets the kind of the next token to be consumed. auto PositionKind() const -> TokenKind { return tokens_.GetKind(*position_); } @@ -205,43 +225,66 @@ class Parser2 { // Pushes a constructed state onto the stack. auto PushState(StateStackEntry state) -> void { state_stack_.push_back(state); + // Verify the stack doesn't grow unbounded by programming error. + CARBON_CHECK(state_stack_.size() < (1 << 20)); } // Propagates an error up the state stack, to the parent state. auto ReturnErrorOnState() -> void { state_stack_.back().has_error = true; } + // Returns the appropriate ParserState for the input kind. + static auto BraceExpressionKindToParserState(BraceExpressionKind kind, + ParserState type, + ParserState value, + ParserState unknown) + -> ParserState; + + // Prints a diagnostic for brace expression syntax errors. + auto HandleBraceExpressionParameterError(StateStackEntry state, + BraceExpressionKind kind) -> void; + + // Handles BraceExpressionParameterAs(Type|Value|Unknown). + auto HandleBraceExpressionParameter(BraceExpressionKind kind) -> void; + + // Handles BraceExpressionParameterAfterDesignatorAs(Type|Value|Unknown). + auto HandleBraceExpressionParameterAfterDesignator(BraceExpressionKind kind) + -> void; + + // Handles BraceExpressionParameterFinishAs(Type|Value|Unknown). + auto HandleBraceExpressionParameterFinish(BraceExpressionKind kind) -> void; + + // Handles BraceExpressionFinishAs(Type|Value|Unknown). + auto HandleBraceExpressionFinish(BraceExpressionKind kind) -> void; + + // Handles DesignatorAs. + auto HandleDesignator(bool as_struct) -> void; + // When handling errors before the start of the definition, treat it as a // declaration. Recover to a semicolon when it makes sense as a possible // function end, otherwise use the fn token for the error. auto HandleFunctionError(StateStackEntry state, bool skip_past_likely_end) -> void; - // Handles a code block in the context of a statement scope. - auto HandleCodeBlock() -> void; - - // Handles parsing of a function parameter list, including commas and the - // close paren. - auto HandleFunctionParameterList(bool is_start) -> void; + // Handles ParenExpressionParameterFinish(AsUnknown|AsTuple) + auto HandleParenExpressionParameterFinish(bool as_tuple) -> void; // Handles the start of a pattern. // If the start of the pattern is invalid, it's the responsibility of the // outside context to advance past the pattern. auto HandlePatternStart(PatternKind pattern_kind) -> void; - // Handles a single statement. While typically within a statement block, this - // can also be used for error recovery where we expect a statement block and - // are missing braces. - auto HandleStatement(TokenKind token_kind) -> void; - - // Handles a `if` statement at the start `if` token. - auto HandleStatementIf() -> void; + // Handles the end of a pattern. + auto HandlePatternFinish() -> bool; // Handles the `;` after a keyword statement. auto HandleStatementKeywordFinish(TokenKind token_kind, ParseNodeKind node_kind) -> void; - // Handles a `while` statement at the start `while` token. - auto HandleStatementWhile() -> void; + // Handles VarAs(RequireSemicolon|NoSemicolon). + auto HandleVar(bool require_semicolon) -> void; + + // Handles VarFinishAs(RequireSemicolon|NoSemicolon). + auto HandleVarFinish(bool require_semicolon) -> void; // `clang-format` has a bug with spacing around `->` returns in macros. See // https://bugs.llvm.org/show_bug.cgi?id=48320 for details. diff --git a/toolchain/parser/parser_state.def b/toolchain/parser/parser_state.def index 38731af928b3..0cdbefc8e862 100644 --- a/toolchain/parser/parser_state.def +++ b/toolchain/parser/parser_state.def @@ -13,100 +13,521 @@ #error "Must define the x-macro to use this file." #endif +// Handles the `{` of a brace expression. +// +// If `CloseCurlyBrace`: +// 1. BraceExpressionFinishAsUnknown +// Else: +// 1. BraceExpressionParameterAsUnknown +// 2. BraceExpressionFinishAsUnknown +CARBON_PARSER_STATE(BraceExpression) + +// Handles a brace expression parameter. Note this will always start as unknown, +// but should be known after the first valid parameter. All later inconsistent +// parameters are invalid. +// +// If valid: +// 1. DesignatorExpressionAsStruct +// 2. BraceExpressionParameterAfterDesignatorAs(Type|Value|Unknown) +// Else: +// 1. BraceExpressionParameterFinishAs(Type|Value|Unknown) +CARBON_PARSER_STATE(BraceExpressionParameterAsType) +CARBON_PARSER_STATE(BraceExpressionParameterAsValue) +CARBON_PARSER_STATE(BraceExpressionParameterAsUnknown) + +// Handles a brace expression parameter after the initial designator. This +// should be at a `:` or `=`, depending on whether it's a type or value literal. +// +// If valid: +// 1. Expression +// 2. BraceExpressionParameterFinishAs(Type|Value|Unknown) +// Else: +// 1. BraceExpressionParameterFinishAs(Type|Value|Unknown) +CARBON_PARSER_STATE(BraceExpressionParameterAfterDesignatorAsType) +CARBON_PARSER_STATE(BraceExpressionParameterAfterDesignatorAsValue) +CARBON_PARSER_STATE(BraceExpressionParameterAfterDesignatorAsUnknown) + +// Handles the end of a brace expression parameter. +// +// If `Comma`: +// 1. BraceExpressionParameterAsUnknown +// Else: +// (state done) +CARBON_PARSER_STATE(BraceExpressionParameterFinishAsType) +CARBON_PARSER_STATE(BraceExpressionParameterFinishAsValue) +CARBON_PARSER_STATE(BraceExpressionParameterFinishAsUnknown) + +// Handles the `}` of a brace expression. +// +// Always: +// (state done) +CARBON_PARSER_STATE(BraceExpressionFinishAsType) +CARBON_PARSER_STATE(BraceExpressionFinishAsValue) +CARBON_PARSER_STATE(BraceExpressionFinishAsUnknown) + +// Handles a call expression `(...)`. +// +// If `CloseParen`: +// 1. CallExpressionFinish +// Else: +// 1. Expression +// 2. CallExpressionParameterFinish +// 3. CallExpressionFinish +CARBON_PARSER_STATE(CallExpression) + +// Handles the `,` or `)` after a call parameter. +// +// If `Comma`: +// 1. Expression +// 2. CallExpressionParameterFinish +// Else: +// (state done) +CARBON_PARSER_STATE(CallExpressionParameterFinish) + +// Handles finishing the call expression. +// +// Always: +// (state done) +CARBON_PARSER_STATE(CallExpressionFinish) + +// Handles processing at the `{` on a typical code block. +// +// If `OpenCurlyBrace`: +// 1. StatementScopeLoop +// 2. CodeBlockFinish +// Else: +// 1. Statement +// 2. CodeBlockFinish +CARBON_PARSER_STATE(CodeBlock) + // Handles processing at the `}` on a typical code block, after a statement // scope is done. +// +// Always: +// (state done) CARBON_PARSER_STATE(CodeBlockFinish) // Handles processing of a declaration scope. Things like fn, class, interface, // and so on. -CARBON_PARSER_STATE(Declaration) +// +// If `EndOfFile`: +// (state done) +// If `Fn`: +// 1. FunctionIntroducer +// 2. DeclarationLoop +// If `Package`: +// 1. Package +// 2. DeclarationLoop +// If `Semi`: +// 1. DeclarationLoop +// If `Var`: +// 1. Var +// 2. DeclarationLoop +// Else: +// 1. DeclarationLoop +CARBON_PARSER_STATE(DeclarationLoop) // Handles a designator expression, such as `.z` in `x.(y.z)`. -CARBON_PARSER_STATE(DesignatorExpression) +// +// Always: +// (state done) +CARBON_PARSER_STATE(DesignatorAsExpression) +CARBON_PARSER_STATE(DesignatorAsStruct) // Handles processing of an expression. +// +// If valid prefix operator: +// 1. Expression +// 2. ExpressionLoopForPrefix +// Else: +// 1. ExpressionInPostfix +// 2. ExpressionLoop CARBON_PARSER_STATE(Expression) -// Handles processing of an expression. -CARBON_PARSER_STATE(ExpressionLoop) -CARBON_PARSER_STATE(ExpressionLoopForBinary) -CARBON_PARSER_STATE(ExpressionLoopForPrefix) - // Handles the initial part of postfix expressions, such as an identifier or // literal value, then proceeds to the loop. +// +// If `Identifier` or literal (including type literals): +// 1. ExpressionInPostfixLoop +// If `OpenCurlyBrace`: +// 1. BraceExpression +// 2. ExpressionInPostfixLoop +// If `OpenParen`: +// 1. ParenExpression +// 2. ExpressionInPostfixLoop +// Else: +// (state done) CARBON_PARSER_STATE(ExpressionInPostfix) // Handles looping through elements following the initial postfix expression, // such as designators or parenthesized parameters. +// +// If `Period`: +// 1. DesignatorAsExpression +// 2. ExpressionInPostfixLoop +// If `OpenParen`: +// 1. CallExpression +// 2. ExpressionInPostfixLoop +// Else: +// (state done) CARBON_PARSER_STATE(ExpressionInPostfixLoop) +// Handles processing of an expression. +// +// If binary operator: +// 1. Expression +// 2. ExpressionLoopForBinary +// If postfix operator: +// 1. ExpressionLoop +// Else: +// (state done) +CARBON_PARSER_STATE(ExpressionLoop) + +// Completes an ExpressionLoop pass by adding an infix operator, then goes back +// to ExpressionLoop. +// +// Always: +// 1. ExpressionLoop +CARBON_PARSER_STATE(ExpressionLoopForBinary) + +// Completes an ExpressionLoop pass by adding a prefix operator, then goes back +// to ExpressionLoop. +// +// Always: +// 1. ExpressionLoop +CARBON_PARSER_STATE(ExpressionLoopForPrefix) + // Handles the `;` for an expression statement, which is different from most // keyword statements. +// +// Always: +// (state done) CARBON_PARSER_STATE(ExpressionStatementFinish) // Handles processing of a function's `fn (`, and enqueues parameter list // handling. +// +// If invalid: +// (state done) +// If parenthesized parameters: +// 1. PatternAsFunctionParameter +// 2. FunctionParameterListFinish +// 3. FunctionAfterParameterList +// Else: +// 1. FunctionParameterListFinish +// 2. FunctionAfterParameterList CARBON_PARSER_STATE(FunctionIntroducer) // Handles processing of a function's parameter list `)`. +// +// Always: +// (state done) CARBON_PARSER_STATE(FunctionParameterListFinish) // Handles processing of a function's syntax after `)`, primarily the // possibility a `->` return type is there. Always enqueues signature finish // handling. +// +// If `MinusGreater`: +// 1. Expression +// 2. FunctionReturnTypeFinish +// 3. FunctionSignatureFinish +// Else: +// 1. FunctionSignatureFinish CARBON_PARSER_STATE(FunctionAfterParameterList) // Finishes a function return type. +// +// Always: +// (state done) CARBON_PARSER_STATE(FunctionReturnTypeFinish) // Finishes a function signature. If it's a declaration, the function is done; // otherwise, this also starts definition processing. +// +// If `Semi`: +// (state done) +// If `OpenCurlyBrace`: +// 1. StatementScopeLoop +// 2. FunctionDefinitionFinish +// Else: +// (state done) CARBON_PARSER_STATE(FunctionSignatureFinish) // Finishes a function definition. +// +// Always: +// (state done) CARBON_PARSER_STATE(FunctionDefinitionFinish) +// Handles `package`. +// +// Always: +// (state done) +CARBON_PARSER_STATE(Package) + // Handles the processing of a `(condition)` up through the expression. +// +// Always: +// 1. Expression +// 2. ParenConditionFinish CARBON_PARSER_STATE(ParenCondition) // Finishes the processing of a `(condition)` after the expression. +// +// Always: +// (state done) CARBON_PARSER_STATE(ParenConditionFinish) +// Handles the `(` of a parenthesized expression. +// +// If `CloseParen`: +// 1. ParenExpressionFinishAsTuple +// Else: +// 1. Expression +// 2. ParenExpressionParameterFinishAsUnknown +// 3. ParenExpressionFinish +CARBON_PARSER_STATE(ParenExpression) + +// Handles the end of a parenthesized expression's parameter. This will start as +// AsUnknown on the first parameter; if there are more, it switches to AsTuple +// processing. +// +// If `Comma` without `CloseParen`: +// 1. Expression +// 2. ParenExpressionParameterFinishAsTuple +// SPECIAL: Parent becomes ParenExpressionFinishAsTuple +// If `Comma` with `CloseParen`: +// (state done) +// SPECIAL: Parent becomes ParenExpressionFinishAsTuple +// Else `CloseParen`: +// (state done) +CARBON_PARSER_STATE(ParenExpressionParameterFinishAsUnknown) +CARBON_PARSER_STATE(ParenExpressionParameterFinishAsTuple) + +// Handles the `)` of a parenthesized expression. +// +// Always: +// (state done) +CARBON_PARSER_STATE(ParenExpressionFinish) +CARBON_PARSER_STATE(ParenExpressionFinishAsTuple) + +// Handles pattern parsing for a function parameter, enqueuing type expression +// processing. Proceeds to the matching Finish state when done. +// +// If valid: +// 1. Expression +// 2. PatternAsFunctionParameterFinish +// Else: +// 1. PatternAsFunctionParameterFinish +CARBON_PARSER_STATE(PatternAsFunctionParameter) + +// Finishes function parameter processing, including `,`. If there are more +// parameters, enqueues another parameter processing state. +// +// If `Comma` without `CloseParen`: +// 1. PatternAsFunctionParameter +// Else: +// (state done) +CARBON_PARSER_STATE(PatternAsFunctionParameterFinish) + +// Handles pattern parsing for a `var` statement, enqueuing type expression +// processing. Proceeds to the matching Finish state when done. +// +// +// If valid: +// 1. Expression +// 2. PatternAsVariableFinish +// Else: +// 1. PatternAsVariableFinish +CARBON_PARSER_STATE(PatternAsVariable) + +// Finishes `var` pattern processing. +// +// Always: +// (state done) +CARBON_PARSER_STATE(PatternAsVariableFinish) + +// Handles a single statement. While typically within a statement block, this +// can also be used for error recovery where we expect a statement block and +// are missing braces. +// +// If `Break`: +// 1. StatementBreakFinish +// (state done) +// If `Continue`: +// 1. StatementContinueFinish +// (state done) +// If `For`: +// 1. StatementForHeader +// 2. StatementForFinish +// If `If`: +// 1. StatementIf +// If `Return`: +// 1. StatementReturn +// If `Var`: +// 1. VarAsRequireSemicolon +// If `While`: +// 1. StatementWhile +// Else: +// 1. Expression +// 2. ExpressionStatementFinish +CARBON_PARSER_STATE(Statement) + // Handles `break` processing at the `;`. +// +// Always: +// (state done) CARBON_PARSER_STATE(StatementBreakFinish) // Handles `continue` processing at the `;`. +// +// Always: +// (state done) CARBON_PARSER_STATE(StatementContinueFinish) +// Handles `for` processing of `(var`, proceeding to a pattern before +// continuing. +// +// If no `OpenParen`: +// 1. CodeBlock +// If `Var`: +// 1. VarAsNoSemicolon +// 2. StatementForHeaderIn +// Else: +// 1. StatementForHeaderIn +CARBON_PARSER_STATE(StatementForHeader) + +// Handles `for` procesisng of `in`, proceeding to an expression before +// continuing. +// +// If `In` or `Colon`: +// 1. Expression +// 2. StatementForHeaderFinish +// Else: +// 1. StatementForHeaderFinish +CARBON_PARSER_STATE(StatementForHeaderIn) + +// Handles `for` processing of `)`, proceeding to the statement block. +// +// Always: +// 1. CodeBlock +CARBON_PARSER_STATE(StatementForHeaderFinish) + +// Handles `for` processing at the final `}`. +// +// Always: +// (state done) +CARBON_PARSER_STATE(StatementForFinish) + +// Handles `if` processing at the start. +// +// Always: +// 1. ParenCondition +// 2. StatementIfConditionFinish +CARBON_PARSER_STATE(StatementIf) + // Handles `if` processing between the condition and start of the first code // block. +// +// Always: +// 1. CodeBlock +// 2. StatementIfThenBlockFinish CARBON_PARSER_STATE(StatementIfConditionFinish) // Handles `if` processing after the end of the first code block, with the // optional `else`. +// +// If `Else` then `If`: +// 1. CodeBlock +// 2. StatementIfElseBlockFinish +// If `Else`: +// 1. StatementIf +// 2. StatementIfElseBlockFinish +// Else: +// (state done) CARBON_PARSER_STATE(StatementIfThenBlockFinish) // Handles `if` processing after a provided `else` code block. +// +// Always: +// (state done) CARBON_PARSER_STATE(StatementIfElseBlockFinish) -// Handles `return` processing at the `;`. +// Handles `return` processing. +// +// If `Semi`: +// 1. StatementReturnFinish +// Else: +// 1. Expression +// 2. StatementReturnFinish +CARBON_PARSER_STATE(StatementReturn) + +// Handles `return` processing at the `;` when there's an expression. +// +// Always: +// (state done) CARBON_PARSER_STATE(StatementReturnFinish) +// Handles processing of statements within a scope. +// +// If `CloseCurlyBrace`: +// (state done) +// Else: +// 1. Statement +// 2. StatementScopeLoop +CARBON_PARSER_STATE(StatementScopeLoop) + +// Handles `while` processing. +// +// Always: +// 1. ParenCondition +// 2. StatementWhileConditionFinish +CARBON_PARSER_STATE(StatementWhile) + // Handles `while` processing between the condition and start of the code block. +// +// Always: +// 1. CodeBlock +// 2. StatementWhileBlockFinish CARBON_PARSER_STATE(StatementWhileConditionFinish) // Handles `while` processing after the end of the code block. +// +// Always: +// (state done) CARBON_PARSER_STATE(StatementWhileBlockFinish) -// Handles pattern parsing for a function parameter, enqueuing type expression -// processing. Proceeds to the matching Finish state when done. -CARBON_PARSER_STATE(PatternForFunctionParameter) +// Handles the start of a `var`. +// +// Always: +// 1. PatternAsVariable +// 2. VarAfterPattern +// 3. VarFinishAs(RequireSemicolon|NoSemicolon) +CARBON_PARSER_STATE(VarAsRequireSemicolon) +CARBON_PARSER_STATE(VarAsNoSemicolon) -// Finishes function parameter processing, including `,`. If there are more -// parameters, enqueues another parameter processing state. -CARBON_PARSER_STATE(PatternForFunctionParameterFinish) +// Handles `var` after the pattern, either followed by an initializer or the +// semicolon. +// +// If `Equal`: +// 1. Expression +// 2. VarAfterInitializer +// Else: +// (state done) +CARBON_PARSER_STATE(VarAfterPattern) -// Handles processing of statements within a scope. -CARBON_PARSER_STATE(StatementScopeLoop) +// Handles `var` after the initializer, wrapping up its subtree. +// +// Always: +// (state done) +CARBON_PARSER_STATE(VarAfterInitializer) + +// Handles `var` parsing at the end. +// +// Always: +// (state done) +CARBON_PARSER_STATE(VarFinishAsRequireSemicolon) +CARBON_PARSER_STATE(VarFinishAsNoSemicolon) #undef CARBON_PARSER_STATE