From d3be5c2827e4835ca70562af96302d1504f3c3e6 Mon Sep 17 00:00:00 2001 From: Jon Ross-Perkins Date: Thu, 10 Nov 2022 09:24:54 -0800 Subject: [PATCH] Parser2 support for while and expressions (#2379) Adds `while` and most of the expression support. Splits apart the fixity test to demonstrate more closely which bits are still failing. ``` //toolchain/parser/testdata:basics/fail_paren_match_regression.carbon.test FAILED in 0.8s //toolchain/parser/testdata:basics/function_call.carbon.test FAILED in 0.7s //toolchain/parser/testdata:basics/package.carbon.test FAILED in 0.7s //toolchain/parser/testdata:basics/structs.carbon.test FAILED in 0.7s //toolchain/parser/testdata:basics/tuples.carbon.test FAILED in 0.6s //toolchain/parser/testdata:basics/var.carbon.test FAILED in 0.7s //toolchain/parser/testdata:for/fail_colon_instead_of_in.carbon.test FAILED in 0.7s //toolchain/parser/testdata:for/fail_missing_in.carbon.test FAILED in 0.7s //toolchain/parser/testdata:for/fail_missing_var.carbon.test FAILED in 0.7s //toolchain/parser/testdata:for/nested.carbon.test FAILED in 0.6s //toolchain/parser/testdata:for/simple.carbon.test FAILED in 0.8s //toolchain/parser/testdata:function/definition/with_params.carbon.test FAILED in 0.8s //toolchain/parser/testdata:operators/fixity_in_call.carbon.test FAILED in 0.7s //toolchain/parser/testdata:operators/fixity_in_var.carbon.test FAILED in 0.8s ``` Co-authored-by: Chandler Carruth --- toolchain/parser/parser2.cpp | 461 +++++++++++++++--- toolchain/parser/parser2.h | 115 ++++- toolchain/parser/parser_state.def | 37 +- .../testdata/operators/fixity_in_call.carbon | 29 ++ .../operators/fixity_in_params.carbon | 26 + .../testdata/operators/fixity_in_var.carbon | 36 ++ ...ixity.carbon => fixity_with_assign.carbon} | 45 +- 7 files changed, 614 insertions(+), 135 deletions(-) create mode 100644 toolchain/parser/testdata/operators/fixity_in_call.carbon create mode 100644 toolchain/parser/testdata/operators/fixity_in_params.carbon create mode 100644 toolchain/parser/testdata/operators/fixity_in_var.carbon rename toolchain/parser/testdata/operators/{fixity.carbon => fixity_with_assign.carbon} (50%) diff --git a/toolchain/parser/parser2.cpp b/toolchain/parser/parser2.cpp index 1354c9608ac4..f1c6c79aa927 100644 --- a/toolchain/parser/parser2.cpp +++ b/toolchain/parser/parser2.cpp @@ -17,6 +17,35 @@ namespace Carbon { +// May be emitted a couple different ways as part of operator parsing. +CARBON_DIAGNOSTIC( + OperatorRequiresParentheses, Error, + "Parentheses are required to disambiguate operator precedence."); + +// A relative location for characters in errors. +enum class RelativeLocation : int8_t { + Around, + After, + Before, +}; + +// Adapts RelativeLocation for use with formatv. +static auto operator<<(llvm::raw_ostream& out, RelativeLocation loc) + -> llvm::raw_ostream& { + switch (loc) { + case RelativeLocation::Around: + out << "around"; + break; + case RelativeLocation::After: + out << "after"; + break; + case RelativeLocation::Before: + out << "before"; + break; + } + return out; +} + class Parser2::PrettyStackTraceParseState : public llvm::PrettyStackTraceEntry { public: explicit PrettyStackTraceParseState(const Parser2* parser) @@ -30,7 +59,7 @@ class Parser2::PrettyStackTraceParseState : public llvm::PrettyStackTraceEntry { output << "\t" << i << ".\t" << entry.state; Print(output, entry.token); } - output << "\tabort\tposition_"; + output << "\tcursor\tposition_"; Print(output, *parser_->position_); } @@ -142,24 +171,6 @@ auto Parser2::FindNextOf(std::initializer_list desired_kinds) } } -auto Parser2::Parse() -> void { - // Traces state_stack_. This runs even in opt because it's low overhead. - PrettyStackTraceParseState pretty_stack(this); - - PushState(ParserState::Declaration()); - while (!state_stack_.empty()) { - switch (state_stack_.back().state) { -#define CARBON_PARSER_STATE(Name) \ - case ParserState::Name(): \ - Handle##Name##State(); \ - break; -#include "toolchain/parser/parser_state.def" - } - } - - AddLeafNode(ParseNodeKind::FileEnd(), *position_); -} - auto Parser2::SkipMatchingGroup() -> bool { if (!PositionKind().IsOpeningSymbol()) { return false; @@ -228,7 +239,7 @@ auto Parser2::HandleCodeBlock() -> void { PushState(ParserState::CodeBlockFinish()); if (ConsumeAndAddLeafNodeIf(TokenKind::OpenCurlyBrace(), ParseNodeKind::CodeBlockStart())) { - PushState(ParserState::StatementScope()); + PushState(ParserState::StatementScopeLoop()); } else { AddLeafNode(ParseNodeKind::CodeBlockStart(), *position_, /*has_error=*/true); @@ -241,6 +252,143 @@ auto Parser2::HandleCodeBlock() -> void { } } +// Determines whether the given token is considered to be the start of an +// operand according to the rules for infix operator parsing. +static auto IsAssumedStartOfOperand(TokenKind kind) -> bool { + return kind.IsOneOf({TokenKind::OpenParen(), TokenKind::Identifier(), + TokenKind::IntegerLiteral(), TokenKind::RealLiteral(), + TokenKind::StringLiteral()}); +} + +// Determines whether the given token is considered to be the end of an +// operand according to the rules for infix operator parsing. +static auto IsAssumedEndOfOperand(TokenKind kind) -> bool { + return kind.IsOneOf({TokenKind::CloseParen(), TokenKind::CloseCurlyBrace(), + TokenKind::CloseSquareBracket(), TokenKind::Identifier(), + TokenKind::IntegerLiteral(), TokenKind::RealLiteral(), + TokenKind::StringLiteral()}); +} + +// Determines whether the given token could possibly be the start of an +// operand. This is conservatively correct, and will never incorrectly return +// `false`, but can incorrectly return `true`. +static auto IsPossibleStartOfOperand(TokenKind kind) -> bool { + return !kind.IsOneOf({TokenKind::CloseParen(), TokenKind::CloseCurlyBrace(), + TokenKind::CloseSquareBracket(), TokenKind::Comma(), + TokenKind::Semi(), TokenKind::Colon()}); +} + +auto Parser2::IsLexicallyValidInfixOperator() -> bool { + CARBON_CHECK(position_ != end_) << "Expected an operator token."; + + bool leading_space = tokens_.HasLeadingWhitespace(*position_); + bool trailing_space = tokens_.HasTrailingWhitespace(*position_); + + // If there's whitespace on both sides, it's an infix operator. + if (leading_space && trailing_space) { + return true; + } + + // If there's whitespace on exactly one side, it's not an infix operator. + if (leading_space || trailing_space) { + return false; + } + + // Otherwise, for an infix operator, the preceding token must be any close + // bracket, identifier, or literal and the next token must be an open paren, + // identifier, or literal. + if (position_ == tokens_.tokens().begin() || + !IsAssumedEndOfOperand(tokens_.GetKind(*(position_ - 1))) || + !IsAssumedStartOfOperand(tokens_.GetKind(*(position_ + 1)))) { + return false; + } + + return true; +} + +auto Parser2::IsTrailingOperatorInfix() -> bool { + if (position_ == end_) { + return false; + } + + // An operator that follows the infix operator rules is parsed as + // infix, unless the next token means that it can't possibly be. + if (IsLexicallyValidInfixOperator() && + IsPossibleStartOfOperand(tokens_.GetKind(*(position_ + 1)))) { + return true; + } + + // A trailing operator with leading whitespace that's not valid as infix is + // not valid at all. If the next token looks like the start of an operand, + // then parse as infix, otherwise as postfix. Either way we'll produce a + // diagnostic later on. + if (tokens_.HasLeadingWhitespace(*position_) && + IsAssumedStartOfOperand(tokens_.GetKind(*(position_ + 1)))) { + return true; + } + + return false; +} + +auto Parser2::DiagnoseOperatorFixity(OperatorFixity fixity) -> void { + if (fixity == OperatorFixity::Infix) { + // Infix operators must satisfy the infix operator rules. + if (!IsLexicallyValidInfixOperator()) { + CARBON_DIAGNOSTIC(BinaryOperatorRequiresWhitespace, Error, + "Whitespace missing {0} binary operator.", + RelativeLocation); + emitter_.Emit(*position_, BinaryOperatorRequiresWhitespace, + tokens_.HasLeadingWhitespace(*position_) + ? RelativeLocation::After + : (tokens_.HasTrailingWhitespace(*position_) + ? RelativeLocation::Before + : RelativeLocation::Around)); + } + } else { + bool prefix = fixity == OperatorFixity::Prefix; + + // Whitespace is not permitted between a symbolic pre/postfix operator and + // its operand. + if (PositionKind().IsSymbol() && + (prefix ? tokens_.HasTrailingWhitespace(*position_) + : tokens_.HasLeadingWhitespace(*position_))) { + CARBON_DIAGNOSTIC(UnaryOperatorHasWhitespace, Error, + "Whitespace is not allowed {0} this unary operator.", + RelativeLocation); + emitter_.Emit( + *position_, UnaryOperatorHasWhitespace, + prefix ? RelativeLocation::After : RelativeLocation::Before); + } + // Pre/postfix operators must not satisfy the infix operator rules. + if (IsLexicallyValidInfixOperator()) { + CARBON_DIAGNOSTIC(UnaryOperatorRequiresWhitespace, Error, + "Whitespace is required {0} this unary operator.", + RelativeLocation); + emitter_.Emit( + *position_, UnaryOperatorRequiresWhitespace, + prefix ? RelativeLocation::Before : RelativeLocation::After); + } + } +} + +auto Parser2::Parse() -> void { + // Traces state_stack_. This runs even in opt because it's low overhead. + PrettyStackTraceParseState pretty_stack(this); + + PushState(ParserState::Declaration()); + while (!state_stack_.empty()) { + switch (state_stack_.back().state) { +#define CARBON_PARSER_STATE(Name) \ + case ParserState::Name(): \ + Handle##Name##State(); \ + break; +#include "toolchain/parser/parser_state.def" + } + } + + AddLeafNode(ParseNodeKind::FileEnd(), *position_); +} + auto Parser2::HandleCodeBlockFinishState() -> void { auto state = PopState(); @@ -258,6 +406,7 @@ auto Parser2::HandleCodeBlockFinishState() -> void { auto Parser2::HandleDeclarationState() -> void { // This maintains the current state unless we're at the end of the file. + // TODO: INCOMPLETE switch (PositionKind()) { case TokenKind::EndOfFile(): { PopAndDiscardState(); @@ -288,8 +437,71 @@ auto Parser2::HandleDeclarationState() -> void { } } -auto Parser2::HandleExpressionFormPrimary() -> void { - // TODO: Handle OpenParen and OpenCurlyBrace. +auto Parser2::HandleDesignatorExpressionState() -> void { + auto state = PopState(); + + // `.` identifier + auto dot = ConsumeIf(TokenKind::Period()); + CARBON_CHECK(dot); + if (!ConsumeAndAddLeafNodeIf(TokenKind::Identifier(), + ParseNodeKind::DesignatedName())) { + CARBON_DIAGNOSTIC(ExpectedIdentifierAfterDot, Error, + "Expected identifier after `.`."); + emitter_.Emit(*position_, ExpectedIdentifierAfterDot); + // If we see a keyword, assume it was intended to be the designated name. + // TODO: Should keywords be valid in designators? + if (PositionKind().IsKeyword()) { + AddLeafNode(ParseNodeKind::DesignatedName(), *position_, + /*has_error=*/true); + ++position_; + } else { + state.has_error = true; + ReturnErrorOnState(); + } + } + + AddNode(ParseNodeKind::DesignatorExpression(), *dot, state.subtree_start, + state.has_error); +} + +auto Parser2::HandleExpressionState() -> void { + auto state = PopState(); + + // Check for a prefix operator. + if (auto operator_precedence = PrecedenceGroup::ForLeading(PositionKind())) { + if (PrecedenceGroup::GetPriority(state.ambient_precedence, + *operator_precedence) != + OperatorPriority::RightFirst) { + // The precedence rules don't permit this prefix operator in this + // context. Diagnose this, but carry on and parse it anyway. + emitter_.Emit(*position_, OperatorRequiresParentheses); + } else { + // Check that this operator follows the proper whitespace rules. + DiagnoseOperatorFixity(OperatorFixity::Prefix); + } + + PushStateForExpressionLoop(ParserState::ExpressionLoopForPrefix(), + state.ambient_precedence, *operator_precedence); + ++position_; + PushStateForExpression(*operator_precedence); + } else { + PushStateForExpressionLoop(ParserState::ExpressionLoop(), + state.ambient_precedence, + PrecedenceGroup::ForPostfixExpression()); + PushState(ParserState::ExpressionInPostfix()); + } +} + +auto Parser2::HandleExpressionInPostfixState() -> void { + PopAndDiscardState(); + + // Regardless of success or failure, we'll continue to the Loop state. + PushState(ParserState::ExpressionInPostfixLoop()); + + // Parses a primary expression, which is either a terminal portion of an + // expression tree, such as an identifier or literal, or a parenthesized + // expression. + // TODO: INCOMPLETE: Handle OpenParen and OpenCurlyBrace. switch (PositionKind()) { case TokenKind::Identifier(): AddLeafNode(ParseNodeKind::NameReference(), *position_); @@ -313,22 +525,105 @@ auto Parser2::HandleExpressionFormPrimary() -> void { ++position_; } -auto Parser2::HandleExpressionState() -> void { - // TODO: This is temporary, we should need this state. If not, maybe add an - // overload that uses pop_back instead of pop_back_val. +auto Parser2::HandleExpressionInPostfixLoopState() -> void { + // This is a cyclic state that repeats, so this state is typically pushed back + // on. auto state = PopState(); - (void)state; - HandleExpressionFormPrimary(); + switch (PositionKind()) { + case TokenKind::Period(): + PushState(state); + state.state = ParserState::DesignatorExpression(); + PushState(state); + break; + + case TokenKind::OpenParen(): + // expression = ParseCallExpression(start, !expression); + CARBON_FATAL() << "TODO: INCOMPLETE"; + break; + + default: + if (state.has_error) { + ReturnErrorOnState(); + } + break; + } } -auto Parser2::HandleExpressionForTypeState() -> void { - // TODO: This is temporary, we should need this state. If not, maybe add an - // overload that uses pop_back instead of pop_back_val. +auto Parser2::HandleExpressionLoopState() -> void { auto state = PopState(); - (void)state; - HandleExpressionFormPrimary(); + auto trailing_operator = + PrecedenceGroup::ForTrailing(PositionKind(), IsTrailingOperatorInfix()); + if (!trailing_operator) { + if (state.has_error) { + ReturnErrorOnState(); + } + return; + } + auto [operator_precedence, is_binary] = *trailing_operator; + + // TODO: If this operator is ambiguous with either the ambient precedence + // or the LHS precedence, and there's a variant with a different fixity + // that would work, use that one instead for error recovery. + if (PrecedenceGroup::GetPriority(state.ambient_precedence, + operator_precedence) != + OperatorPriority::RightFirst) { + // The precedence rules don't permit this operator in this context. Try + // again in the enclosing expression context. + if (state.has_error) { + ReturnErrorOnState(); + } + return; + } + + if (PrecedenceGroup::GetPriority(state.lhs_precedence, operator_precedence) != + OperatorPriority::LeftFirst) { + // Either the LHS operator and this operator are ambiguous, or the + // LHS operator is a unary operator that can't be nested within + // this operator. Either way, parentheses are required. + emitter_.Emit(*position_, OperatorRequiresParentheses); + state.has_error = true; + } else { + DiagnoseOperatorFixity(is_binary ? OperatorFixity::Infix + : OperatorFixity::Postfix); + } + + state.token = *position_; + ++position_; + + state.lhs_precedence = operator_precedence; + + if (is_binary) { + state.state = ParserState::ExpressionLoopForBinary(); + PushState(state); + PushStateForExpression(operator_precedence); + } else { + AddNode(ParseNodeKind::PostfixOperator(), state.token, state.subtree_start, + state.has_error); + state.has_error = false; + PushState(state); + } +} + +auto Parser2::HandleExpressionLoopForBinaryState() -> void { + auto state = PopState(); + + AddNode(ParseNodeKind::InfixOperator(), state.token, state.subtree_start, + state.has_error); + state.state = ParserState::ExpressionLoop(); + state.has_error = false; + PushState(state); +} + +auto Parser2::HandleExpressionLoopForPrefixState() -> void { + auto state = PopState(); + + AddNode(ParseNodeKind::PrefixOperator(), state.token, state.subtree_start, + state.has_error); + state.state = ParserState::ExpressionLoop(); + state.has_error = false; + PushState(state); } auto Parser2::HandleExpressionStatementFinishState() -> void { @@ -429,7 +724,7 @@ auto Parser2::HandleFunctionAfterParameterListState() -> void { if (PositionIs(TokenKind::MinusGreater())) { PushState(ParserState::FunctionReturnTypeFinish()); ++position_; - PushState(ParserState::ExpressionForType()); + PushStateForExpression(PrecedenceGroup::ForType()); } } @@ -458,7 +753,7 @@ auto Parser2::HandleFunctionSignatureFinishState() -> void { state.has_error = false; state.state = ParserState::FunctionDefinitionFinish(); PushState(state); - PushState(ParserState::StatementScope()); + PushState(ParserState::StatementScopeLoop()); break; } default: { @@ -492,7 +787,7 @@ auto Parser2::HandlePatternStart(PatternKind pattern_kind) -> void { break; } case PatternKind::Variable: { - CARBON_FATAL() << "TODO"; + CARBON_FATAL() << "TODO: INCOMPLETE"; break; } } @@ -523,34 +818,11 @@ auto Parser2::HandlePatternStart(PatternKind pattern_kind) -> void { // node. state.token = *(position_ + 1); PushState(state); - PushState(ParserState::ExpressionForType()); + PushStateForExpression(PrecedenceGroup::ForType()); AddLeafNode(ParseNodeKind::DeclaredName(), *position_); position_ += 2; } -auto Parser2::HandleKeywordStatementFinish(TokenKind token_kind, - ParseNodeKind node_kind) -> void { - auto state = PopState(); - - auto semi = - ConsumeAndAddLeafNodeIf(TokenKind::Semi(), ParseNodeKind::StatementEnd()); - if (!semi) { - CARBON_DIAGNOSTIC(ExpectedSemiAfter, Error, "Expected `;` after `{0}`.", - TokenKind); - emitter_.Emit(*position_, ExpectedSemiAfter, token_kind); - if (auto semi_token = SkipPastLikelyEnd(state.token)) { - AddLeafNode(ParseNodeKind::StatementEnd(), *semi_token, - /*has_error=*/true); - } - } - AddNode(node_kind, state.token, state.subtree_start, state.has_error); -} - -auto Parser2::HandleKeywordStatementFinishForReturnState() -> void { - HandleKeywordStatementFinish(TokenKind::Return(), - ParseNodeKind::ReturnStatement()); -} - auto Parser2::HandleParenConditionState() -> void { auto state = PopState(); @@ -647,6 +919,16 @@ auto Parser2::HandlePatternForFunctionParameterFinishState() -> void { } } +auto Parser2::HandleStatementBreakFinishState() -> void { + HandleStatementKeywordFinish(TokenKind::Break(), + ParseNodeKind::BreakStatement()); +} + +auto Parser2::HandleStatementContinueFinishState() -> void { + HandleStatementKeywordFinish(TokenKind::Continue(), + ParseNodeKind::ContinueStatement()); +} + auto Parser2::HandleStatementIf() -> void { PushState(ParserState::StatementIfConditionFinish()); PushState(ParserState::ParenCondition()); @@ -686,12 +968,65 @@ auto Parser2::HandleStatementIfElseBlockFinishState() -> void { state.has_error); } +auto Parser2::HandleStatementKeywordFinish(TokenKind token_kind, + ParseNodeKind node_kind) -> void { + auto state = PopState(); + + if (!ConsumeAndAddLeafNodeIf(TokenKind::Semi(), + ParseNodeKind::StatementEnd())) { + CARBON_DIAGNOSTIC(ExpectedSemiAfter, Error, "Expected `;` after `{0}`.", + TokenKind); + emitter_.Emit(*position_, ExpectedSemiAfter, token_kind); + if (auto semi_token = SkipPastLikelyEnd(state.token)) { + AddLeafNode(ParseNodeKind::StatementEnd(), *semi_token, + /*has_error=*/true); + } + } + AddNode(node_kind, state.token, state.subtree_start, state.has_error); +} + +auto Parser2::HandleStatementReturnFinishState() -> void { + HandleStatementKeywordFinish(TokenKind::Return(), + ParseNodeKind::ReturnStatement()); +} + +auto Parser2::HandleStatementWhile() -> void { + PushState(ParserState::StatementWhileConditionFinish()); + PushState(ParserState::ParenCondition()); + ++position_; +} + +auto Parser2::HandleStatementWhileConditionFinishState() -> void { + auto state = PopState(); + + state.state = ParserState::StatementWhileBlockFinish(); + PushState(state); + HandleCodeBlock(); +} + +auto Parser2::HandleStatementWhileBlockFinishState() -> void { + auto state = PopState(); + + AddNode(ParseNodeKind::WhileStatement(), state.token, state.subtree_start, + state.has_error); +} + auto Parser2::HandleStatement(TokenKind token_kind) -> void { switch (token_kind) { case TokenKind::If(): { HandleStatementIf(); break; } + case TokenKind::Break(): { + PushState(ParserState::StatementBreakFinish()); + ++position_; + break; + } + case TokenKind::Continue(): { + PushState(ParserState::StatementContinueFinish()); + ++position_; + break; + } case TokenKind::Return(): { auto return_token = *position_; if (tokens_.GetKind(*(position_ + 1)) == TokenKind::Semi()) { @@ -701,12 +1036,16 @@ auto Parser2::HandleStatement(TokenKind token_kind) -> void { /*has_error=*/false); position_ += 2; } else { - PushState(ParserState::KeywordStatementFinishForReturn()); + PushState(ParserState::StatementReturnFinish()); ++position_; PushState(ParserState::Expression()); } break; } + case TokenKind::While(): { + HandleStatementWhile(); + break; + } default: { PushState(ParserState::ExpressionStatementFinish()); PushState(ParserState::Expression()); @@ -715,7 +1054,7 @@ auto Parser2::HandleStatement(TokenKind token_kind) -> void { } } -auto Parser2::HandleStatementScopeState() -> void { +auto Parser2::HandleStatementScopeLoopState() -> void { // This maintains the current state until we're at the end of the scope. auto token_kind = PositionKind(); diff --git a/toolchain/parser/parser2.h b/toolchain/parser/parser2.h index 3da287095900..2e551f2fb715 100644 --- a/toolchain/parser/parser2.h +++ b/toolchain/parser/parser2.h @@ -11,6 +11,7 @@ #include "toolchain/parser/parse_node_kind.h" #include "toolchain/parser/parse_tree.h" #include "toolchain/parser/parser_state.h" +#include "toolchain/parser/precedence.h" namespace Carbon { @@ -28,6 +29,9 @@ class Parser2 { } private: + // Possible operator fixities for errors. + enum class OperatorFixity { Prefix, Infix, Postfix }; + // Supported kinds of patterns for HandlePattern. enum class PatternKind { Parameter, Variable }; @@ -38,21 +42,52 @@ class Parser2 { struct StateStackEntry { StateStackEntry(ParserState state, TokenizedBuffer::Token token, int32_t subtree_start) - : state(state), token(token), subtree_start(subtree_start) {} + : StateStackEntry(state, PrecedenceGroup::ForTopLevelExpression(), + PrecedenceGroup::ForTopLevelExpression(), token, + subtree_start) {} + + StateStackEntry(ParserState state, PrecedenceGroup ambient_precedence, + PrecedenceGroup lhs_precedence, + TokenizedBuffer::Token token, int32_t subtree_start) + : state(state), + ambient_precedence(ambient_precedence), + lhs_precedence(lhs_precedence), + token(token), + subtree_start(subtree_start) {} // The state. ParserState state; + // Set to true to indicate that an error was found, and that contextual + // error recovery may be needed. + bool has_error = false; + + // Precedence information used by expression states in order to determine + // operator precedence. The ambient_precedence deals with how the expression + // should interact with outside context, while the lhs_precedence is + // specific to the lhs of an operator expression. + PrecedenceGroup ambient_precedence; + PrecedenceGroup lhs_precedence; + // A token providing context based on the subtree. This will typically be // the first token in the subtree, but may sometimes be a token within. It // will typically be used for the subtree's root node. TokenizedBuffer::Token token; // The offset within the ParseTree of the subtree start. int32_t subtree_start; - // Set to true to indicate that an error was found, and that contextual - // error recovery may be needed. - bool has_error = false; }; + // We expect StateStackEntry to fit into 12 bytes: + // state = 1 byte + // has_error = 1 byte + // ambient_precedence = 1 byte + // lhs_precedence = 1 byte + // token = 4 bytes + // subtree_start = 4 bytes + // If it becomes bigger, it'd be worth examining better packing; it should be + // feasible to pack the 1-byte entries more tightly. + static_assert(sizeof(StateStackEntry) == 12, + "StateStackEntry has unexpected size!"); + Parser2(ParseTree& tree, TokenizedBuffer& tokens, TokenDiagnosticEmitter& emitter); @@ -89,14 +124,6 @@ class Parser2 { auto FindNextOf(std::initializer_list desired_kinds) -> llvm::Optional; - // Gets the kind of the next token to be consumed. - auto PositionKind() const -> TokenKind { return tokens_.GetKind(*position_); } - - // Tests whether the next token to be consumed is of the specified kind. - auto PositionIs(TokenKind kind) const -> bool { - return PositionKind() == kind; - } - // If the token is an opening symbol for a matched group, skips to the matched // closing symbol and returns true. Otherwise, returns false. auto SkipMatchingGroup() -> bool; @@ -123,11 +150,53 @@ class Parser2 { // Skip forward to the given token. Verifies that it is actually forward. auto SkipTo(TokenizedBuffer::Token t) -> void; + // Returns true if the current token satisfies the lexical validity rules + // for an infix operator. + auto IsLexicallyValidInfixOperator() -> bool; + + // Determines whether the current trailing operator should be treated as + // infix. + auto IsTrailingOperatorInfix() -> bool; + + // Diagnoses whether the current token is not written properly for the given + // fixity. For example, because mandatory whitespace is missing. Regardless of + // whether there's an error, it's expected that parsing continues. + auto DiagnoseOperatorFixity(OperatorFixity fixity) -> void; + + // Gets the kind of the next token to be consumed. + auto PositionKind() const -> TokenKind { return tokens_.GetKind(*position_); } + + // Tests whether the next token to be consumed is of the specified kind. + auto PositionIs(TokenKind kind) const -> bool { + return PositionKind() == kind; + } + + // Pops the state and keeps the value for inspection. + auto PopState() -> StateStackEntry { return state_stack_.pop_back_val(); } + + // Pops the state and discards it. + auto PopAndDiscardState() -> void { state_stack_.pop_back(); } + // Pushes a new state with the current position for context. auto PushState(ParserState state) -> void { PushState(StateStackEntry(state, *position_, tree_.size())); } + // Pushes a new expression state with specific precedence. + auto PushStateForExpression(PrecedenceGroup ambient_precedence) -> void { + PushState(StateStackEntry(ParserState::Expression(), ambient_precedence, + PrecedenceGroup::ForTopLevelExpression(), + *position_, tree_.size())); + } + + // Pushes a new state with detailed precedence for expression resume states. + auto PushStateForExpressionLoop(ParserState state, + PrecedenceGroup ambient_precedence, + PrecedenceGroup lhs_precedence) -> void { + PushState(StateStackEntry(state, ambient_precedence, lhs_precedence, + *position_, tree_.size())); + } + // Pushes a new state with the token for context. auto PushState(ParserState state, TokenizedBuffer::Token token) -> void { PushState(StateStackEntry(state, token, tree_.size())); @@ -138,20 +207,9 @@ class Parser2 { state_stack_.push_back(state); } - // Pops the state and keeps the value for inspection. - auto PopState() -> StateStackEntry { return state_stack_.pop_back_val(); } - - // Pops the state and discards it. - auto PopAndDiscardState() -> void { state_stack_.pop_back(); } - // Propagates an error up the state stack, to the parent state. auto ReturnErrorOnState() -> void { state_stack_.back().has_error = true; } - // Parses a primary expression, which is either a terminal portion of an - // expression tree, such as an identifier or literal, or a parenthesized - // expression. - auto HandleExpressionFormPrimary() -> void; - // When handling errors before the start of the definition, treat it as a // declaration. Recover to a semicolon when it makes sense as a possible // function end, otherwise use the fn token for the error. @@ -165,10 +223,6 @@ class Parser2 { // close paren. auto HandleFunctionParameterList(bool is_start) -> void; - // Handles the `;` after a keyword statement. - auto HandleKeywordStatementFinish(TokenKind token_kind, - ParseNodeKind node_kind) -> void; - // Handles the start of a pattern. // If the start of the pattern is invalid, it's the responsibility of the // outside context to advance past the pattern. @@ -182,6 +236,13 @@ class Parser2 { // Handles a `if` statement at the start `if` token. auto HandleStatementIf() -> void; + // Handles the `;` after a keyword statement. + auto HandleStatementKeywordFinish(TokenKind token_kind, + ParseNodeKind node_kind) -> void; + + // Handles a `while` statement at the start `while` token. + auto HandleStatementWhile() -> void; + // `clang-format` has a bug with spacing around `->` returns in macros. See // https://bugs.llvm.org/show_bug.cgi?id=48320 for details. #define CARBON_PARSER_STATE(Name) auto Handle##Name##State()->void; diff --git a/toolchain/parser/parser_state.def b/toolchain/parser/parser_state.def index e3003a3e2316..38731af928b3 100644 --- a/toolchain/parser/parser_state.def +++ b/toolchain/parser/parser_state.def @@ -21,11 +21,24 @@ CARBON_PARSER_STATE(CodeBlockFinish) // and so on. CARBON_PARSER_STATE(Declaration) +// Handles a designator expression, such as `.z` in `x.(y.z)`. +CARBON_PARSER_STATE(DesignatorExpression) + // Handles processing of an expression. CARBON_PARSER_STATE(Expression) -// As with Expression, but specifically uses the Type precedence grouping. -CARBON_PARSER_STATE(ExpressionForType) +// Handles processing of an expression. +CARBON_PARSER_STATE(ExpressionLoop) +CARBON_PARSER_STATE(ExpressionLoopForBinary) +CARBON_PARSER_STATE(ExpressionLoopForPrefix) + +// Handles the initial part of postfix expressions, such as an identifier or +// literal value, then proceeds to the loop. +CARBON_PARSER_STATE(ExpressionInPostfix) + +// Handles looping through elements following the initial postfix expression, +// such as designators or parenthesized parameters. +CARBON_PARSER_STATE(ExpressionInPostfixLoop) // Handles the `;` for an expression statement, which is different from most // keyword statements. @@ -53,15 +66,18 @@ CARBON_PARSER_STATE(FunctionSignatureFinish) // Finishes a function definition. CARBON_PARSER_STATE(FunctionDefinitionFinish) -// StatementFinish variants are just handling the `;` expectation. -CARBON_PARSER_STATE(KeywordStatementFinishForReturn) - // Handles the processing of a `(condition)` up through the expression. CARBON_PARSER_STATE(ParenCondition) // Finishes the processing of a `(condition)` after the expression. CARBON_PARSER_STATE(ParenConditionFinish) +// Handles `break` processing at the `;`. +CARBON_PARSER_STATE(StatementBreakFinish) + +// Handles `continue` processing at the `;`. +CARBON_PARSER_STATE(StatementContinueFinish) + // Handles `if` processing between the condition and start of the first code // block. CARBON_PARSER_STATE(StatementIfConditionFinish) @@ -73,6 +89,15 @@ CARBON_PARSER_STATE(StatementIfThenBlockFinish) // Handles `if` processing after a provided `else` code block. CARBON_PARSER_STATE(StatementIfElseBlockFinish) +// Handles `return` processing at the `;`. +CARBON_PARSER_STATE(StatementReturnFinish) + +// Handles `while` processing between the condition and start of the code block. +CARBON_PARSER_STATE(StatementWhileConditionFinish) + +// Handles `while` processing after the end of the code block. +CARBON_PARSER_STATE(StatementWhileBlockFinish) + // Handles pattern parsing for a function parameter, enqueuing type expression // processing. Proceeds to the matching Finish state when done. CARBON_PARSER_STATE(PatternForFunctionParameter) @@ -82,6 +107,6 @@ CARBON_PARSER_STATE(PatternForFunctionParameter) CARBON_PARSER_STATE(PatternForFunctionParameterFinish) // Handles processing of statements within a scope. -CARBON_PARSER_STATE(StatementScope) +CARBON_PARSER_STATE(StatementScopeLoop) #undef CARBON_PARSER_STATE diff --git a/toolchain/parser/testdata/operators/fixity_in_call.carbon b/toolchain/parser/testdata/operators/fixity_in_call.carbon new file mode 100644 index 000000000000..9f24c3098ea2 --- /dev/null +++ b/toolchain/parser/testdata/operators/fixity_in_call.carbon @@ -0,0 +1,29 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// AUTOUPDATE +// RUN: %{carbon-run-parser} +// CHECK:STDOUT: [ +// CHECK:STDOUT: {kind: 'FunctionIntroducer', text: 'fn'}, +// CHECK:STDOUT: {kind: 'DeclaredName', text: 'F'}, +// CHECK:STDOUT: {kind: 'ParameterListEnd', text: ')'}, +// CHECK:STDOUT: {kind: 'ParameterList', text: '(', subtree_size: 2}, +// CHECK:STDOUT: {kind: 'FunctionDefinitionStart', text: '{', subtree_size: 5}, +// CHECK:STDOUT: {kind: 'NameReference', text: 'G'}, +// CHECK:STDOUT: {kind: 'Literal', text: 'i32'}, +// CHECK:STDOUT: {kind: 'PostfixOperator', text: '*', subtree_size: 2}, +// CHECK:STDOUT: {kind: 'CallExpressionComma', text: ','}, +// CHECK:STDOUT: {kind: 'NameReference', text: 'n'}, +// CHECK:STDOUT: {kind: 'NameReference', text: 'n'}, +// CHECK:STDOUT: {kind: 'InfixOperator', text: '*', subtree_size: 3}, +// CHECK:STDOUT: {kind: 'CallExpressionEnd', text: ')'}, +// CHECK:STDOUT: {kind: 'CallExpression', text: '(', subtree_size: 9}, +// CHECK:STDOUT: {kind: 'ExpressionStatement', text: ';', subtree_size: 10}, +// CHECK:STDOUT: {kind: 'FunctionDefinition', text: '}', subtree_size: 16}, +// CHECK:STDOUT: {kind: 'FileEnd', text: ''}, +// CHECK:STDOUT: ] + +fn F() { + G(i32*, n * n); +} diff --git a/toolchain/parser/testdata/operators/fixity_in_params.carbon b/toolchain/parser/testdata/operators/fixity_in_params.carbon new file mode 100644 index 000000000000..92741d639335 --- /dev/null +++ b/toolchain/parser/testdata/operators/fixity_in_params.carbon @@ -0,0 +1,26 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// AUTOUPDATE +// RUN: %{carbon-run-parser} +// CHECK:STDOUT: [ +// CHECK:STDOUT: {kind: 'FunctionIntroducer', text: 'fn'}, +// CHECK:STDOUT: {kind: 'DeclaredName', text: 'F'}, +// CHECK:STDOUT: {kind: 'DeclaredName', text: 'p'}, +// CHECK:STDOUT: {kind: 'Literal', text: 'i32'}, +// CHECK:STDOUT: {kind: 'PostfixOperator', text: '*', subtree_size: 2}, +// CHECK:STDOUT: {kind: 'PatternBinding', text: ':', subtree_size: 4}, +// CHECK:STDOUT: {kind: 'ParameterListComma', text: ','}, +// CHECK:STDOUT: {kind: 'DeclaredName', text: 'n'}, +// CHECK:STDOUT: {kind: 'Literal', text: 'i32'}, +// CHECK:STDOUT: {kind: 'PatternBinding', text: ':', subtree_size: 3}, +// CHECK:STDOUT: {kind: 'ParameterListEnd', text: ')'}, +// CHECK:STDOUT: {kind: 'ParameterList', text: '(', subtree_size: 10}, +// CHECK:STDOUT: {kind: 'FunctionDefinitionStart', text: '{', subtree_size: 13}, +// CHECK:STDOUT: {kind: 'FunctionDefinition', text: '}', subtree_size: 14}, +// CHECK:STDOUT: {kind: 'FileEnd', text: ''}, +// CHECK:STDOUT: ] + +fn F(p: i32*, n: i32) { +} diff --git a/toolchain/parser/testdata/operators/fixity_in_var.carbon b/toolchain/parser/testdata/operators/fixity_in_var.carbon new file mode 100644 index 000000000000..358631b7d20a --- /dev/null +++ b/toolchain/parser/testdata/operators/fixity_in_var.carbon @@ -0,0 +1,36 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// AUTOUPDATE +// RUN: %{carbon-run-parser} +// CHECK:STDOUT: [ +// CHECK:STDOUT: {kind: 'FunctionIntroducer', text: 'fn'}, +// CHECK:STDOUT: {kind: 'DeclaredName', text: 'F'}, +// CHECK:STDOUT: {kind: 'ParameterListEnd', text: ')'}, +// CHECK:STDOUT: {kind: 'ParameterList', text: '(', subtree_size: 2}, +// CHECK:STDOUT: {kind: 'FunctionDefinitionStart', text: '{', subtree_size: 5}, +// CHECK:STDOUT: {kind: 'DeclaredName', text: 'q'}, +// CHECK:STDOUT: {kind: 'Literal', text: 'i32'}, +// CHECK:STDOUT: {kind: 'PostfixOperator', text: '*', subtree_size: 2}, +// CHECK:STDOUT: {kind: 'PatternBinding', text: ':', subtree_size: 4}, +// CHECK:STDOUT: {kind: 'NameReference', text: 'p'}, +// CHECK:STDOUT: {kind: 'VariableInitializer', text: '=', subtree_size: 2}, +// CHECK:STDOUT: {kind: 'DeclarationEnd', text: ';'}, +// CHECK:STDOUT: {kind: 'VariableDeclaration', text: 'var', subtree_size: 8}, +// CHECK:STDOUT: {kind: 'DeclaredName', text: 't'}, +// CHECK:STDOUT: {kind: 'NameReference', text: 'Type'}, +// CHECK:STDOUT: {kind: 'PatternBinding', text: ':', subtree_size: 3}, +// CHECK:STDOUT: {kind: 'Literal', text: 'i32'}, +// CHECK:STDOUT: {kind: 'PostfixOperator', text: '*', subtree_size: 2}, +// CHECK:STDOUT: {kind: 'VariableInitializer', text: '=', subtree_size: 3}, +// CHECK:STDOUT: {kind: 'DeclarationEnd', text: ';'}, +// CHECK:STDOUT: {kind: 'VariableDeclaration', text: 'var', subtree_size: 8}, +// CHECK:STDOUT: {kind: 'FunctionDefinition', text: '}', subtree_size: 22}, +// CHECK:STDOUT: {kind: 'FileEnd', text: ''}, +// CHECK:STDOUT: ] + +fn F() { + var q: i32* = p; + var t: Type = i32*; +} diff --git a/toolchain/parser/testdata/operators/fixity.carbon b/toolchain/parser/testdata/operators/fixity_with_assign.carbon similarity index 50% rename from toolchain/parser/testdata/operators/fixity.carbon rename to toolchain/parser/testdata/operators/fixity_with_assign.carbon index 7e227c900681..f1cd4eb17801 100644 --- a/toolchain/parser/testdata/operators/fixity.carbon +++ b/toolchain/parser/testdata/operators/fixity_with_assign.carbon @@ -7,33 +7,9 @@ // CHECK:STDOUT: [ // CHECK:STDOUT: {kind: 'FunctionIntroducer', text: 'fn'}, // CHECK:STDOUT: {kind: 'DeclaredName', text: 'F'}, -// CHECK:STDOUT: {kind: 'DeclaredName', text: 'p'}, -// CHECK:STDOUT: {kind: 'Literal', text: 'i32'}, -// CHECK:STDOUT: {kind: 'PostfixOperator', text: '*', subtree_size: 2}, -// CHECK:STDOUT: {kind: 'PatternBinding', text: ':', subtree_size: 4}, -// CHECK:STDOUT: {kind: 'ParameterListComma', text: ','}, -// CHECK:STDOUT: {kind: 'DeclaredName', text: 'n'}, -// CHECK:STDOUT: {kind: 'Literal', text: 'i32'}, -// CHECK:STDOUT: {kind: 'PatternBinding', text: ':', subtree_size: 3}, // CHECK:STDOUT: {kind: 'ParameterListEnd', text: ')'}, -// CHECK:STDOUT: {kind: 'ParameterList', text: '(', subtree_size: 10}, -// CHECK:STDOUT: {kind: 'FunctionDefinitionStart', text: '{', subtree_size: 13}, -// CHECK:STDOUT: {kind: 'DeclaredName', text: 'q'}, -// CHECK:STDOUT: {kind: 'Literal', text: 'i32'}, -// CHECK:STDOUT: {kind: 'PostfixOperator', text: '*', subtree_size: 2}, -// CHECK:STDOUT: {kind: 'PatternBinding', text: ':', subtree_size: 4}, -// CHECK:STDOUT: {kind: 'NameReference', text: 'p'}, -// CHECK:STDOUT: {kind: 'VariableInitializer', text: '=', subtree_size: 2}, -// CHECK:STDOUT: {kind: 'DeclarationEnd', text: ';'}, -// CHECK:STDOUT: {kind: 'VariableDeclaration', text: 'var', subtree_size: 8}, -// CHECK:STDOUT: {kind: 'DeclaredName', text: 't'}, -// CHECK:STDOUT: {kind: 'NameReference', text: 'Type'}, -// CHECK:STDOUT: {kind: 'PatternBinding', text: ':', subtree_size: 3}, -// CHECK:STDOUT: {kind: 'Literal', text: 'i32'}, -// CHECK:STDOUT: {kind: 'PostfixOperator', text: '*', subtree_size: 2}, -// CHECK:STDOUT: {kind: 'VariableInitializer', text: '=', subtree_size: 3}, -// CHECK:STDOUT: {kind: 'DeclarationEnd', text: ';'}, -// CHECK:STDOUT: {kind: 'VariableDeclaration', text: 'var', subtree_size: 8}, +// CHECK:STDOUT: {kind: 'ParameterList', text: '(', subtree_size: 2}, +// CHECK:STDOUT: {kind: 'FunctionDefinitionStart', text: '{', subtree_size: 5}, // CHECK:STDOUT: {kind: 'NameReference', text: 't'}, // CHECK:STDOUT: {kind: 'NameReference', text: 't'}, // CHECK:STDOUT: {kind: 'PostfixOperator', text: '*', subtree_size: 2}, @@ -59,26 +35,13 @@ // CHECK:STDOUT: {kind: 'InfixOperator', text: '*', subtree_size: 3}, // CHECK:STDOUT: {kind: 'InfixOperator', text: '=', subtree_size: 5}, // CHECK:STDOUT: {kind: 'ExpressionStatement', text: ';', subtree_size: 6}, -// CHECK:STDOUT: {kind: 'NameReference', text: 'G'}, -// CHECK:STDOUT: {kind: 'Literal', text: 'i32'}, -// CHECK:STDOUT: {kind: 'PostfixOperator', text: '*', subtree_size: 2}, -// CHECK:STDOUT: {kind: 'CallExpressionComma', text: ','}, -// CHECK:STDOUT: {kind: 'NameReference', text: 'n'}, -// CHECK:STDOUT: {kind: 'NameReference', text: 'n'}, -// CHECK:STDOUT: {kind: 'InfixOperator', text: '*', subtree_size: 3}, -// CHECK:STDOUT: {kind: 'CallExpressionEnd', text: ')'}, -// CHECK:STDOUT: {kind: 'CallExpression', text: '(', subtree_size: 9}, -// CHECK:STDOUT: {kind: 'ExpressionStatement', text: ';', subtree_size: 10}, -// CHECK:STDOUT: {kind: 'FunctionDefinition', text: '}', subtree_size: 65}, +// CHECK:STDOUT: {kind: 'FunctionDefinition', text: '}', subtree_size: 31}, // CHECK:STDOUT: {kind: 'FileEnd', text: ''}, // CHECK:STDOUT: ] -fn F(p: i32*, n: i32) { - var q: i32* = p; - var t: Type = i32*; +fn F() { t = t**; n = n * n; n = n * *p; n = n*n; - G(i32*, n * n); }