diff --git a/toolchain/parse/context.cpp b/toolchain/parse/context.cpp index 434ebafcea49..20124a3867c5 100644 --- a/toolchain/parse/context.cpp +++ b/toolchain/parse/context.cpp @@ -39,87 +39,57 @@ Context::Context(Tree* tree, Lex::TokenizedBuffer* tokens, tokens_->GetKind(*end_)); } -auto Context::ReplacePlaceholderNode(int32_t position, NodeKind kind, - Lex::TokenIndex token, bool has_error) - -> void { - CARBON_CHECK( - kind != NodeKind::InvalidParse && kind != NodeKind::InvalidParseSubtree, - "{0} shouldn't occur in Placeholder use-cases", kind); - CARBON_CHECK(position >= 0 && position < tree_->size(), - "position: {0} size: {1}", position, tree_->size()); - auto* node_impl = &tree_->node_impls_[position]; - CARBON_CHECK(node_impl->kind() == NodeKind::Placeholder); - *node_impl = Tree::NodeImpl(kind, has_error, token); +auto Context::VlogAddNode() const -> void { + CARBON_VLOG("Add #node{0}: {1}", + llvm::format_hex_no_prefix(tree_->node_impls_.size() - 1, 0, + /*Upper=*/true), + tree_->node_impls_.back()); } -auto Context::ConsumeAndAddOpenCurlyBrace(Lex::TokenIndex default_token, - NodeKind start_kind) - -> std::optional { - if (auto open_curly = ConsumeIf(Lex::TokenKind::OpenCurlyBrace)) { - AddLeafNode(start_kind, *open_curly, /*has_error=*/false); - return open_curly; - } else { - CARBON_DIAGNOSTIC(ExpectedCurlyBraceAfter, Error, - "expected `{` after `{0}`", Lex::TokenKind); - emitter_.Emit(*position_, ExpectedCurlyBraceAfter, - tokens().GetKind(default_token)); - AddLeafNode(start_kind, default_token, /*has_error=*/true); - return std::nullopt; - } +auto Context::VlogPushState() const -> void { + CARBON_VLOG("Push {0}: {1}\n", state_stack_.size() - 1, state_stack_.back()); } -auto Context::ConsumeAndAddOpenParen(Lex::TokenIndex default_token, - NodeKind start_kind) - -> std::optional { - if (auto open_paren = ConsumeIf(Lex::TokenKind::OpenParen)) { - AddLeafNode(start_kind, *open_paren, /*has_error=*/false); - return open_paren; - } else { - CARBON_DIAGNOSTIC(ExpectedParenAfter, Error, "expected `(` after `{0}`", - Lex::TokenKind); - emitter_.Emit(*position_, ExpectedParenAfter, - tokens().GetKind(default_token)); - AddLeafNode(start_kind, default_token, /*has_error=*/true); - return std::nullopt; - } +auto Context::VlogPopState(llvm::StringLiteral label) const -> void { + CARBON_VLOG("{0} {1}: {2}\n", label, state_stack_.size() - 1, + state_stack_.back()); } -auto Context::ConsumeAndAddCloseSymbol(State state, NodeKind close_kind) +auto Context::DiagnoseExpectedOpenCurlyBrace(Lex::TokenIndex default_token, + NodeKind start_kind) -> void { + CARBON_DIAGNOSTIC(ExpectedCurlyBraceAfter, Error, "expected `{` after `{0}`", + Lex::TokenKind); + emitter_.Emit(*position_, ExpectedCurlyBraceAfter, + tokens().GetKind(default_token)); + AddLeafNode(start_kind, default_token, /*has_error=*/true); +} + +auto Context::DiagnoseExpectedOpenParen(Lex::TokenIndex default_token, + NodeKind start_kind) -> void { + CARBON_DIAGNOSTIC(ExpectedParenAfter, Error, "expected `(` after `{0}`", + Lex::TokenKind); + emitter_.Emit(*position_, ExpectedParenAfter, + tokens().GetKind(default_token)); + AddLeafNode(start_kind, default_token, /*has_error=*/true); +} + +auto Context::DiagnoseAndSkipToCloseSymbol(State state, NodeKind close_kind) -> void { Lex::TokenKind open_token_kind = tokens().GetKind(state.token); - if (!open_token_kind.is_opening_symbol()) { AddNode(close_kind, state.token, /*has_error=*/true); - } else if (auto close_token = ConsumeIf(open_token_kind.closing_symbol())) { - AddNode(close_kind, *close_token, state.has_error); - } else { - // TODO: Include the location of the matching opening delimiter in the - // diagnostic. - CARBON_DIAGNOSTIC(ExpectedCloseSymbol, Error, - "unexpected tokens before `{0}`", Lex::TokenKind); - emitter_.Emit(*position_, ExpectedCloseSymbol, - open_token_kind.closing_symbol()); - - SkipTo(tokens().GetMatchedClosingToken(state.token)); - AddNode(close_kind, Consume(), /*has_error=*/true); - } -} - -auto Context::ConsumeAndAddLeafNodeIf(Lex::TokenKind token_kind, - NodeKind node_kind) -> bool { - auto token = ConsumeIf(token_kind); - if (!token) { - return false; + return; } - AddLeafNode(node_kind, *token); - return true; -} + // TODO: Include the location of the matching opening delimiter in the + // diagnostic. + CARBON_DIAGNOSTIC(ExpectedCloseSymbol, Error, + "unexpected tokens before `{0}`", Lex::TokenKind); + emitter_.Emit(*position_, ExpectedCloseSymbol, + open_token_kind.closing_symbol()); -auto Context::ConsumeChecked(Lex::TokenKind kind) -> Lex::TokenIndex { - CARBON_CHECK(PositionIs(kind), "Required {0}, found {1}", kind, - PositionKind()); - return Consume(); + SkipTo(tokens().GetMatchedClosingToken(state.token)); + AddNode(close_kind, Consume(), /*has_error=*/true); } auto Context::FindNextOf(std::initializer_list desired_kinds) @@ -207,107 +177,20 @@ auto Context::SkipTo(Lex::TokenIndex t) -> void { CARBON_CHECK(position_ != end_, "Skipped past EOF."); } -// Determines whether the given token is considered to be the start of an -// operand according to the rules for infix operator parsing. -static auto IsAssumedStartOfOperand(Lex::TokenKind kind) -> bool { - return kind.IsOneOf({Lex::TokenKind::OpenParen, Lex::TokenKind::Identifier, - Lex::TokenKind::IntLiteral, Lex::TokenKind::RealLiteral, - Lex::TokenKind::StringLiteral}); -} - -// Determines whether the given token is considered to be the end of an -// operand according to the rules for infix operator parsing. -static auto IsAssumedEndOfOperand(Lex::TokenKind kind) -> bool { - return kind.IsOneOf( - {Lex::TokenKind::CloseParen, Lex::TokenKind::CloseCurlyBrace, - Lex::TokenKind::CloseSquareBracket, Lex::TokenKind::Identifier, - Lex::TokenKind::IntLiteral, Lex::TokenKind::RealLiteral, - Lex::TokenKind::StringLiteral}); -} - -// Determines whether the given token could possibly be the start of an -// operand. This is conservatively correct, and will never incorrectly return -// `false`, but can incorrectly return `true`. -static auto IsPossibleStartOfOperand(Lex::TokenKind kind) -> bool { - return !kind.IsOneOf( - {Lex::TokenKind::CloseParen, Lex::TokenKind::CloseCurlyBrace, - Lex::TokenKind::CloseSquareBracket, Lex::TokenKind::Comma, - Lex::TokenKind::Semi, Lex::TokenKind::Colon}); -} - -auto Context::IsLexicallyValidInfixOperator() -> bool { - CARBON_CHECK(position_ != end_, "Expected an operator token."); - - bool leading_space = tokens().HasLeadingWhitespace(*position_); - bool trailing_space = tokens().HasTrailingWhitespace(*position_); - - // If there's whitespace on both sides, it's an infix operator. - if (leading_space && trailing_space) { - return true; - } - - // If there's whitespace on exactly one side, it's not an infix operator. - if (leading_space || trailing_space) { - return false; - } - - // Otherwise, for an infix operator, the preceding token must be any close - // bracket, identifier, or literal and the next token must be an open paren, - // identifier, or literal. - if (position_ == tokens().tokens().begin() || - !IsAssumedEndOfOperand(tokens().GetKind(*(position_ - 1))) || - !IsAssumedStartOfOperand(tokens().GetKind(*(position_ + 1)))) { - return false; - } - - return true; -} - -auto Context::IsTrailingOperatorInfix() -> bool { - if (position_ == end_) { - return false; - } - - // An operator that follows the infix operator rules is parsed as - // infix, unless the next token means that it can't possibly be. - if (IsLexicallyValidInfixOperator() && - IsPossibleStartOfOperand(tokens().GetKind(*(position_ + 1)))) { - return true; - } - - // A trailing operator with leading whitespace that's not valid as infix is - // not valid at all. If the next token looks like the start of an operand, - // then parse as infix, otherwise as postfix. Either way we'll produce a - // diagnostic later on. - if (tokens().HasLeadingWhitespace(*position_) && - IsAssumedStartOfOperand(tokens().GetKind(*(position_ + 1)))) { - return true; - } - - return false; -} - -auto Context::DiagnoseOperatorFixity(OperatorFixity fixity) -> void { - if (!PositionKind().is_symbol()) { - // Whitespace-based fixity rules only apply to symbolic operators. - return; - } - +auto Context::DiagnoseInvalidOperatorFixity(OperatorFixity fixity) -> void { if (fixity == OperatorFixity::Infix) { // Infix operators must satisfy the infix operator rules. - if (!IsLexicallyValidInfixOperator()) { - CARBON_DIAGNOSTIC(BinaryOperatorRequiresWhitespace, Error, - "whitespace missing {0:=-1:before|=0:around|=1:after} " - "binary operator", - Diagnostics::IntAsSelect); - Diagnostics::IntAsSelect pos(0); - if (tokens().HasLeadingWhitespace(*position_)) { - pos.value = 1; - } else if (tokens().HasTrailingWhitespace(*position_)) { - pos.value = -1; - } - emitter_.Emit(*position_, BinaryOperatorRequiresWhitespace, pos); + CARBON_DIAGNOSTIC(BinaryOperatorRequiresWhitespace, Error, + "whitespace missing {0:=-1:before|=0:around|=1:after} " + "binary operator", + Diagnostics::IntAsSelect); + Diagnostics::IntAsSelect pos(0); + if (tokens().HasLeadingWhitespace(*position_)) { + pos.value = 1; + } else if (tokens().HasTrailingWhitespace(*position_)) { + pos.value = -1; } + emitter_.Emit(*position_, BinaryOperatorRequiresWhitespace, pos); } else { bool prefix = fixity == OperatorFixity::Prefix; @@ -320,7 +203,7 @@ auto Context::DiagnoseOperatorFixity(OperatorFixity fixity) -> void { "whitespace is not allowed {0:after|before} this unary operator", Diagnostics::BoolAsSelect); emitter_.Emit(*position_, UnaryOperatorHasWhitespace, prefix); - } else if (IsLexicallyValidInfixOperator()) { + } else { // Pre/postfix operators must not satisfy the infix operator rules. CARBON_DIAGNOSTIC( UnaryOperatorRequiresWhitespace, Error, @@ -331,60 +214,41 @@ auto Context::DiagnoseOperatorFixity(OperatorFixity fixity) -> void { } } -auto Context::ConsumeListToken(NodeKind comma_kind, Lex::TokenKind close_kind, - bool already_has_error) -> ListTokenKind { - if (!PositionIs(Lex::TokenKind::Comma) && !PositionIs(close_kind)) { - // Don't error a second time on the same element. - if (!already_has_error) { - CARBON_DIAGNOSTIC(UnexpectedTokenAfterListElement, Error, - "expected `,` or `{0}`", Lex::TokenKind); - emitter_.Emit(*position_, UnexpectedTokenAfterListElement, close_kind); - ReturnErrorOnState(); - } - - // Recover from the invalid token. - auto end_of_element = FindNextOf({Lex::TokenKind::Comma, close_kind}); - // The lexer guarantees that parentheses are balanced. - CARBON_CHECK(end_of_element, "missing matching `{0}` for `{1}`", - close_kind.opening_symbol(), close_kind); - - SkipTo(*end_of_element); +auto Context::RecoverFromInvalidListToken(Lex::TokenKind close_kind, + bool already_has_error) -> void { + // Don't error a second time on the same element. + if (!already_has_error) { + CARBON_DIAGNOSTIC(UnexpectedTokenAfterListElement, Error, + "expected `,` or `{0}`", Lex::TokenKind); + emitter_.Emit(*position_, UnexpectedTokenAfterListElement, close_kind); + ReturnErrorOnState(); } - if (PositionIs(close_kind)) { - return ListTokenKind::Close; - } else { - AddLeafNode(comma_kind, Consume(), - /*has_error=*/comma_kind == NodeKind::InvalidParse); - return PositionIs(close_kind) ? ListTokenKind::CommaClose - : ListTokenKind::Comma; - } + // Recover from the invalid token. + auto end_of_element = FindNextOf({Lex::TokenKind::Comma, close_kind}); + // The lexer guarantees that parentheses are balanced. + CARBON_CHECK(end_of_element, "missing matching `{0}` for `{1}`", + close_kind.opening_symbol(), close_kind); + + SkipTo(*end_of_element); } -auto Context::AddNodeExpectingDeclSemi(State state, NodeKind node_kind, - Lex::TokenKind decl_kind, - bool is_def_allowed) -> void { +auto Context::RecoverFromMissingDeclSemi(State state, NodeKind node_kind, + Lex::TokenKind decl_kind, + bool is_def_allowed) -> void { // TODO: This could better handle things like: // base: { } // var n: i32; // ^ Ends up at `n`, instead of `var`. - if (state.has_error) { - RecoverFromDeclError(state, node_kind, - /*skip_past_likely_end=*/true); - return; - } - - if (auto semi = ConsumeIf(Lex::TokenKind::Semi)) { - AddNode(node_kind, *semi, /*has_error=*/false); - } else { + if (!state.has_error) { if (is_def_allowed) { DiagnoseExpectedDeclSemiOrDefinition(decl_kind); } else { DiagnoseExpectedDeclSemi(decl_kind); } - RecoverFromDeclError(state, node_kind, - /*skip_past_likely_end=*/true); } + RecoverFromDeclError(state, node_kind, + /*skip_past_likely_end=*/true); } auto Context::RecoverFromDeclError(State state, NodeKind node_kind, diff --git a/toolchain/parse/context.h b/toolchain/parse/context.h index cc2e4d2785f7..1548ed46bbe9 100644 --- a/toolchain/parse/context.h +++ b/toolchain/parse/context.h @@ -9,6 +9,7 @@ #include "common/check.h" #include "common/vlog.h" +#include "llvm/ADT/StringRef.h" #include "toolchain/lex/token_kind.h" #include "toolchain/lex/tokenized_buffer.h" #include "toolchain/parse/node_ids.h" @@ -173,10 +174,9 @@ class Context { kind != NodeKind::InvalidParseSubtree), "{0} nodes must always have an error", kind); tree_->node_impls_.push_back(Tree::NodeImpl(kind, has_error, token)); - CARBON_VLOG("Add #node{0}: {1}", - llvm::format_hex_no_prefix(tree_->node_impls_.size() - 1, 0, - /*Upper=*/true), - tree_->node_impls_.back()); + if (vlog_stream_ != nullptr) { + VlogAddNode(); + } } // Adds a node and returns its typed NodeId. @@ -209,7 +209,16 @@ class Context { // ``` auto ReplacePlaceholderNode(int32_t position, NodeKind kind, Lex::TokenIndex token, bool has_error = false) - -> void; + -> void { + CARBON_CHECK( + kind != NodeKind::InvalidParse && kind != NodeKind::InvalidParseSubtree, + "{0} shouldn't occur in Placeholder use-cases", kind); + CARBON_CHECK(position >= 0 && position < tree_->size(), + "position: {0} size: {1}", position, tree_->size()); + auto* node_impl = &tree_->node_impls_[position]; + CARBON_CHECK(node_impl->kind() == NodeKind::Placeholder); + *node_impl = Tree::NodeImpl(kind, has_error, token); + } // Returns the current position and moves past it. auto Consume() -> Lex::TokenIndex { return *(position_++); } @@ -223,30 +232,66 @@ class Context { // was found. auto ConsumeAndAddOpenCurlyBrace(Lex::TokenIndex default_token, NodeKind start_kind) - -> std::optional; + -> std::optional { + if (!PositionIs(Lex::TokenKind::OpenCurlyBrace)) { + DiagnoseExpectedOpenCurlyBrace(default_token, start_kind); + return std::nullopt; + } + auto open_curly = Consume(); + AddLeafNode(start_kind, open_curly, /*has_error=*/false); + return open_curly; + } // Parses an open paren token, possibly diagnosing if necessary. Creates a // leaf parse node of the specified start kind. The default_token is used when // there's no open paren. Returns the open paren token if it was found. auto ConsumeAndAddOpenParen(Lex::TokenIndex default_token, NodeKind start_kind) - -> std::optional; + -> std::optional { + if (!PositionIs(Lex::TokenKind::OpenParen)) { + DiagnoseExpectedOpenParen(default_token, start_kind); + return std::nullopt; + } + auto open_paren = Consume(); + AddLeafNode(start_kind, open_paren, /*has_error=*/false); + return open_paren; + } // Parses a closing symbol corresponding to the opening symbol `state.token`, // possibly skipping forward and diagnosing if necessary. Creates a parse node // of the specified close kind. If `state.token` is not an opening symbol, // no input will be consumed, and no diagnostic will be emitted, but the parse // node will still be marked as having an error. - auto ConsumeAndAddCloseSymbol(State state, NodeKind close_kind) -> void; + auto ConsumeAndAddCloseSymbol(State state, NodeKind close_kind) -> void { + Lex::TokenKind open_token_kind = tokens().GetKind(state.token); + if (open_token_kind.is_opening_symbol() && + PositionIs(open_token_kind.closing_symbol())) { + AddNode(close_kind, Consume(), state.has_error); + } else { + DiagnoseAndSkipToCloseSymbol(state, close_kind); + } + } // Composes `ConsumeIf` and `AddLeafNode`, returning false when ConsumeIf // fails. auto ConsumeAndAddLeafNodeIf(Lex::TokenKind token_kind, NodeKind node_kind) - -> bool; + -> bool { + auto token = ConsumeIf(token_kind); + if (!token) { + return false; + } + + AddLeafNode(node_kind, *token); + return true; + } // Returns the current position and moves past it. Requires the token is the // expected kind. - auto ConsumeChecked(Lex::TokenKind kind) -> Lex::TokenIndex; + auto ConsumeChecked(Lex::TokenKind kind) -> Lex::TokenIndex { + CARBON_CHECK(PositionIs(kind), "Required {0}, found {1}", kind, + PositionKind()); + return Consume(); + } // If the current position's token matches this `Kind`, returns it and // advances to the next position. Otherwise returns an empty optional. @@ -289,16 +334,88 @@ class Context { // Returns true if the current token satisfies the lexical validity rules // for an infix operator. - auto IsLexicallyValidInfixOperator() -> bool; + auto IsLexicallyValidInfixOperator() -> bool { + CARBON_CHECK(position_ != end_, "Expected an operator token."); + + bool leading_space = tokens().HasLeadingWhitespace(*position_); + bool trailing_space = tokens().HasTrailingWhitespace(*position_); + + // If there's whitespace on both sides, it's an infix operator. + if (leading_space && trailing_space) { + return true; + } + + // If there's whitespace on exactly one side, it's not an infix operator. + if (leading_space || trailing_space) { + return false; + } + + // Otherwise, for an infix operator, the preceding token must be any close + // bracket, identifier, or literal and the next token must be an open paren, + // identifier, or literal. + if (position_ == tokens().tokens().begin() || + !IsAssumedEndOfOperand(tokens().GetKind(*(position_ - 1))) || + !IsAssumedStartOfOperand(tokens().GetKind(*(position_ + 1)))) { + return false; + } + + return true; + } // Determines whether the current trailing operator should be treated as // infix. - auto IsTrailingOperatorInfix() -> bool; + auto IsTrailingOperatorInfix() -> bool { + if (position_ == end_) { + return false; + } + + // An operator that follows the infix operator rules is parsed as + // infix, unless the next token means that it can't possibly be. + if (IsLexicallyValidInfixOperator() && + IsPossibleStartOfOperand(tokens().GetKind(*(position_ + 1)))) { + return true; + } + + // A trailing operator with leading whitespace that's not valid as infix is + // not valid at all. If the next token looks like the start of an operand, + // then parse as infix, otherwise as postfix. Either way we'll produce a + // diagnostic later on. + if (tokens().HasLeadingWhitespace(*position_) && + IsAssumedStartOfOperand(tokens().GetKind(*(position_ + 1)))) { + return true; + } + + return false; + } // Diagnoses whether the current token is not written properly for the given // fixity. For example, because mandatory whitespace is missing. Regardless of // whether there's an error, it's expected that parsing continues. - auto DiagnoseOperatorFixity(OperatorFixity fixity) -> void; + auto DiagnoseOperatorFixity(OperatorFixity fixity) -> void { + if (!PositionKind().is_symbol()) { + // Whitespace-based fixity rules only apply to symbolic operators. + return; + } + + if (fixity == OperatorFixity::Infix) { + // Infix operators must satisfy the infix operator rules. + if (IsLexicallyValidInfixOperator()) { + return; + } + } else { + bool prefix = fixity == OperatorFixity::Prefix; + + // Whitespace is not permitted between a symbolic pre/postfix operator + // and its operand, and pre/postfix operators must not satisfy the infix + // operator rules. + if (!(prefix ? tokens().HasTrailingWhitespace(*position_) + : tokens().HasLeadingWhitespace(*position_)) && + !IsLexicallyValidInfixOperator()) { + return; + } + } + DiagnoseInvalidOperatorFixity(fixity); + } // If the current position is a `,`, consumes it, adds the provided token, and // returns `Comma`. Returns `Close` if the current position is close_token @@ -307,7 +424,20 @@ class Context { // position, and may emit errors. Pass already_has_error in order to suppress // duplicate errors. auto ConsumeListToken(NodeKind comma_kind, Lex::TokenKind close_kind, - bool already_has_error) -> ListTokenKind; + bool already_has_error) -> ListTokenKind { + if (!PositionIs(Lex::TokenKind::Comma) && !PositionIs(close_kind)) { + RecoverFromInvalidListToken(close_kind, already_has_error); + } + + if (PositionIs(close_kind)) { + return ListTokenKind::Close; + } else { + AddLeafNode(comma_kind, Consume(), + /*has_error=*/comma_kind == NodeKind::InvalidParse); + return PositionIs(close_kind) ? ListTokenKind::CommaClose + : ListTokenKind::Comma; + } + } // Gets the kind of the next token to be consumed. If `lookahead` is // provided, it specifies which token to inspect. @@ -325,15 +455,17 @@ class Context { // Pops the state and keeps the value for inspection. auto PopState() -> State { - auto back = state_stack_.pop_back_val(); - CARBON_VLOG("Pop {0}: {1}\n", state_stack_.size(), back); - return back; + if (vlog_stream_ != nullptr) { + VlogPopState("Pop"); + } + return state_stack_.pop_back_val(); } // Pops the state and discards it. auto PopAndDiscardState() -> void { - CARBON_VLOG("PopAndDiscard {0}: {1}\n", state_stack_.size() - 1, - state_stack_.back()); + if (vlog_stream_ != nullptr) { + VlogPopState("PopAndDiscard"); + } state_stack_.pop_back(); } @@ -397,8 +529,10 @@ class Context { // Pushes a constructed state onto the stack. auto PushState(State state) -> void { - CARBON_VLOG("Push {0}: {1}\n", state_stack_.size(), state); state_stack_.push_back(state); + if (vlog_stream_ != nullptr) { + VlogPushState(); + } CARBON_CHECK(state_stack_.size() < (1 << 20), "Excessive stack size: likely infinite loop"); } @@ -417,7 +551,13 @@ class Context { // diagnostics. auto AddNodeExpectingDeclSemi(State state, NodeKind node_kind, Lex::TokenKind decl_kind, bool is_def_allowed) - -> void; + -> void { + if (!state.has_error && PositionIs(Lex::TokenKind::Semi)) { + AddNode(node_kind, Consume(), /*has_error=*/false); + return; + } + RecoverFromMissingDeclSemi(state, node_kind, decl_kind, is_def_allowed); + } // Emits a diagnostic for a declaration missing a semi. auto DiagnoseExpectedDeclSemi(Lex::TokenKind expected_kind) -> void; @@ -506,6 +646,71 @@ class Context { } private: + // Determines whether the given token is considered to be the start of an + // operand according to the rules for infix operator parsing. + static auto IsAssumedStartOfOperand(Lex::TokenKind kind) -> bool { + return kind.IsOneOf({Lex::TokenKind::OpenParen, Lex::TokenKind::Identifier, + Lex::TokenKind::IntLiteral, + Lex::TokenKind::RealLiteral, + Lex::TokenKind::StringLiteral}); + } + + // Determines whether the given token is considered to be the end of an + // operand according to the rules for infix operator parsing. + static auto IsAssumedEndOfOperand(Lex::TokenKind kind) -> bool { + return kind.IsOneOf( + {Lex::TokenKind::CloseParen, Lex::TokenKind::CloseCurlyBrace, + Lex::TokenKind::CloseSquareBracket, Lex::TokenKind::Identifier, + Lex::TokenKind::IntLiteral, Lex::TokenKind::RealLiteral, + Lex::TokenKind::StringLiteral}); + } + + // Determines whether the given token could possibly be the start of an + // operand. This is conservatively correct, and will never incorrectly return + // `false`, but can incorrectly return `true`. + static auto IsPossibleStartOfOperand(Lex::TokenKind kind) -> bool { + return !kind.IsOneOf( + {Lex::TokenKind::CloseParen, Lex::TokenKind::CloseCurlyBrace, + Lex::TokenKind::CloseSquareBracket, Lex::TokenKind::Comma, + Lex::TokenKind::Semi, Lex::TokenKind::Colon}); + } + + // The following are the slow paths of inline methods above, which are kept + // out of line so that the inline fast paths stay small. They handle verbose + // logging, diagnostics, and error recovery. + + // Logs the most recently added node. + [[gnu::cold]] auto VlogAddNode() const -> void; + // Logs the most recently pushed state. + [[gnu::cold]] auto VlogPushState() const -> void; + // Logs the state that is about to be popped, prefixed by `label`. + [[gnu::cold]] auto VlogPopState(llvm::StringLiteral label) const -> void; + + // Diagnoses a missing open curly brace in `ConsumeAndAddOpenCurlyBrace`. + [[gnu::cold]] auto DiagnoseExpectedOpenCurlyBrace( + Lex::TokenIndex default_token, NodeKind start_kind) -> void; + // Diagnoses a missing open paren in `ConsumeAndAddOpenParen`. + [[gnu::cold]] auto DiagnoseExpectedOpenParen(Lex::TokenIndex default_token, + NodeKind start_kind) -> void; + // Handles the error cases of `ConsumeAndAddCloseSymbol`, diagnosing and + // skipping to the matching closing symbol when `state.token` is an opening + // symbol. + [[gnu::cold]] auto DiagnoseAndSkipToCloseSymbol(State state, + NodeKind close_kind) -> void; + // Diagnoses the current operator token not being written properly for the + // given fixity in `DiagnoseOperatorFixity`. + [[gnu::cold]] auto DiagnoseInvalidOperatorFixity(OperatorFixity fixity) + -> void; + // Diagnoses and skips past an invalid token in `ConsumeListToken`, leaving + // the position at either a `,` or `close_kind`. + [[gnu::cold]] auto RecoverFromInvalidListToken(Lex::TokenKind close_kind, + bool already_has_error) + -> void; + // Handles the error cases of `AddNodeExpectingDeclSemi`. + [[gnu::cold]] auto RecoverFromMissingDeclSemi(State state, NodeKind node_kind, + Lex::TokenKind decl_kind, + bool is_def_allowed) -> void; + // Prints a single token for a stack dump. Used by PrintForStackDump. auto PrintTokenForStackDump(llvm::raw_ostream& output, Lex::TokenIndex token) const -> void;