From 6433a1bab6bbfa308d84d1c6ea919e3c64b2817a Mon Sep 17 00:00:00 2001 From: Jon Ross-Perkins Date: Fri, 11 Nov 2022 13:35:45 -0800 Subject: [PATCH] Rename Parser2 to Parser, and delete the original (#2383) --- toolchain/README.md | 23 +- toolchain/parser/BUILD | 6 +- toolchain/parser/parse_tree.cpp | 5 +- toolchain/parser/parse_tree.h | 16 +- toolchain/parser/{parser2.cpp => parser.cpp} | 229 ++- toolchain/parser/{parser2.h => parser.h} | 14 +- toolchain/parser/parser_impl.cpp | 1342 ------------------ toolchain/parser/parser_impl.h | 301 ---- 8 files changed, 128 insertions(+), 1808 deletions(-) rename toolchain/parser/{parser2.cpp => parser.cpp} (87%) rename toolchain/parser/{parser2.h => parser.h} (97%) delete mode 100644 toolchain/parser/parser_impl.cpp delete mode 100644 toolchain/parser/parser_impl.h diff --git a/toolchain/README.md b/toolchain/README.md index f97416ce9029..c3013efb334d 100644 --- a/toolchain/README.md +++ b/toolchain/README.md @@ -50,7 +50,7 @@ token has a matching closing bracket token. ## Parsing The [ParseTree](parser/parse_tree.h) is the output of parsing, but most logic is -in [ParserImpl](parser/parser_impl.h). +in [Parser](parser/parser.h). The parse tree faithfully represents the tree structure of the source program, interpreted according to the Carbon grammar. No semantics are associated with @@ -65,14 +65,6 @@ did not match the grammar, but we were still able to parse some subexpressions, as an aid for non-compiler tools such as syntax highlighters or refactoring tools. -Many functions in the parser return `llvm::Optional`. A return value of -`llvm::None` indicates that parsing has failed and an error diagnostic has -already been produced, and that the current region of the parse tree might not -meet its invariants so that the caller should create an invalid parse tree node. -Other return values indicate that parsing was either successful or that any -encountered errors have been recovered from, so the caller can create a valid -parse tree node. - The produced `ParseTree` is in postorder. For example, given the code: ```carbon @@ -108,18 +100,7 @@ grammatical construct that is the parent: this is so that a postorder traversal of the tree can see the kind of grammatical construct being built first, and handle child nodes taking that into account. -### Stack overflow - -The `ParseTree` has been prone to stack overflows. As a consequence, -`CARBON_RETURN_IF_STACK_LIMITED` is checked at the start of most functions in -order to avoid errors. This manages depth increments and, when the scope exits, -decrements. - -#### Future work - -We are interested in eventually exploring ways to adjust the parser design to be -non-recursive and remove this limitation, but it hasn't yet been a priority and -keeping the code simple seems better until the language design stabilizes. +TODO: Document flow. ## Semantics diff --git a/toolchain/parser/BUILD b/toolchain/parser/BUILD index 066651dd0c02..f8844eef4a9c 100644 --- a/toolchain/parser/BUILD +++ b/toolchain/parser/BUILD @@ -56,10 +56,8 @@ cc_library( name = "parse_tree", srcs = [ "parse_tree.cpp", - "parser_impl.cpp", - "parser_impl.h", - "parser2.cpp", - "parser2.h", + "parser.cpp", + "parser.h", ], hdrs = ["parse_tree.h"], deps = [ diff --git a/toolchain/parser/parse_tree.cpp b/toolchain/parser/parse_tree.cpp index 930e68b5d01d..83067011bf7c 100644 --- a/toolchain/parser/parse_tree.cpp +++ b/toolchain/parser/parse_tree.cpp @@ -16,8 +16,7 @@ #include "llvm/Support/raw_ostream.h" #include "toolchain/lexer/token_kind.h" #include "toolchain/parser/parse_node_kind.h" -#include "toolchain/parser/parser2.h" -#include "toolchain/parser/parser_impl.h" +#include "toolchain/parser/parser.h" namespace Carbon { @@ -28,7 +27,7 @@ auto ParseTree::Parse(TokenizedBuffer& tokens, DiagnosticConsumer& consumer) TokenDiagnosticEmitter emitter(translator, consumer); // Delegate to the parser. - return Parser2::Parse(tokens, emitter); + return Parser::Parse(tokens, emitter); } auto ParseTree::postorder() const -> llvm::iterator_range { diff --git a/toolchain/parser/parse_tree.h b/toolchain/parser/parse_tree.h index 8f5b59db1fe6..84430213eece 100644 --- a/toolchain/parser/parse_tree.h +++ b/toolchain/parser/parse_tree.h @@ -46,11 +46,6 @@ class ParseTree { class PostorderIterator; class SiblingIterator; - // The maximum stack depth allowed while recursing the parse tree. - // This is meant to approximate system stack limits, but we may need to find a - // better way to track what the system is enforcing. - static constexpr int StackDepthLimit = 200; - // Parses the token buffer into a `ParseTree`. // // This is the factory function which is used to build parse trees. @@ -155,19 +150,11 @@ class ParseTree { [[nodiscard]] auto Verify() const -> bool; private: - class Parser; - friend Parser; - friend class Parser2; + friend class Parser; // The in-memory representation of data used for a particular node in the // tree. struct NodeImpl { - explicit NodeImpl(ParseNodeKind k, TokenizedBuffer::Token t, - int subtree_size_arg) - : kind(k), token(t), subtree_size(subtree_size_arg) {} - - // TODO: Parser2 only uses this construct. Can remove the other if we - // switch. NodeImpl(ParseNodeKind kind, bool has_error, TokenizedBuffer::Token token, int subtree_size) : kind(kind), @@ -294,7 +281,6 @@ class ParseTree::Node { private: friend ParseTree; - friend Parser; friend PostorderIterator; friend SiblingIterator; diff --git a/toolchain/parser/parser2.cpp b/toolchain/parser/parser.cpp similarity index 87% rename from toolchain/parser/parser2.cpp rename to toolchain/parser/parser.cpp index be4b4c14d186..838fea98a297 100644 --- a/toolchain/parser/parser2.cpp +++ b/toolchain/parser/parser.cpp @@ -2,7 +2,7 @@ // Exceptions. See /LICENSE for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "toolchain/parser/parser2.h" +#include "toolchain/parser/parser.h" #include #include @@ -49,10 +49,9 @@ static auto operator<<(llvm::raw_ostream& out, RelativeLocation loc) return out; } -class Parser2::PrettyStackTraceParseState : public llvm::PrettyStackTraceEntry { +class Parser::PrettyStackTraceParseState : public llvm::PrettyStackTraceEntry { public: - explicit PrettyStackTraceParseState(const Parser2* parser) - : parser_(parser) {} + explicit PrettyStackTraceParseState(const Parser* parser) : parser_(parser) {} ~PrettyStackTraceParseState() override = default; auto print(llvm::raw_ostream& output) const -> void override { @@ -76,11 +75,11 @@ class Parser2::PrettyStackTraceParseState : public llvm::PrettyStackTraceEntry { << parser_->tokens_.GetKind(token).Name() << "\n"; } - const Parser2* parser_; + const Parser* parser_; }; -Parser2::Parser2(ParseTree& tree_arg, TokenizedBuffer& tokens_arg, - TokenDiagnosticEmitter& emitter) +Parser::Parser(ParseTree& tree_arg, TokenizedBuffer& tokens_arg, + TokenDiagnosticEmitter& emitter) : tree_(tree_arg), tokens_(tokens_arg), emitter_(emitter), @@ -93,8 +92,8 @@ Parser2::Parser2(ParseTree& tree_arg, TokenizedBuffer& tokens_arg, << tokens_.GetKind(*end_).Name(); } -auto Parser2::AddLeafNode(ParseNodeKind kind, TokenizedBuffer::Token token, - bool has_error) -> void { +auto Parser::AddLeafNode(ParseNodeKind kind, TokenizedBuffer::Token token, + bool has_error) -> void { tree_.node_impls_.push_back( ParseTree::NodeImpl(kind, has_error, token, /*subtree_size=*/1)); if (has_error) { @@ -102,8 +101,8 @@ auto Parser2::AddLeafNode(ParseNodeKind kind, TokenizedBuffer::Token token, } } -auto Parser2::AddNode(ParseNodeKind kind, TokenizedBuffer::Token token, - int subtree_start, bool has_error) -> void { +auto Parser::AddNode(ParseNodeKind kind, TokenizedBuffer::Token token, + int subtree_start, bool has_error) -> void { int subtree_size = tree_.size() - subtree_start + 1; tree_.node_impls_.push_back( ParseTree::NodeImpl(kind, has_error, token, subtree_size)); @@ -112,8 +111,8 @@ auto Parser2::AddNode(ParseNodeKind kind, TokenizedBuffer::Token token, } } -auto Parser2::ConsumeAndAddCloseParen(TokenizedBuffer::Token open_paren, - ParseNodeKind close_kind) -> bool { +auto Parser::ConsumeAndAddCloseParen(TokenizedBuffer::Token open_paren, + ParseNodeKind close_kind) -> bool { if (ConsumeAndAddLeafNodeIf(TokenKind::CloseParen(), close_kind)) { return true; } @@ -128,8 +127,8 @@ auto Parser2::ConsumeAndAddCloseParen(TokenizedBuffer::Token open_paren, return false; } -auto Parser2::ConsumeAndAddLeafNodeIf(TokenKind token_kind, - ParseNodeKind node_kind) -> bool { +auto Parser::ConsumeAndAddLeafNodeIf(TokenKind token_kind, + ParseNodeKind node_kind) -> bool { auto token = ConsumeIf(token_kind); if (!token) { return false; @@ -139,7 +138,7 @@ auto Parser2::ConsumeAndAddLeafNodeIf(TokenKind token_kind, return true; } -auto Parser2::ConsumeIf(TokenKind kind) +auto Parser::ConsumeIf(TokenKind kind) -> llvm::Optional { if (!PositionIs(kind)) { return llvm::None; @@ -149,7 +148,7 @@ auto Parser2::ConsumeIf(TokenKind kind) return token; } -auto Parser2::FindNextOf(std::initializer_list desired_kinds) +auto Parser::FindNextOf(std::initializer_list desired_kinds) -> llvm::Optional { auto new_position = position_; while (true) { @@ -174,7 +173,7 @@ auto Parser2::FindNextOf(std::initializer_list desired_kinds) } } -auto Parser2::SkipMatchingGroup() -> bool { +auto Parser::SkipMatchingGroup() -> bool { if (!PositionKind().IsOpeningSymbol()) { return false; } @@ -184,7 +183,7 @@ auto Parser2::SkipMatchingGroup() -> bool { return true; } -auto Parser2::SkipPastLikelyEnd(TokenizedBuffer::Token skip_root) +auto Parser::SkipPastLikelyEnd(TokenizedBuffer::Token skip_root) -> llvm::Optional { if (position_ == end_) { return llvm::None; @@ -231,14 +230,14 @@ auto Parser2::SkipPastLikelyEnd(TokenizedBuffer::Token skip_root) return llvm::None; } -auto Parser2::SkipTo(TokenizedBuffer::Token t) -> void { +auto Parser::SkipTo(TokenizedBuffer::Token t) -> void { CARBON_CHECK(t >= *position_) << "Tried to skip backwards from " << position_ << " to " << TokenizedBuffer::TokenIterator(t); position_ = TokenizedBuffer::TokenIterator(t); CARBON_CHECK(position_ != end_) << "Skipped past EOF."; } -auto Parser2::HandleCodeBlockState() -> void { +auto Parser::HandleCodeBlockState() -> void { PopAndDiscardState(); PushState(ParserState::CodeBlockFinish()); @@ -283,7 +282,7 @@ static auto IsPossibleStartOfOperand(TokenKind kind) -> bool { TokenKind::Semi(), TokenKind::Colon()}); } -auto Parser2::IsLexicallyValidInfixOperator() -> bool { +auto Parser::IsLexicallyValidInfixOperator() -> bool { CARBON_CHECK(position_ != end_) << "Expected an operator token."; bool leading_space = tokens_.HasLeadingWhitespace(*position_); @@ -311,7 +310,7 @@ auto Parser2::IsLexicallyValidInfixOperator() -> bool { return true; } -auto Parser2::IsTrailingOperatorInfix() -> bool { +auto Parser::IsTrailingOperatorInfix() -> bool { if (position_ == end_) { return false; } @@ -335,7 +334,7 @@ auto Parser2::IsTrailingOperatorInfix() -> bool { return false; } -auto Parser2::DiagnoseOperatorFixity(OperatorFixity fixity) -> void { +auto Parser::DiagnoseOperatorFixity(OperatorFixity fixity) -> void { if (fixity == OperatorFixity::Infix) { // Infix operators must satisfy the infix operator rules. if (!IsLexicallyValidInfixOperator()) { @@ -376,8 +375,8 @@ auto Parser2::DiagnoseOperatorFixity(OperatorFixity fixity) -> void { } } -auto Parser2::ConsumeListToken(ParseNodeKind comma_kind, TokenKind close_kind, - bool already_has_error) -> ListTokenKind { +auto Parser::ConsumeListToken(ParseNodeKind comma_kind, TokenKind close_kind, + bool already_has_error) -> ListTokenKind { if (!PositionIs(TokenKind::Comma()) && !PositionIs(close_kind)) { // Don't error a second time on the same element. if (!already_has_error) { @@ -406,7 +405,7 @@ auto Parser2::ConsumeListToken(ParseNodeKind comma_kind, TokenKind close_kind, } } -auto Parser2::Parse() -> void { +auto Parser::Parse() -> void { // Traces state_stack_. This runs even in opt because it's low overhead. PrettyStackTraceParseState pretty_stack(this); @@ -424,7 +423,7 @@ auto Parser2::Parse() -> void { AddLeafNode(ParseNodeKind::FileEnd(), *position_); } -auto Parser2::HandleBraceExpressionState() -> void { +auto Parser::HandleBraceExpressionState() -> void { auto state = PopState(); state.state = ParserState::BraceExpressionFinishAsUnknown(); @@ -441,10 +440,10 @@ auto Parser2::HandleBraceExpressionState() -> void { } } -auto Parser2::BraceExpressionKindToParserState(BraceExpressionKind kind, - ParserState type, - ParserState value, - ParserState unknown) +auto Parser::BraceExpressionKindToParserState(BraceExpressionKind kind, + ParserState type, + ParserState value, + ParserState unknown) -> ParserState { switch (kind) { case BraceExpressionKind::Type: { @@ -459,8 +458,8 @@ auto Parser2::BraceExpressionKindToParserState(BraceExpressionKind kind, } } -auto Parser2::HandleBraceExpressionParameterError(StateStackEntry state, - BraceExpressionKind kind) +auto Parser::HandleBraceExpressionParameterError(StateStackEntry state, + BraceExpressionKind kind) -> void { CARBON_DIAGNOSTIC(ExpectedStructLiteralField, Error, "Expected {0}{1}{2}.", llvm::StringRef, llvm::StringRef, llvm::StringRef); @@ -479,7 +478,7 @@ auto Parser2::HandleBraceExpressionParameterError(StateStackEntry state, PushState(state); } -auto Parser2::HandleBraceExpressionParameter(BraceExpressionKind kind) -> void { +auto Parser::HandleBraceExpressionParameter(BraceExpressionKind kind) -> void { auto state = PopState(); if (!PositionIs(TokenKind::Period())) { @@ -495,19 +494,19 @@ auto Parser2::HandleBraceExpressionParameter(BraceExpressionKind kind) -> void { PushState(ParserState::DesignatorAsStruct()); } -auto Parser2::HandleBraceExpressionParameterAsTypeState() -> void { +auto Parser::HandleBraceExpressionParameterAsTypeState() -> void { HandleBraceExpressionParameter(BraceExpressionKind::Type); } -auto Parser2::HandleBraceExpressionParameterAsValueState() -> void { +auto Parser::HandleBraceExpressionParameterAsValueState() -> void { HandleBraceExpressionParameter(BraceExpressionKind::Value); } -auto Parser2::HandleBraceExpressionParameterAsUnknownState() -> void { +auto Parser::HandleBraceExpressionParameterAsUnknownState() -> void { HandleBraceExpressionParameter(BraceExpressionKind::Unknown); } -auto Parser2::HandleBraceExpressionParameterAfterDesignator( +auto Parser::HandleBraceExpressionParameterAfterDesignator( BraceExpressionKind kind) -> void { auto state = PopState(); @@ -565,22 +564,22 @@ auto Parser2::HandleBraceExpressionParameterAfterDesignator( PushState(ParserState::Expression()); } -auto Parser2::HandleBraceExpressionParameterAfterDesignatorAsTypeState() +auto Parser::HandleBraceExpressionParameterAfterDesignatorAsTypeState() -> void { HandleBraceExpressionParameterAfterDesignator(BraceExpressionKind::Type); } -auto Parser2::HandleBraceExpressionParameterAfterDesignatorAsValueState() +auto Parser::HandleBraceExpressionParameterAfterDesignatorAsValueState() -> void { HandleBraceExpressionParameterAfterDesignator(BraceExpressionKind::Value); } -auto Parser2::HandleBraceExpressionParameterAfterDesignatorAsUnknownState() +auto Parser::HandleBraceExpressionParameterAfterDesignatorAsUnknownState() -> void { HandleBraceExpressionParameterAfterDesignator(BraceExpressionKind::Unknown); } -auto Parser2::HandleBraceExpressionParameterFinish(BraceExpressionKind kind) +auto Parser::HandleBraceExpressionParameterFinish(BraceExpressionKind kind) -> void { auto state = PopState(); @@ -598,19 +597,19 @@ auto Parser2::HandleBraceExpressionParameterFinish(BraceExpressionKind kind) } } -auto Parser2::HandleBraceExpressionParameterFinishAsTypeState() -> void { +auto Parser::HandleBraceExpressionParameterFinishAsTypeState() -> void { HandleBraceExpressionParameterFinish(BraceExpressionKind::Type); } -auto Parser2::HandleBraceExpressionParameterFinishAsValueState() -> void { +auto Parser::HandleBraceExpressionParameterFinishAsValueState() -> void { HandleBraceExpressionParameterFinish(BraceExpressionKind::Value); } -auto Parser2::HandleBraceExpressionParameterFinishAsUnknownState() -> void { +auto Parser::HandleBraceExpressionParameterFinishAsUnknownState() -> void { HandleBraceExpressionParameterFinish(BraceExpressionKind::Unknown); } -auto Parser2::HandleBraceExpressionFinish(BraceExpressionKind kind) -> void { +auto Parser::HandleBraceExpressionFinish(BraceExpressionKind kind) -> void { auto state = PopState(); AddLeafNode(ParseNodeKind::StructEnd(), Consume()); @@ -619,19 +618,19 @@ auto Parser2::HandleBraceExpressionFinish(BraceExpressionKind kind) -> void { state.token, state.subtree_start, state.has_error); } -auto Parser2::HandleBraceExpressionFinishAsTypeState() -> void { +auto Parser::HandleBraceExpressionFinishAsTypeState() -> void { HandleBraceExpressionFinish(BraceExpressionKind::Type); } -auto Parser2::HandleBraceExpressionFinishAsValueState() -> void { +auto Parser::HandleBraceExpressionFinishAsValueState() -> void { HandleBraceExpressionFinish(BraceExpressionKind::Value); } -auto Parser2::HandleBraceExpressionFinishAsUnknownState() -> void { +auto Parser::HandleBraceExpressionFinishAsUnknownState() -> void { HandleBraceExpressionFinish(BraceExpressionKind::Unknown); } -auto Parser2::HandleCallExpressionState() -> void { +auto Parser::HandleCallExpressionState() -> void { auto state = PopState(); // TODO: When swapping () start/end, this should AddLeafNode the open before @@ -646,7 +645,7 @@ auto Parser2::HandleCallExpressionState() -> void { } } -auto Parser2::HandleCallExpressionParameterFinishState() -> void { +auto Parser::HandleCallExpressionParameterFinishState() -> void { auto state = PopState(); if (state.has_error) { @@ -661,7 +660,7 @@ auto Parser2::HandleCallExpressionParameterFinishState() -> void { } } -auto Parser2::HandleCallExpressionFinishState() -> void { +auto Parser::HandleCallExpressionFinishState() -> void { auto state = PopState(); AddLeafNode(ParseNodeKind::CallExpressionEnd(), Consume()); @@ -669,7 +668,7 @@ auto Parser2::HandleCallExpressionFinishState() -> void { state.has_error); } -auto Parser2::HandleCodeBlockFinishState() -> void { +auto Parser::HandleCodeBlockFinishState() -> void { auto state = PopState(); // If the block started with an open curly, this is a close curly. @@ -683,7 +682,7 @@ auto Parser2::HandleCodeBlockFinishState() -> void { } } -auto Parser2::HandleDeclarationLoopState() -> void { +auto Parser::HandleDeclarationLoopState() -> void { // This maintains the current state unless we're at the end of the file. switch (PositionKind()) { @@ -725,7 +724,7 @@ auto Parser2::HandleDeclarationLoopState() -> void { } } -auto Parser2::HandleDesignator(bool as_struct) -> void { +auto Parser::HandleDesignator(bool as_struct) -> void { auto state = PopState(); // `.` identifier @@ -753,15 +752,15 @@ auto Parser2::HandleDesignator(bool as_struct) -> void { *dot, state.subtree_start, state.has_error); } -auto Parser2::HandleDesignatorAsExpressionState() -> void { +auto Parser::HandleDesignatorAsExpressionState() -> void { HandleDesignator(/*as_struct=*/false); } -auto Parser2::HandleDesignatorAsStructState() -> void { +auto Parser::HandleDesignatorAsStructState() -> void { HandleDesignator(/*as_struct=*/true); } -auto Parser2::HandleExpressionState() -> void { +auto Parser::HandleExpressionState() -> void { auto state = PopState(); // Check for a prefix operator. @@ -789,7 +788,7 @@ auto Parser2::HandleExpressionState() -> void { } } -auto Parser2::HandleExpressionInPostfixState() -> void { +auto Parser::HandleExpressionInPostfixState() -> void { auto state = PopState(); // Continue to the loop state. @@ -833,7 +832,7 @@ auto Parser2::HandleExpressionInPostfixState() -> void { } } -auto Parser2::HandleExpressionInPostfixLoopState() -> void { +auto Parser::HandleExpressionInPostfixLoopState() -> void { // This is a cyclic state that repeats, so this state is typically pushed back // on. auto state = PopState(); @@ -862,7 +861,7 @@ auto Parser2::HandleExpressionInPostfixLoopState() -> void { } } -auto Parser2::HandleExpressionLoopState() -> void { +auto Parser::HandleExpressionLoopState() -> void { auto state = PopState(); auto trailing_operator = @@ -916,7 +915,7 @@ auto Parser2::HandleExpressionLoopState() -> void { } } -auto Parser2::HandleExpressionLoopForBinaryState() -> void { +auto Parser::HandleExpressionLoopForBinaryState() -> void { auto state = PopState(); AddNode(ParseNodeKind::InfixOperator(), state.token, state.subtree_start, @@ -926,7 +925,7 @@ auto Parser2::HandleExpressionLoopForBinaryState() -> void { PushState(state); } -auto Parser2::HandleExpressionLoopForPrefixState() -> void { +auto Parser::HandleExpressionLoopForPrefixState() -> void { auto state = PopState(); AddNode(ParseNodeKind::PrefixOperator(), state.token, state.subtree_start, @@ -936,7 +935,7 @@ auto Parser2::HandleExpressionLoopForPrefixState() -> void { PushState(state); } -auto Parser2::HandleExpressionStatementFinishState() -> void { +auto Parser::HandleExpressionStatementFinishState() -> void { auto state = PopState(); if (auto semi = ConsumeIf(TokenKind::Semi())) { @@ -960,8 +959,8 @@ auto Parser2::HandleExpressionStatementFinishState() -> void { ReturnErrorOnState(); } -auto Parser2::HandleFunctionError(StateStackEntry state, - bool skip_past_likely_end) -> void { +auto Parser::HandleFunctionError(StateStackEntry state, + bool skip_past_likely_end) -> void { auto token = state.token; if (skip_past_likely_end) { if (auto semi = SkipPastLikelyEnd(token)) { @@ -972,7 +971,7 @@ auto Parser2::HandleFunctionError(StateStackEntry state, /*has_error=*/true); } -auto Parser2::HandleFunctionIntroducerState() -> void { +auto Parser::HandleFunctionIntroducerState() -> void { auto state = PopState(); if (!ConsumeAndAddLeafNodeIf(TokenKind::Identifier(), @@ -1009,7 +1008,7 @@ auto Parser2::HandleFunctionIntroducerState() -> void { } } -auto Parser2::HandleFunctionParameterListFinishState() -> void { +auto Parser::HandleFunctionParameterListFinishState() -> void { auto state = PopState(); CARBON_CHECK(ConsumeAndAddLeafNodeIf(TokenKind::CloseParen(), @@ -1019,7 +1018,7 @@ auto Parser2::HandleFunctionParameterListFinishState() -> void { state.has_error); } -auto Parser2::HandleFunctionAfterParameterListState() -> void { +auto Parser::HandleFunctionAfterParameterListState() -> void { auto state = PopState(); // Regardless of whether there's a return type, we'll finish the signature. @@ -1035,14 +1034,14 @@ auto Parser2::HandleFunctionAfterParameterListState() -> void { } } -auto Parser2::HandleFunctionReturnTypeFinishState() -> void { +auto Parser::HandleFunctionReturnTypeFinishState() -> void { auto state = PopState(); AddNode(ParseNodeKind::ReturnType(), state.token, state.subtree_start, state.has_error); } -auto Parser2::HandleFunctionSignatureFinishState() -> void { +auto Parser::HandleFunctionSignatureFinishState() -> void { auto state = PopState(); switch (PositionKind()) { @@ -1075,13 +1074,13 @@ auto Parser2::HandleFunctionSignatureFinishState() -> void { } } -auto Parser2::HandleFunctionDefinitionFinishState() -> void { +auto Parser::HandleFunctionDefinitionFinishState() -> void { auto state = PopState(); AddNode(ParseNodeKind::FunctionDefinition(), Consume(), state.subtree_start, state.has_error); } -auto Parser2::HandlePackageState() -> void { +auto Parser::HandlePackageState() -> void { auto state = PopState(); auto exit_on_parse_error = [&]() { @@ -1160,7 +1159,7 @@ auto Parser2::HandlePackageState() -> void { /*has_error=*/false); } -auto Parser2::HandleParenConditionState() -> void { +auto Parser::HandleParenConditionState() -> void { auto state = PopState(); auto open_paren = ConsumeIf(TokenKind::OpenParen()); @@ -1179,7 +1178,7 @@ auto Parser2::HandleParenConditionState() -> void { PushState(ParserState::Expression()); } -auto Parser2::HandleParenConditionFinishState() -> void { +auto Parser::HandleParenConditionFinishState() -> void { auto state = PopState(); if (tokens_.GetKind(state.token) != TokenKind::OpenParen()) { @@ -1197,7 +1196,7 @@ auto Parser2::HandleParenConditionFinishState() -> void { /*has_error=*/state.has_error || !close_paren); } -auto Parser2::HandleParenExpressionState() -> void { +auto Parser::HandleParenExpressionState() -> void { auto state = PopState(); // TODO: When swapping () start/end, this should AddLeafNode the open before @@ -1217,7 +1216,7 @@ auto Parser2::HandleParenExpressionState() -> void { } } -auto Parser2::HandleParenExpressionParameterFinish(bool as_tuple) -> void { +auto Parser::HandleParenExpressionParameterFinish(bool as_tuple) -> void { auto state = PopState(); auto list_token_kind = @@ -1247,15 +1246,15 @@ auto Parser2::HandleParenExpressionParameterFinish(bool as_tuple) -> void { } } -auto Parser2::HandleParenExpressionParameterFinishAsUnknownState() -> void { +auto Parser::HandleParenExpressionParameterFinishAsUnknownState() -> void { HandleParenExpressionParameterFinish(/*as_tuple=*/false); } -auto Parser2::HandleParenExpressionParameterFinishAsTupleState() -> void { +auto Parser::HandleParenExpressionParameterFinishAsTupleState() -> void { HandleParenExpressionParameterFinish(/*as_tuple=*/true); } -auto Parser2::HandleParenExpressionFinishState() -> void { +auto Parser::HandleParenExpressionFinishState() -> void { auto state = PopState(); AddLeafNode(ParseNodeKind::ParenExpressionEnd(), Consume()); @@ -1263,7 +1262,7 @@ auto Parser2::HandleParenExpressionFinishState() -> void { state.has_error); } -auto Parser2::HandleParenExpressionFinishAsTupleState() -> void { +auto Parser::HandleParenExpressionFinishAsTupleState() -> void { auto state = PopState(); AddLeafNode(ParseNodeKind::TupleLiteralEnd(), Consume()); @@ -1271,7 +1270,7 @@ auto Parser2::HandleParenExpressionFinishAsTupleState() -> void { state.has_error); } -auto Parser2::HandlePatternStart(PatternKind pattern_kind) -> void { +auto Parser::HandlePatternStart(PatternKind pattern_kind) -> void { auto state = PopState(); // Ensure the finish state always follows. @@ -1317,7 +1316,7 @@ auto Parser2::HandlePatternStart(PatternKind pattern_kind) -> void { position_ += 2; } -auto Parser2::HandlePatternFinish() -> bool { +auto Parser::HandlePatternFinish() -> bool { auto state = PopState(); // If an error was encountered, propagate it without adding a node. @@ -1332,11 +1331,11 @@ auto Parser2::HandlePatternFinish() -> bool { return false; } -auto Parser2::HandlePatternAsFunctionParameterState() -> void { +auto Parser::HandlePatternAsFunctionParameterState() -> void { HandlePatternStart(PatternKind::Parameter); } -auto Parser2::HandlePatternAsFunctionParameterFinishState() -> void { +auto Parser::HandlePatternAsFunctionParameterFinishState() -> void { bool has_error = HandlePatternFinish(); if (ConsumeListToken(ParseNodeKind::ParameterListComma(), @@ -1346,15 +1345,15 @@ auto Parser2::HandlePatternAsFunctionParameterFinishState() -> void { } } -auto Parser2::HandlePatternAsVariableState() -> void { +auto Parser::HandlePatternAsVariableState() -> void { HandlePatternStart(PatternKind::Variable); } -auto Parser2::HandlePatternAsVariableFinishState() -> void { +auto Parser::HandlePatternAsVariableFinishState() -> void { HandlePatternFinish(); } -auto Parser2::HandleStatementState() -> void { +auto Parser::HandleStatementState() -> void { PopAndDiscardState(); switch (PositionKind()) { @@ -1401,17 +1400,17 @@ auto Parser2::HandleStatementState() -> void { } } -auto Parser2::HandleStatementBreakFinishState() -> void { +auto Parser::HandleStatementBreakFinishState() -> void { HandleStatementKeywordFinish(TokenKind::Break(), ParseNodeKind::BreakStatement()); } -auto Parser2::HandleStatementContinueFinishState() -> void { +auto Parser::HandleStatementContinueFinishState() -> void { HandleStatementKeywordFinish(TokenKind::Continue(), ParseNodeKind::ContinueStatement()); } -auto Parser2::HandleStatementForHeaderState() -> void { +auto Parser::HandleStatementForHeaderState() -> void { auto state = PopState(); auto open_paren = ConsumeIf(TokenKind::OpenParen()); @@ -1449,7 +1448,7 @@ auto Parser2::HandleStatementForHeaderState() -> void { } } -auto Parser2::HandleStatementForHeaderInState() -> void { +auto Parser::HandleStatementForHeaderInState() -> void { auto state = PopState(); state.state = ParserState::StatementForHeaderFinish(); @@ -1477,7 +1476,7 @@ auto Parser2::HandleStatementForHeaderInState() -> void { PushState(ParserState::Expression()); } -auto Parser2::HandleStatementForHeaderFinishState() -> void { +auto Parser::HandleStatementForHeaderFinishState() -> void { auto state = PopState(); if (!ConsumeAndAddCloseParen(state.token, ParseNodeKind::ForHeaderEnd())) { @@ -1490,14 +1489,14 @@ auto Parser2::HandleStatementForHeaderFinishState() -> void { PushState(ParserState::CodeBlock()); } -auto Parser2::HandleStatementForFinishState() -> void { +auto Parser::HandleStatementForFinishState() -> void { auto state = PopState(); AddNode(ParseNodeKind::ForStatement(), state.token, state.subtree_start, state.has_error); } -auto Parser2::HandleStatementIfState() -> void { +auto Parser::HandleStatementIfState() -> void { PopAndDiscardState(); PushState(ParserState::StatementIfConditionFinish()); @@ -1505,7 +1504,7 @@ auto Parser2::HandleStatementIfState() -> void { ++position_; } -auto Parser2::HandleStatementIfConditionFinishState() -> void { +auto Parser::HandleStatementIfConditionFinishState() -> void { auto state = PopState(); state.state = ParserState::StatementIfThenBlockFinish(); @@ -1513,7 +1512,7 @@ auto Parser2::HandleStatementIfConditionFinishState() -> void { PushState(ParserState::CodeBlock()); } -auto Parser2::HandleStatementIfThenBlockFinishState() -> void { +auto Parser::HandleStatementIfThenBlockFinishState() -> void { auto state = PopState(); if (ConsumeAndAddLeafNodeIf(TokenKind::Else(), @@ -1529,14 +1528,14 @@ auto Parser2::HandleStatementIfThenBlockFinishState() -> void { } } -auto Parser2::HandleStatementIfElseBlockFinishState() -> void { +auto Parser::HandleStatementIfElseBlockFinishState() -> void { auto state = PopState(); AddNode(ParseNodeKind::IfStatement(), state.token, state.subtree_start, state.has_error); } -auto Parser2::HandleStatementKeywordFinish(TokenKind token_kind, - ParseNodeKind node_kind) -> void { +auto Parser::HandleStatementKeywordFinish(TokenKind token_kind, + ParseNodeKind node_kind) -> void { auto state = PopState(); if (!ConsumeAndAddLeafNodeIf(TokenKind::Semi(), @@ -1552,7 +1551,7 @@ auto Parser2::HandleStatementKeywordFinish(TokenKind token_kind, AddNode(node_kind, state.token, state.subtree_start, state.has_error); } -auto Parser2::HandleStatementReturnState() -> void { +auto Parser::HandleStatementReturnState() -> void { auto state = PopState(); state.state = ParserState::StatementReturnFinish(); @@ -1563,12 +1562,12 @@ auto Parser2::HandleStatementReturnState() -> void { } } -auto Parser2::HandleStatementReturnFinishState() -> void { +auto Parser::HandleStatementReturnFinishState() -> void { HandleStatementKeywordFinish(TokenKind::Return(), ParseNodeKind::ReturnStatement()); } -auto Parser2::HandleStatementScopeLoopState() -> void { +auto Parser::HandleStatementScopeLoopState() -> void { // This maintains the current state until we're at the end of the scope. auto token_kind = PositionKind(); @@ -1582,7 +1581,7 @@ auto Parser2::HandleStatementScopeLoopState() -> void { } } -auto Parser2::HandleStatementWhileState() -> void { +auto Parser::HandleStatementWhileState() -> void { PopAndDiscardState(); PushState(ParserState::StatementWhileConditionFinish()); @@ -1590,7 +1589,7 @@ auto Parser2::HandleStatementWhileState() -> void { ++position_; } -auto Parser2::HandleStatementWhileConditionFinishState() -> void { +auto Parser::HandleStatementWhileConditionFinishState() -> void { auto state = PopState(); state.state = ParserState::StatementWhileBlockFinish(); @@ -1598,14 +1597,14 @@ auto Parser2::HandleStatementWhileConditionFinishState() -> void { PushState(ParserState::CodeBlock()); } -auto Parser2::HandleStatementWhileBlockFinishState() -> void { +auto Parser::HandleStatementWhileBlockFinishState() -> void { auto state = PopState(); AddNode(ParseNodeKind::WhileStatement(), state.token, state.subtree_start, state.has_error); } -auto Parser2::HandleVar(bool require_semicolon) -> void { +auto Parser::HandleVar(bool require_semicolon) -> void { PopAndDiscardState(); PushState(require_semicolon ? ParserState::VarFinishAsRequireSemicolon() @@ -1615,15 +1614,15 @@ auto Parser2::HandleVar(bool require_semicolon) -> void { PushState(ParserState::PatternAsVariable()); } -auto Parser2::HandleVarAsRequireSemicolonState() -> void { +auto Parser::HandleVarAsRequireSemicolonState() -> void { HandleVar(/*require_semicolon=*/true); } -auto Parser2::HandleVarAsNoSemicolonState() -> void { +auto Parser::HandleVarAsNoSemicolonState() -> void { HandleVar(/*require_semicolon=*/false); } -auto Parser2::HandleVarAfterPatternState() -> void { +auto Parser::HandleVarAfterPatternState() -> void { auto state = PopState(); if (state.has_error) { @@ -1641,14 +1640,14 @@ auto Parser2::HandleVarAfterPatternState() -> void { } } -auto Parser2::HandleVarAfterInitializerState() -> void { +auto Parser::HandleVarAfterInitializerState() -> void { auto state = PopState(); AddNode(ParseNodeKind::VariableInitializer(), state.token, state.subtree_start, state.has_error); } -auto Parser2::HandleVarFinish(bool require_semicolon) -> void { +auto Parser::HandleVarFinish(bool require_semicolon) -> void { auto state = PopState(); if (require_semicolon) { @@ -1669,11 +1668,11 @@ auto Parser2::HandleVarFinish(bool require_semicolon) -> void { state.subtree_start, state.has_error); } -auto Parser2::HandleVarFinishAsRequireSemicolonState() -> void { +auto Parser::HandleVarFinishAsRequireSemicolonState() -> void { HandleVarFinish(/*require_semicolon=*/true); } -auto Parser2::HandleVarFinishAsNoSemicolonState() -> void { +auto Parser::HandleVarFinishAsNoSemicolonState() -> void { HandleVarFinish(/*require_semicolon=*/false); } diff --git a/toolchain/parser/parser2.h b/toolchain/parser/parser.h similarity index 97% rename from toolchain/parser/parser2.h rename to toolchain/parser/parser.h index d47cd5c45944..e8a8f26eb5e5 100644 --- a/toolchain/parser/parser2.h +++ b/toolchain/parser/parser.h @@ -2,8 +2,8 @@ // Exceptions. See /LICENSE for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#ifndef CARBON_TOOLCHAIN_PARSER_PARSER2_H_ -#define CARBON_TOOLCHAIN_PARSER_PARSER2_H_ +#ifndef CARBON_TOOLCHAIN_PARSER_PARSER_H_ +#define CARBON_TOOLCHAIN_PARSER_PARSER_H_ #include "common/check.h" #include "llvm/ADT/Optional.h" @@ -16,7 +16,7 @@ namespace Carbon { -class Parser2 { +class Parser { public: // Parses the tokens into a parse tree, emitting any errors encountered. // @@ -24,7 +24,7 @@ class Parser2 { static auto Parse(TokenizedBuffer& tokens, TokenDiagnosticEmitter& emitter) -> ParseTree { ParseTree tree(tokens); - Parser2 parser(tree, tokens, emitter); + Parser parser(tree, tokens, emitter); parser.Parse(); return tree; } @@ -99,8 +99,8 @@ class Parser2 { // The kind of brace expression being evaluated. enum class BraceExpressionKind { Unknown, Value, Type }; - Parser2(ParseTree& tree, TokenizedBuffer& tokens, - TokenDiagnosticEmitter& emitter); + Parser(ParseTree& tree, TokenizedBuffer& tokens, + TokenDiagnosticEmitter& emitter); auto Parse() -> void; @@ -305,4 +305,4 @@ class Parser2 { } // namespace Carbon -#endif // CARBON_TOOLCHAIN_PARSER_PARSER2_H_ +#endif // CARBON_TOOLCHAIN_PARSER_PARSER_H_ diff --git a/toolchain/parser/parser_impl.cpp b/toolchain/parser/parser_impl.cpp deleted file mode 100644 index 3a459e4c634f..000000000000 --- a/toolchain/parser/parser_impl.cpp +++ /dev/null @@ -1,1342 +0,0 @@ -// Part of the Carbon Language project, under the Apache License v2.0 with LLVM -// Exceptions. See /LICENSE for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -#include "toolchain/parser/parser_impl.h" - -#include - -#include "common/check.h" -#include "llvm/ADT/Optional.h" -#include "llvm/Support/FormatVariadic.h" -#include "llvm/Support/raw_ostream.h" -#include "toolchain/lexer/token_kind.h" -#include "toolchain/lexer/tokenized_buffer.h" -#include "toolchain/parser/parse_node_kind.h" -#include "toolchain/parser/parse_tree.h" - -namespace Carbon { - -CARBON_DIAGNOSTIC(ExpectedSemiAfterExpression, Error, - "Expected `;` after expression."); - -// Manages the parser's stack depth, particularly decrementing on destruction. -// This should only be instantiated through RETURN_IF_STACK_LIMITED. -class ParseTree::Parser::ScopedStackStep { - public: - explicit ScopedStackStep(ParseTree::Parser* parser) : parser_(parser) { - ++parser_->stack_depth_; - } - ~ScopedStackStep() { --parser_->stack_depth_; } - - auto VerifyUnderLimit() -> bool { - if (parser_->stack_depth_ >= StackDepthLimit) { - CARBON_DIAGNOSTIC(StackLimitExceeded, Error, - "Exceeded recursion limit ({0})", int); - parser_->emitter_.Emit(*parser_->position_, StackLimitExceeded, - ParseTree::StackDepthLimit); - return false; - } - return true; - } - - private: - ParseTree::Parser* parser_; -}; - -// Encapsulates checking the stack and erroring if needed. This should be called -// at the start of every parse function. -#define CARBON_RETURN_IF_STACK_LIMITED(error_return_expr) \ - ScopedStackStep scoped_stack_step(this); \ - if (!scoped_stack_step.VerifyUnderLimit()) { \ - return (error_return_expr); \ - } - -// A relative location for characters in errors. -enum class RelativeLocation : int8_t { - Around, - After, - Before, -}; - -// Adapts RelativeLocation for use with formatv. -static auto operator<<(llvm::raw_ostream& out, RelativeLocation loc) - -> llvm::raw_ostream& { - switch (loc) { - case RelativeLocation::Around: - out << "around"; - break; - case RelativeLocation::After: - out << "after"; - break; - case RelativeLocation::Before: - out << "before"; - break; - } - return out; -} - -ParseTree::Parser::Parser(ParseTree& tree_arg, TokenizedBuffer& tokens_arg, - TokenDiagnosticEmitter& emitter) - : tree_(tree_arg), - tokens_(tokens_arg), - emitter_(emitter), - position_(tokens_.tokens().begin()), - end_(tokens_.tokens().end()) { - CARBON_CHECK(std::find_if(position_, end_, - [&](TokenizedBuffer::Token t) { - return tokens_.GetKind(t) == - TokenKind::EndOfFile(); - }) != end_) - << "No EndOfFileToken in token buffer."; -} - -auto ParseTree::Parser::Parse(TokenizedBuffer& tokens, - TokenDiagnosticEmitter& emitter) -> ParseTree { - ParseTree tree(tokens); - - // Reserve the space we expect to need for nodes in order to avoid allocation - // and copying overhead. This should be a one-to-one correspondence in an - // error-free tree. - tree.node_impls_.reserve(tokens.size()); - - Parser parser(tree, tokens, emitter); - while (!parser.AtEndOfFile()) { - if (!parser.ParseDeclaration()) { - // We don't have an enclosing parse tree node to mark as erroneous, so - // just mark the tree as a whole. - tree.has_errors_ = true; - } - } - - parser.AddLeafNode(ParseNodeKind::FileEnd(), *parser.position_); - - CARBON_CHECK(tree.has_errors() || tree.size() == tokens.size()) - << "Failed to correctly calculate size: expected " << tokens.size() - << ", got " << tree.size(); - CARBON_CHECK(tree.Verify()) << "Parse tree built but does not verify!"; - return tree; -} - -auto ParseTree::Parser::Consume(TokenKind kind) -> TokenizedBuffer::Token { - CARBON_CHECK(kind != TokenKind::EndOfFile()) - << "Cannot consume the EOF token!"; - CARBON_CHECK(NextTokenIs(kind)) << "The current token is the wrong kind!"; - TokenizedBuffer::Token t = *position_; - ++position_; - CARBON_CHECK(position_ != end_) - << "Reached end of tokens without finding EOF token."; - return t; -} - -auto ParseTree::Parser::ConsumeIf(TokenKind kind) - -> llvm::Optional { - if (!NextTokenIs(kind)) { - return {}; - } - return Consume(kind); -} - -auto ParseTree::Parser::AddLeafNode(ParseNodeKind kind, - TokenizedBuffer::Token token, - bool has_error) -> Node { - Node n(tree_.node_impls_.size()); - tree_.node_impls_.push_back(NodeImpl(kind, token, /*subtree_size_arg=*/1)); - if (has_error) { - MarkNodeError(n); - } - return n; -} - -auto ParseTree::Parser::ConsumeAndAddLeafNodeIf(TokenKind t_kind, - ParseNodeKind n_kind) - -> llvm::Optional { - auto t = ConsumeIf(t_kind); - if (!t) { - return {}; - } - - return AddLeafNode(n_kind, *t); -} - -auto ParseTree::Parser::MarkNodeError(Node n) -> void { - tree_.node_impls_[n.index_].has_error = true; - tree_.has_errors_ = true; -} - -// A marker for the start of a node's subtree. -// -// This is used to track the size of the node's subtree. It can be used -// repeatedly if multiple subtrees start at the same position. -struct ParseTree::Parser::SubtreeStart { - int tree_size; -}; - -auto ParseTree::Parser::GetSubtreeStartPosition() -> SubtreeStart { - return {static_cast(tree_.node_impls_.size())}; -} - -auto ParseTree::Parser::AddNode(ParseNodeKind n_kind, TokenizedBuffer::Token t, - SubtreeStart start, bool has_error) -> Node { - // The size of the subtree is the change in size from when we started this - // subtree to now, but including the node we're about to add. - int tree_stop_size = static_cast(tree_.node_impls_.size()) + 1; - int subtree_size = tree_stop_size - start.tree_size; - - Node n(tree_.node_impls_.size()); - tree_.node_impls_.push_back(NodeImpl(n_kind, t, subtree_size)); - if (has_error) { - MarkNodeError(n); - } - - return n; -} - -auto ParseTree::Parser::SkipMatchingGroup() -> bool { - TokenizedBuffer::Token t = *position_; - TokenKind t_kind = tokens_.GetKind(t); - if (!t_kind.IsOpeningSymbol()) { - return false; - } - - SkipTo(tokens_.GetMatchedClosingToken(t)); - Consume(t_kind.GetClosingSymbol()); - return true; -} - -auto ParseTree::Parser::SkipTo(TokenizedBuffer::Token t) -> void { - CARBON_CHECK(t >= *position_) << "Tried to skip backwards."; - position_ = TokenizedBuffer::TokenIterator(t); - CARBON_CHECK(position_ != end_) << "Skipped past EOF."; -} - -auto ParseTree::Parser::FindNextOf( - std::initializer_list desired_kinds) - -> llvm::Optional { - auto new_position = position_; - while (true) { - TokenizedBuffer::Token token = *new_position; - TokenKind kind = tokens_.GetKind(token); - if (kind.IsOneOf(desired_kinds)) { - return token; - } - - // Step to the next token at the current bracketing level. - if (kind.IsClosingSymbol() || kind == TokenKind::EndOfFile()) { - // There are no more tokens at this level. - return llvm::None; - } else if (kind.IsOpeningSymbol()) { - new_position = - TokenizedBuffer::TokenIterator(tokens_.GetMatchedClosingToken(token)); - // Advance past the closing token. - ++new_position; - } else { - ++new_position; - } - } -} - -auto ParseTree::Parser::SkipPastLikelyEnd(TokenizedBuffer::Token skip_root) - -> llvm::Optional { - if (AtEndOfFile()) { - return llvm::None; - } - - TokenizedBuffer::Line root_line = tokens_.GetLine(skip_root); - int root_line_indent = tokens_.GetIndentColumnNumber(root_line); - - // We will keep scanning through tokens on the same line as the root or - // lines with greater indentation than root's line. - auto is_same_line_or_indent_greater_than_root = - [&](TokenizedBuffer::Token t) { - TokenizedBuffer::Line l = tokens_.GetLine(t); - if (l == root_line) { - return true; - } - - return tokens_.GetIndentColumnNumber(l) > root_line_indent; - }; - - do { - if (NextTokenKind() == TokenKind::CloseCurlyBrace()) { - // Immediately bail out if we hit an unmatched close curly, this will - // pop us up a level of the syntax grouping. - return llvm::None; - } - - // We assume that a semicolon is always intended to be the end of the - // current construct. - if (auto semi = ConsumeIf(TokenKind::Semi())) { - return semi; - } - - // Skip over any matching group of tokens_. - if (SkipMatchingGroup()) { - continue; - } - - // Otherwise just step forward one token. - Consume(NextTokenKind()); - } while (!AtEndOfFile() && - is_same_line_or_indent_greater_than_root(*position_)); - - return llvm::None; -} - -auto ParseTree::Parser::ParseCloseParen(TokenizedBuffer::Token open_paren, - ParseNodeKind kind) - -> llvm::Optional { - if (auto close_paren = - ConsumeAndAddLeafNodeIf(TokenKind::CloseParen(), kind)) { - return close_paren; - } - - // TODO: Include the location of the matching open_paren in the diagnostic. - CARBON_DIAGNOSTIC(ExpectedCloseParen, Error, "Unexpected tokens before `)`."); - emitter_.Emit(*position_, ExpectedCloseParen); - SkipTo(tokens_.GetMatchedClosingToken(open_paren)); - AddLeafNode(kind, Consume(TokenKind::CloseParen())); - return llvm::None; -} - -template -auto ParseTree::Parser::ParseList(TokenKind open, TokenKind close, - ListElementParser list_element_parser, - ParseNodeKind comma_kind, - ListCompletionHandler list_handler, - bool allow_trailing_comma) - -> llvm::Optional { - // `(` element-list[opt] `)` - // - // element-list ::= element - // ::= element `,` element-list - TokenizedBuffer::Token open_paren = Consume(open); - - bool has_errors = false; - bool any_commas = false; - int64_t num_elements = 0; - - // Parse elements, if any are specified. - if (!NextTokenIs(close)) { - while (true) { - bool element_error = !list_element_parser(); - has_errors |= element_error; - ++num_elements; - - if (!NextTokenIsOneOf({close, TokenKind::Comma()})) { - if (!element_error) { - CARBON_DIAGNOSTIC(UnexpectedTokenAfterListElement, Error, - "Expected `,` or `{0}`.", TokenKind); - emitter_.Emit(*position_, UnexpectedTokenAfterListElement, close); - } - has_errors = true; - - auto end_of_element = FindNextOf({TokenKind::Comma(), close}); - // The lexer guarantees that parentheses are balanced. - CARBON_CHECK(end_of_element) << "missing matching `)` for `(`"; - SkipTo(*end_of_element); - } - - if (NextTokenIs(close)) { - break; - } - - AddLeafNode(comma_kind, Consume(TokenKind::Comma())); - any_commas = true; - - if (allow_trailing_comma && NextTokenIs(close)) { - break; - } - } - } - - bool is_single_item = num_elements == 1 && !any_commas; - return list_handler(open_paren, is_single_item, Consume(close), has_errors); -} - -auto ParseTree::Parser::ParsePattern(PatternKind kind) -> llvm::Optional { - CARBON_RETURN_IF_STACK_LIMITED(llvm::None); - if (NextTokenIs(TokenKind::Identifier()) && - tokens_.GetKind(*(position_ + 1)) == TokenKind::Colon()) { - // identifier `:` type - auto start = GetSubtreeStartPosition(); - AddLeafNode(ParseNodeKind::DeclaredName(), - Consume(TokenKind::Identifier())); - auto colon = Consume(TokenKind::Colon()); - auto type = ParseType(); - return AddNode(ParseNodeKind::PatternBinding(), colon, start, - /*has_error=*/!type); - } - - switch (kind) { - case PatternKind::Parameter: - CARBON_DIAGNOSTIC(ExpectedParameterName, Error, - "Expected parameter declaration."); - emitter_.Emit(*position_, ExpectedParameterName); - break; - - case PatternKind::Variable: - CARBON_DIAGNOSTIC(ExpectedVariableName, Error, - "Expected pattern in `var` declaration."); - emitter_.Emit(*position_, ExpectedVariableName); - break; - } - - return llvm::None; -} - -auto ParseTree::Parser::ParseFunctionParameter() -> llvm::Optional { - CARBON_RETURN_IF_STACK_LIMITED(llvm::None); - return ParsePattern(PatternKind::Parameter); -} - -auto ParseTree::Parser::ParseFunctionSignature() -> bool { - CARBON_RETURN_IF_STACK_LIMITED(false); - auto start = GetSubtreeStartPosition(); - - auto params = ParseParenList( - [&] { return ParseFunctionParameter(); }, - ParseNodeKind::ParameterListComma(), - [&](TokenizedBuffer::Token open_paren, bool /*is_single_item*/, - TokenizedBuffer::Token close_paren, bool has_errors) { - AddLeafNode(ParseNodeKind::ParameterListEnd(), close_paren); - return AddNode(ParseNodeKind::ParameterList(), open_paren, start, - has_errors); - }); - - auto start_return_type = GetSubtreeStartPosition(); - if (auto arrow = ConsumeIf(TokenKind::MinusGreater())) { - auto return_type = ParseType(); - AddNode(ParseNodeKind::ReturnType(), *arrow, start_return_type, - /*has_error=*/!return_type); - if (!return_type) { - return false; - } - } - - return params.has_value(); -} - -auto ParseTree::Parser::ParseCodeBlock() -> llvm::Optional { - return ParseCodeBlock(GetSubtreeStartPosition(), - ParseNodeKind::CodeBlockStart(), - ParseNodeKind::CodeBlock()); -} - -auto ParseTree::Parser::ParseCodeBlock(SubtreeStart subtree_start, - ParseNodeKind start_kind, - ParseNodeKind end_kind) - -> llvm::Optional { - CARBON_RETURN_IF_STACK_LIMITED(llvm::None); - llvm::Optional open_curly = - ConsumeIf(TokenKind::OpenCurlyBrace()); - if (!open_curly) { - // Recover by parsing a single statement. - CARBON_DIAGNOSTIC(ExpectedCodeBlock, Error, "Expected braced code block."); - emitter_.Emit(*position_, ExpectedCodeBlock); - // Use the unexpected token for the block start and end. - TokenizedBuffer::Token recovery_start = *position_; - AddNode(start_kind, recovery_start, subtree_start, /*has_error=*/true); - ParseStatement(); - return AddNode(end_kind, recovery_start, subtree_start, /*has_error=*/true); - } - - AddNode(start_kind, *open_curly, subtree_start); - - // Loop over all the different possibly nested elements in the code block. - bool has_error = false; - while (!NextTokenIs(TokenKind::CloseCurlyBrace())) { - if (!ParseStatement()) { - // We detected and diagnosed an error of some kind. We can trivially - // skip to the actual close curly brace from here. - // TODO: It would be better to skip to the next semicolon, or the next - // token at the start of a line with the same indent as this one. - SkipTo(tokens_.GetMatchedClosingToken(*open_curly)); - has_error = true; - break; - } - } - - // We always reach here having set our position in the token stream to the - // close curly brace. - return AddNode(end_kind, Consume(TokenKind::CloseCurlyBrace()), subtree_start, - /*has_error=*/has_error); -} - -auto ParseTree::Parser::ParsePackageDirective() -> Node { - TokenizedBuffer::Token package_intro_token = Consume(TokenKind::Package()); - auto package_start = GetSubtreeStartPosition(); - auto create_error_node = [&]() { - return AddNode(ParseNodeKind::PackageDirective(), package_intro_token, - package_start, - /*has_error=*/true); - }; - - CARBON_RETURN_IF_STACK_LIMITED(create_error_node()); - - auto exit_on_parse_error = [&]() { - if (auto semi_token = SkipPastLikelyEnd(package_intro_token)) { - AddLeafNode(ParseNodeKind::PackageEnd(), *semi_token); - } - - return create_error_node(); - }; - - if (!NextTokenIs(TokenKind::Identifier())) { - CARBON_DIAGNOSTIC(ExpectedIdentifierAfterPackage, Error, - "Expected identifier after `package`."); - emitter_.Emit(*position_, ExpectedIdentifierAfterPackage); - return exit_on_parse_error(); - } - - AddLeafNode(ParseNodeKind::DeclaredName(), Consume(TokenKind::Identifier())); - bool library_parsed = false; - - if (tokens_.GetKind(*(position_)) == TokenKind::Library()) { - auto library_start = GetSubtreeStartPosition(); - auto library_decl_token = Consume(TokenKind::Library()); - - if (tokens_.GetKind(*(position_)) != TokenKind::StringLiteral()) { - CARBON_DIAGNOSTIC( - ExpectedLibraryName, Error, - "Expected a string literal to specify the library name."); - emitter_.Emit(*position_, ExpectedLibraryName); - return exit_on_parse_error(); - } - - AddLeafNode(ParseNodeKind::Literal(), Consume(TokenKind::StringLiteral())); - AddNode(ParseNodeKind::PackageLibrary(), library_decl_token, library_start, - /*has_error=*/false); - library_parsed = true; - } - - auto api_or_impl_token = tokens_.GetKind(*(position_)); - - if (api_or_impl_token == TokenKind::Api()) { - AddLeafNode(ParseNodeKind::PackageApi(), Consume(TokenKind::Api())); - } else if (api_or_impl_token == TokenKind::Impl()) { - AddLeafNode(ParseNodeKind::PackageImpl(), Consume(TokenKind::Impl())); - } else if (!library_parsed && - api_or_impl_token == TokenKind::StringLiteral()) { - // If we come acroess a string literal and we didn't parse `library "..."` - // yet, then most probably the user forgot to add `library` before the - // library name. - CARBON_DIAGNOSTIC(MissingLibraryKeyword, Error, - "Missing `library` keyword."); - emitter_.Emit(*position_, MissingLibraryKeyword); - return exit_on_parse_error(); - } else { - CARBON_DIAGNOSTIC(ExpectedApiOrImpl, Error, "Expected a `api` or `impl`."); - emitter_.Emit(*position_, ExpectedApiOrImpl); - return exit_on_parse_error(); - } - - if (tokens_.GetKind(*(position_)) != TokenKind::Semi()) { - CARBON_DIAGNOSTIC(ExpectedSemiToEndPackageDirective, Error, - "Expected `;` to end package directive."); - emitter_.Emit(*position_, ExpectedSemiToEndPackageDirective); - return exit_on_parse_error(); - } - - AddLeafNode(ParseNodeKind::PackageEnd(), Consume(TokenKind::Semi())); - - return AddNode(ParseNodeKind::PackageDirective(), package_intro_token, - package_start, /*has_error=*/false); -} - -auto ParseTree::Parser::ParseFunctionDeclaration() -> Node { - auto start = GetSubtreeStartPosition(); - TokenizedBuffer::Token function_intro_token = Consume(TokenKind::Fn()); - AddLeafNode(ParseNodeKind::FunctionIntroducer(), function_intro_token); - - // When handling errors before the start of the definition, treat it as a - // declaration. Recover to a semicolon when it makes sense as a possible - // function end, otherwise use the fn token for the error. - auto add_error_function_node = [&](bool skip_past_likely_end) { - if (skip_past_likely_end) { - if (auto semi_token = SkipPastLikelyEnd(function_intro_token)) { - return AddNode(ParseNodeKind::FunctionDeclaration(), *semi_token, start, - /*has_error=*/true); - } - } - return AddNode(ParseNodeKind::FunctionDeclaration(), function_intro_token, - start, /*has_error=*/true); - }; - CARBON_RETURN_IF_STACK_LIMITED(add_error_function_node(false)); - - if (!ConsumeAndAddLeafNodeIf(TokenKind::Identifier(), - ParseNodeKind::DeclaredName())) { - CARBON_DIAGNOSTIC(ExpectedFunctionName, Error, - "Expected function name after `fn` keyword."); - emitter_.Emit(*position_, ExpectedFunctionName); - // TODO: We could change the lexer to allow us to synthesize certain - // kinds of tokens and try to "recover" here, but unclear that this is - // really useful. - return add_error_function_node(true); - } - - TokenizedBuffer::Token open_paren = *position_; - if (tokens_.GetKind(open_paren) != TokenKind::OpenParen()) { - CARBON_DIAGNOSTIC(ExpectedFunctionParams, Error, - "Expected `(` after function name."); - emitter_.Emit(open_paren, ExpectedFunctionParams); - return add_error_function_node(true); - } - TokenizedBuffer::Token close_paren = - tokens_.GetMatchedClosingToken(open_paren); - - if (!ParseFunctionSignature()) { - return add_error_function_node(true); - } - - switch (NextTokenKind()) { - case TokenKind::OpenCurlyBrace(): { - // Parse a definition which is represented as a code block. - if (auto node = - ParseCodeBlock(start, ParseNodeKind::FunctionDefinitionStart(), - ParseNodeKind::FunctionDefinition())) { - return *node; - } - return add_error_function_node(false); - } - case TokenKind::Semi(): { - return AddNode(ParseNodeKind::FunctionDeclaration(), - Consume(TokenKind::Semi()), start); - } - default: { - CARBON_DIAGNOSTIC( - ExpectedFunctionBodyOrSemi, Error, - "Expected function definition or `;` after function declaration."); - emitter_.Emit(*position_, ExpectedFunctionBodyOrSemi); - // Only need to skip if we've not already found a new line. - return add_error_function_node(tokens_.GetLine(*position_) == - tokens_.GetLine(close_paren)); - } - } -} - -auto ParseTree::Parser::ParseVariableDeclaration() -> Node { - // `var` pattern [= expression] `;` - TokenizedBuffer::Token var_token = Consume(TokenKind::Var()); - auto start = GetSubtreeStartPosition(); - - CARBON_RETURN_IF_STACK_LIMITED(AddNode(ParseNodeKind::VariableDeclaration(), - var_token, start, - /*has_error=*/true)); - - auto pattern = ParsePattern(PatternKind::Variable); - if (!pattern) { - if (auto after_pattern = - FindNextOf({TokenKind::Equal(), TokenKind::Semi()})) { - SkipTo(*after_pattern); - } - } - - auto start_init = GetSubtreeStartPosition(); - if (auto equal_token = ConsumeIf(TokenKind::Equal())) { - auto init = ParseExpression(); - AddNode(ParseNodeKind::VariableInitializer(), *equal_token, start_init, - /*has_error=*/!init); - } - - auto semi = ConsumeAndAddLeafNodeIf(TokenKind::Semi(), - ParseNodeKind::DeclarationEnd()); - if (!semi) { - emitter_.Emit(*position_, ExpectedSemiAfterExpression); - if (auto semi_token = SkipPastLikelyEnd(var_token)) { - semi = AddLeafNode(ParseNodeKind::DeclarationEnd(), *semi_token, - /*has_error=*/true); - } - } - - return AddNode(ParseNodeKind::VariableDeclaration(), var_token, start, - /*has_error=*/!pattern || !semi); -} - -auto ParseTree::Parser::ParseEmptyDeclaration() -> Node { - return AddLeafNode(ParseNodeKind::EmptyDeclaration(), - Consume(TokenKind::Semi())); -} - -auto ParseTree::Parser::ParseDeclaration() -> llvm::Optional { - CARBON_RETURN_IF_STACK_LIMITED(llvm::None); - switch (NextTokenKind()) { - case TokenKind::Package(): - return ParsePackageDirective(); - case TokenKind::Fn(): - return ParseFunctionDeclaration(); - case TokenKind::Var(): - return ParseVariableDeclaration(); - case TokenKind::Semi(): - return ParseEmptyDeclaration(); - case TokenKind::EndOfFile(): - return llvm::None; - default: - // Errors are handled outside the switch. - break; - } - - // Should happen for packages now. - // We didn't recognize an introducer for a valid declaration. - CARBON_DIAGNOSTIC(UnrecognizedDeclaration, Error, - "Unrecognized declaration introducer."); - emitter_.Emit(*position_, UnrecognizedDeclaration); - - // Skip forward past any end of a declaration we simply didn't understand so - // that we can find the start of the next declaration or the end of a scope. - if (auto semi_token = SkipPastLikelyEnd(*position_)) { - return AddLeafNode(ParseNodeKind::EmptyDeclaration(), *semi_token, - /*has_error=*/true); - } - - // Nothing, not even a semicolon found. - return llvm::None; -} - -auto ParseTree::Parser::ParseParenExpression() -> llvm::Optional { - CARBON_RETURN_IF_STACK_LIMITED(llvm::None); - // parenthesized-expression ::= `(` expression `)` - // tuple-literal ::= `(` `)` - // ::= `(` expression `,` [expression-list [`,`]] `)` - // - // Parse the union of these, `(` [expression-list [`,`]] `)`, and work out - // whether it's a tuple or a parenthesized expression afterwards. - auto start = GetSubtreeStartPosition(); - return ParseParenList( - [&] { return ParseExpression(); }, ParseNodeKind::TupleLiteralComma(), - [&](TokenizedBuffer::Token open_paren, bool is_single_item, - TokenizedBuffer::Token close_paren, bool has_arg_errors) { - AddLeafNode(is_single_item ? ParseNodeKind::ParenExpressionEnd() - : ParseNodeKind::TupleLiteralEnd(), - close_paren); - return AddNode(is_single_item ? ParseNodeKind::ParenExpression() - : ParseNodeKind::TupleLiteral(), - open_paren, start, has_arg_errors); - }, - /*allow_trailing_comma=*/true); -} - -auto ParseTree::Parser::ParseBraceExpression() -> llvm::Optional { - CARBON_RETURN_IF_STACK_LIMITED(llvm::None); - // braced-expression ::= `{` [field-value-list] `}` - // ::= `{` field-type-list `}` - // field-value-list ::= field-value [`,`] - // ::= field-value `,` field-value-list - // field-value ::= `.` identifier `=` expression - // field-type-list ::= field-type [`,`] - // ::= field-type `,` field-type-list - // field-type ::= `.` identifier `:` type - // - // Note that `{` `}` is the first form (an empty struct), but that an empty - // struct value also behaves as an empty struct type. - auto start = GetSubtreeStartPosition(); - enum Kind { Unknown, Value, Type }; - Kind kind = Unknown; - return ParseList( - TokenKind::OpenCurlyBrace(), TokenKind::CloseCurlyBrace(), - [&]() -> llvm::Optional { - auto start_elem = GetSubtreeStartPosition(); - - auto diagnose_invalid_syntax = [&] { - CARBON_DIAGNOSTIC(ExpectedStructLiteralField, Error, - "Expected {0}{1}{2}.", llvm::StringRef, - llvm::StringRef, llvm::StringRef); - bool can_be_type = kind != Value; - bool can_be_value = kind != Type; - emitter_.Emit(*position_, ExpectedStructLiteralField, - can_be_type ? "`.field: type`" : "", - (can_be_type && can_be_value) ? " or " : "", - can_be_value ? "`.field = value`" : ""); - return llvm::None; - }; - - if (!NextTokenIs(TokenKind::Period())) { - return diagnose_invalid_syntax(); - } - auto designator = ParseDesignatorExpression( - start_elem, ParseNodeKind::StructFieldDesignator(), - /*has_errors=*/false); - if (!designator) { - auto recovery_pos = FindNextOf( - {TokenKind::Equal(), TokenKind::Colon(), TokenKind::Comma()}); - if (!recovery_pos || - tokens_.GetKind(*recovery_pos) == TokenKind::Comma()) { - return llvm::None; - } - SkipTo(*recovery_pos); - } - - // Work out the kind of this element - Kind elem_kind = (NextTokenIs(TokenKind::Equal()) ? Value - : NextTokenIs(TokenKind::Colon()) ? Type - : Unknown); - if (elem_kind == Unknown || (kind != Unknown && elem_kind != kind)) { - return diagnose_invalid_syntax(); - } - kind = elem_kind; - - // Struct type fields and value fields use the same grammar except - // that one has a `:` separator and the other has an `=` separator. - auto equal_or_colon_token = - Consume(kind == Type ? TokenKind::Colon() : TokenKind::Equal()); - auto type_or_value = ParseExpression(); - return AddNode(kind == Type ? ParseNodeKind::StructFieldType() - : ParseNodeKind::StructFieldValue(), - equal_or_colon_token, start_elem, - /*has_error=*/!designator || !type_or_value); - }, - ParseNodeKind::StructComma(), - [&](TokenizedBuffer::Token open_brace, bool /*is_single_item*/, - TokenizedBuffer::Token close_brace, bool has_errors) { - AddLeafNode(ParseNodeKind::StructEnd(), close_brace); - return AddNode(kind == Type ? ParseNodeKind::StructTypeLiteral() - : ParseNodeKind::StructLiteral(), - open_brace, start, has_errors); - }, - /*allow_trailing_comma=*/true); -} - -auto ParseTree::Parser::ParsePrimaryExpression() -> llvm::Optional { - CARBON_RETURN_IF_STACK_LIMITED(llvm::None); - llvm::Optional kind; - switch (NextTokenKind()) { - case TokenKind::Identifier(): - kind = ParseNodeKind::NameReference(); - break; - - case TokenKind::IntegerLiteral(): - case TokenKind::RealLiteral(): - case TokenKind::StringLiteral(): - case TokenKind::IntegerTypeLiteral(): - case TokenKind::UnsignedIntegerTypeLiteral(): - case TokenKind::FloatingPointTypeLiteral(): - kind = ParseNodeKind::Literal(); - break; - - case TokenKind::OpenParen(): - return ParseParenExpression(); - - case TokenKind::OpenCurlyBrace(): - return ParseBraceExpression(); - - default: - CARBON_DIAGNOSTIC(ExpectedExpression, Error, "Expected expression."); - emitter_.Emit(*position_, ExpectedExpression); - return llvm::None; - } - - return AddLeafNode(*kind, Consume(NextTokenKind())); -} - -auto ParseTree::Parser::ParseDesignatorExpression(SubtreeStart start, - ParseNodeKind kind, - bool has_errors) - -> llvm::Optional { - // `.` identifier - auto dot = Consume(TokenKind::Period()); - auto name = ConsumeIf(TokenKind::Identifier()); - if (name) { - AddLeafNode(ParseNodeKind::DesignatedName(), *name); - } else { - CARBON_DIAGNOSTIC(ExpectedIdentifierAfterDot, Error, - "Expected identifier after `.`."); - emitter_.Emit(*position_, ExpectedIdentifierAfterDot); - // If we see a keyword, assume it was intended to be the designated name. - // TODO: Should keywords be valid in designators? - if (NextTokenKind().IsKeyword()) { - name = Consume(NextTokenKind()); - auto name_node = AddLeafNode(ParseNodeKind::DesignatedName(), *name); - MarkNodeError(name_node); - } else { - has_errors = true; - } - } - - Node result = AddNode(kind, dot, start, has_errors); - return name ? result : llvm::Optional(); -} - -auto ParseTree::Parser::ParseCallExpression(SubtreeStart start, bool has_errors) - -> llvm::Optional { - CARBON_RETURN_IF_STACK_LIMITED(llvm::None); - // `(` expression-list[opt] `)` - // - // expression-list ::= expression - // ::= expression `,` expression-list - return ParseParenList( - [&] { return ParseExpression(); }, ParseNodeKind::CallExpressionComma(), - [&](TokenizedBuffer::Token open_paren, bool /*is_single_item*/, - TokenizedBuffer::Token close_paren, bool has_arg_errors) { - AddLeafNode(ParseNodeKind::CallExpressionEnd(), close_paren); - return AddNode(ParseNodeKind::CallExpression(), open_paren, start, - has_errors || has_arg_errors); - }); -} - -auto ParseTree::Parser::ParsePostfixExpression() -> llvm::Optional { - CARBON_RETURN_IF_STACK_LIMITED(llvm::None); - auto start = GetSubtreeStartPosition(); - llvm::Optional expression = ParsePrimaryExpression(); - - TokenizedBuffer::TokenIterator last_position = position_; - while (true) { - switch (NextTokenKind()) { - case TokenKind::Period(): - expression = ParseDesignatorExpression( - start, ParseNodeKind::DesignatorExpression(), !expression); - break; - - case TokenKind::OpenParen(): - expression = ParseCallExpression(start, !expression); - break; - - default: - return expression; - } - // This is subject to an infinite loop if a child call fails, so monitor - // for stalling. - if (last_position == position_) { - CARBON_CHECK(expression == llvm::None); - return expression; - } - last_position = position_; - } -} - -// Determines whether the given token is considered to be the start of an -// operand according to the rules for infix operator parsing. -static auto IsAssumedStartOfOperand(TokenKind kind) -> bool { - return kind.IsOneOf({TokenKind::OpenParen(), TokenKind::Identifier(), - TokenKind::IntegerLiteral(), TokenKind::RealLiteral(), - TokenKind::StringLiteral()}); -} - -// Determines whether the given token is considered to be the end of an -// operand according to the rules for infix operator parsing. -static auto IsAssumedEndOfOperand(TokenKind kind) -> bool { - return kind.IsOneOf({TokenKind::CloseParen(), TokenKind::CloseCurlyBrace(), - TokenKind::CloseSquareBracket(), TokenKind::Identifier(), - TokenKind::IntegerLiteral(), TokenKind::RealLiteral(), - TokenKind::StringLiteral()}); -} - -// Determines whether the given token could possibly be the start of an -// operand. This is conservatively correct, and will never incorrectly return -// `false`, but can incorrectly return `true`. -static auto IsPossibleStartOfOperand(TokenKind kind) -> bool { - return !kind.IsOneOf({TokenKind::CloseParen(), TokenKind::CloseCurlyBrace(), - TokenKind::CloseSquareBracket(), TokenKind::Comma(), - TokenKind::Semi(), TokenKind::Colon()}); -} - -auto ParseTree::Parser::IsLexicallyValidInfixOperator() -> bool { - CARBON_CHECK(!AtEndOfFile()) << "Expected an operator token."; - - bool leading_space = tokens_.HasLeadingWhitespace(*position_); - bool trailing_space = tokens_.HasTrailingWhitespace(*position_); - - // If there's whitespace on both sides, it's an infix operator. - if (leading_space && trailing_space) { - return true; - } - - // If there's whitespace on exactly one side, it's not an infix operator. - if (leading_space || trailing_space) { - return false; - } - - // Otherwise, for an infix operator, the preceding token must be any close - // bracket, identifier, or literal and the next token must be an open paren, - // identifier, or literal. - if (position_ == tokens_.tokens().begin() || - !IsAssumedEndOfOperand(tokens_.GetKind(*(position_ - 1))) || - !IsAssumedStartOfOperand(tokens_.GetKind(*(position_ + 1)))) { - return false; - } - - return true; -} - -auto ParseTree::Parser::DiagnoseOperatorFixity(OperatorFixity fixity) -> void { - bool is_valid_as_infix = IsLexicallyValidInfixOperator(); - - if (fixity == OperatorFixity::Infix) { - // Infix operators must satisfy the infix operator rules. - if (!is_valid_as_infix) { - CARBON_DIAGNOSTIC(BinaryOperatorRequiresWhitespace, Error, - "Whitespace missing {0} binary operator.", - RelativeLocation); - emitter_.Emit(*position_, BinaryOperatorRequiresWhitespace, - tokens_.HasLeadingWhitespace(*position_) - ? RelativeLocation::After - : (tokens_.HasTrailingWhitespace(*position_) - ? RelativeLocation::Before - : RelativeLocation::Around)); - } - } else { - bool prefix = fixity == OperatorFixity::Prefix; - - // Whitespace is not permitted between a symbolic pre/postfix operator and - // its operand. - if (NextTokenKind().IsSymbol() && - (prefix ? tokens_.HasTrailingWhitespace(*position_) - : tokens_.HasLeadingWhitespace(*position_))) { - CARBON_DIAGNOSTIC(UnaryOperatorHasWhitespace, Error, - "Whitespace is not allowed {0} this unary operator.", - RelativeLocation); - emitter_.Emit( - *position_, UnaryOperatorHasWhitespace, - prefix ? RelativeLocation::After : RelativeLocation::Before); - } - // Pre/postfix operators must not satisfy the infix operator rules. - if (is_valid_as_infix) { - CARBON_DIAGNOSTIC(UnaryOperatorRequiresWhitespace, Error, - "Whitespace is required {0} this unary operator.", - RelativeLocation); - emitter_.Emit( - *position_, UnaryOperatorRequiresWhitespace, - prefix ? RelativeLocation::Before : RelativeLocation::After); - } - } -} - -auto ParseTree::Parser::IsTrailingOperatorInfix() -> bool { - if (AtEndOfFile()) { - return false; - } - - // An operator that follows the infix operator rules is parsed as - // infix, unless the next token means that it can't possibly be. - if (IsLexicallyValidInfixOperator() && - IsPossibleStartOfOperand(tokens_.GetKind(*(position_ + 1)))) { - return true; - } - - // A trailing operator with leading whitespace that's not valid as infix is - // not valid at all. If the next token looks like the start of an operand, - // then parse as infix, otherwise as postfix. Either way we'll produce a - // diagnostic later on. - if (tokens_.HasLeadingWhitespace(*position_) && - IsAssumedStartOfOperand(tokens_.GetKind(*(position_ + 1)))) { - return true; - } - - return false; -} - -auto ParseTree::Parser::ParseOperatorExpression( - PrecedenceGroup ambient_precedence) -> llvm::Optional { - // May be omitted a couple different ways here. - CARBON_DIAGNOSTIC( - OperatorRequiresParentheses, Error, - "Parentheses are required to disambiguate operator precedence."); - - CARBON_RETURN_IF_STACK_LIMITED(llvm::None); - auto start = GetSubtreeStartPosition(); - - llvm::Optional lhs; - PrecedenceGroup lhs_precedence = PrecedenceGroup::ForPostfixExpression(); - - // Check for a prefix operator. - if (auto operator_precedence = PrecedenceGroup::ForLeading(NextTokenKind()); - !operator_precedence) { - lhs = ParsePostfixExpression(); - } else { - if (PrecedenceGroup::GetPriority(ambient_precedence, - *operator_precedence) != - OperatorPriority::RightFirst) { - // The precedence rules don't permit this prefix operator in this - // context. Diagnose this, but carry on and parse it anyway. - emitter_.Emit(*position_, OperatorRequiresParentheses); - } else { - // Check that this operator follows the proper whitespace rules. - DiagnoseOperatorFixity(OperatorFixity::Prefix); - } - - auto operator_token = Consume(NextTokenKind()); - bool has_errors = !ParseOperatorExpression(*operator_precedence); - lhs = AddNode(ParseNodeKind::PrefixOperator(), operator_token, start, - has_errors); - lhs_precedence = *operator_precedence; - } - - // Consume a sequence of infix and postfix operators. - while (auto trailing_operator = PrecedenceGroup::ForTrailing( - NextTokenKind(), IsTrailingOperatorInfix())) { - auto [operator_precedence, is_binary] = *trailing_operator; - - // TODO: If this operator is ambiguous with either the ambient precedence - // or the LHS precedence, and there's a variant with a different fixity - // that would work, use that one instead for error recovery. - if (PrecedenceGroup::GetPriority(ambient_precedence, operator_precedence) != - OperatorPriority::RightFirst) { - // The precedence rules don't permit this operator in this context. Try - // again in the enclosing expression context. - return lhs; - } - - if (PrecedenceGroup::GetPriority(lhs_precedence, operator_precedence) != - OperatorPriority::LeftFirst) { - // Either the LHS operator and this operator are ambiguous, or the - // LHS operator is a unary operator that can't be nested within - // this operator. Either way, parentheses are required. - emitter_.Emit(*position_, OperatorRequiresParentheses); - lhs = llvm::None; - } else { - DiagnoseOperatorFixity(is_binary ? OperatorFixity::Infix - : OperatorFixity::Postfix); - } - - auto operator_token = Consume(NextTokenKind()); - - if (is_binary) { - auto rhs = ParseOperatorExpression(operator_precedence); - lhs = AddNode(ParseNodeKind::InfixOperator(), operator_token, start, - /*has_error=*/!lhs || !rhs); - } else { - lhs = AddNode(ParseNodeKind::PostfixOperator(), operator_token, start, - /*has_error=*/!lhs); - } - lhs_precedence = operator_precedence; - } - - return lhs; -} - -auto ParseTree::Parser::ParseExpression() -> llvm::Optional { - CARBON_RETURN_IF_STACK_LIMITED(llvm::None); - return ParseOperatorExpression(PrecedenceGroup::ForTopLevelExpression()); -} - -auto ParseTree::Parser::ParseType() -> llvm::Optional { - CARBON_RETURN_IF_STACK_LIMITED(llvm::None); - return ParseOperatorExpression(PrecedenceGroup::ForType()); -} - -auto ParseTree::Parser::ParseExpressionStatement() -> llvm::Optional { - CARBON_RETURN_IF_STACK_LIMITED(llvm::None); - TokenizedBuffer::Token start_token = *position_; - auto start = GetSubtreeStartPosition(); - - bool has_errors = !ParseExpression(); - - if (auto semi = ConsumeIf(TokenKind::Semi())) { - return AddNode(ParseNodeKind::ExpressionStatement(), *semi, start, - has_errors); - } - - if (!has_errors) { - emitter_.Emit(*position_, ExpectedSemiAfterExpression); - } - - if (auto semi_token = SkipPastLikelyEnd(start_token)) { - return AddNode(ParseNodeKind::ExpressionStatement(), *semi_token, start, - /*has_error=*/true); - } - - // Found junk not even followed by a `;`. - return llvm::None; -} - -auto ParseTree::Parser::ParseParenCondition(TokenKind introducer) - -> llvm::Optional { - CARBON_RETURN_IF_STACK_LIMITED(llvm::None); - // `(` expression `)` - auto start = GetSubtreeStartPosition(); - auto open_paren = ConsumeIf(TokenKind::OpenParen()); - if (!open_paren) { - CARBON_DIAGNOSTIC(ExpectedParenAfter, Error, "Expected `(` after `{0}`.", - TokenKind); - emitter_.Emit(*position_, ExpectedParenAfter, introducer); - } - - auto expr = ParseExpression(); - - if (!open_paren) { - // Don't expect a matching closing paren if there wasn't an opening paren. - return llvm::None; - } - - auto close_paren = - ParseCloseParen(*open_paren, ParseNodeKind::ConditionEnd()); - - return AddNode(ParseNodeKind::Condition(), *open_paren, start, - /*has_error=*/!expr || !close_paren); -} - -auto ParseTree::Parser::ParseIfStatement() -> llvm::Optional { - auto start = GetSubtreeStartPosition(); - auto if_token = Consume(TokenKind::If()); - auto cond = ParseParenCondition(TokenKind::If()); - auto then_case = ParseCodeBlock(); - bool else_has_errors = false; - if (ConsumeAndAddLeafNodeIf(TokenKind::Else(), - ParseNodeKind::IfStatementElse())) { - // 'else if' is permitted as a special case. - if (NextTokenIs(TokenKind::If())) { - else_has_errors = !ParseIfStatement(); - } else { - else_has_errors = !ParseCodeBlock(); - } - } - return AddNode(ParseNodeKind::IfStatement(), if_token, start, - /*has_error=*/!cond || !then_case || else_has_errors); -} - -auto ParseTree::Parser::ParseWhileStatement() -> llvm::Optional { - CARBON_RETURN_IF_STACK_LIMITED(llvm::None); - auto start = GetSubtreeStartPosition(); - auto while_token = Consume(TokenKind::While()); - auto cond = ParseParenCondition(TokenKind::While()); - auto body = ParseCodeBlock(); - return AddNode(ParseNodeKind::WhileStatement(), while_token, start, - /*has_error=*/!cond || !body); -} - -auto ParseTree::Parser::ParseForStatement() -> llvm::Optional { - CARBON_RETURN_IF_STACK_LIMITED(llvm::None); - auto for_start = GetSubtreeStartPosition(); - auto for_token = Consume(TokenKind::For()); - - // Parse the loop header. - auto header = [this]() -> llvm::Optional { - auto open_paren = ConsumeIf(TokenKind::OpenParen()); - auto header_start = GetSubtreeStartPosition(); - - if (!open_paren) { - CARBON_DIAGNOSTIC(ExpectedParenAfter, Error, - "Expected `(` after `{0}`. Recovering from missing `(` " - "not implemented yet!", - TokenKind); - emitter_.Emit(*position_, ExpectedParenAfter, TokenKind::For()); - // TODO: A proper recovery strategy is needed here. For now, I assume - // that all brackets are properly balanced (i.e. each open bracket has a - // closing one). - // This is temporary until we come to a conclusion regarding the - // recovery tokens strategy. - return llvm::None; - } - - bool iter_var_parsed = false; - - if (NextTokenIs(TokenKind::Var())) { - auto var_token = Consume(TokenKind::Var()); - auto var_start = GetSubtreeStartPosition(); - auto pattern = ParsePattern(PatternKind::Variable); - AddNode(ParseNodeKind::VariableDeclaration(), var_token, var_start, - !pattern); - iter_var_parsed = true; - } else { - CARBON_DIAGNOSTIC(ExpectedVariableDeclaration, Error, - "Expected `var` declaration."); - emitter_.Emit(*position_, ExpectedVariableDeclaration); - - if (auto next_in = FindNextOf({TokenKind::In()}); next_in) { - SkipTo(*next_in); - } - } - - // A separator is either an `in` or a `:`. Even though `:` is incorrect, - // accidentally typing it by a C++ programmer might be a common mistake - // that warrants special handling. - bool separator_parsed = false; - bool in_parsed = false; - - if (NextTokenIs(TokenKind::In())) { - separator_parsed = true; - in_parsed = true; - AddLeafNode(ParseNodeKind::ForIn(), Consume(TokenKind::In())); - } else if (NextTokenIs(TokenKind::Colon())) { - separator_parsed = true; - CARBON_DIAGNOSTIC(ExpectedIn, Error, "`:` should be replaced by `in`."); - emitter_.Emit(*position_, ExpectedIn); - Consume(TokenKind::Colon()); - } else { - CARBON_DIAGNOSTIC(ExpectedIn, Error, - "Expected `in` after loop `var` declaration."); - emitter_.Emit(*position_, ExpectedIn); - SkipTo(tokens_.GetMatchedClosingToken(*open_paren)); - } - - // Only try to parse the container expression if a separator was parsed. - // This reduces the emitted error messages if the separator was missing - // altogether. - auto container_expr = separator_parsed ? ParseExpression() : llvm::None; - - auto close_paren = - ParseCloseParen(*open_paren, ParseNodeKind::ForHeaderEnd()); - - return AddNode( - ParseNodeKind::ForHeader(), *open_paren, header_start, - !iter_var_parsed || !in_parsed || !container_expr || !close_paren); - }(); - - auto body = ParseCodeBlock(); - - return AddNode(ParseNodeKind::ForStatement(), for_token, for_start, - !header || !body); -} - -auto ParseTree::Parser::ParseKeywordStatement(ParseNodeKind kind, - KeywordStatementArgument argument) - -> llvm::Optional { - CARBON_RETURN_IF_STACK_LIMITED(llvm::None); - auto keyword_kind = NextTokenKind(); - CARBON_CHECK(keyword_kind.IsKeyword()); - - auto start = GetSubtreeStartPosition(); - auto keyword = Consume(keyword_kind); - - bool arg_error = false; - if ((argument == KeywordStatementArgument::Optional && - NextTokenKind() != TokenKind::Semi()) || - argument == KeywordStatementArgument::Mandatory) { - arg_error = !ParseExpression(); - } - - auto semi = - ConsumeAndAddLeafNodeIf(TokenKind::Semi(), ParseNodeKind::StatementEnd()); - if (!semi) { - CARBON_DIAGNOSTIC(ExpectedSemiAfter, Error, "Expected `;` after `{0}`.", - TokenKind); - emitter_.Emit(*position_, ExpectedSemiAfter, keyword_kind); - // TODO: Try to skip to a semicolon to recover. - } - return AddNode(kind, keyword, start, /*has_error=*/!semi || arg_error); -} - -auto ParseTree::Parser::ParseStatement() -> llvm::Optional { - CARBON_RETURN_IF_STACK_LIMITED(llvm::None); - switch (NextTokenKind()) { - case TokenKind::Var(): - return ParseVariableDeclaration(); - - case TokenKind::If(): - return ParseIfStatement(); - - case TokenKind::While(): - return ParseWhileStatement(); - - case TokenKind::For(): - return ParseForStatement(); - - case TokenKind::Continue(): - return ParseKeywordStatement(ParseNodeKind::ContinueStatement(), - KeywordStatementArgument::None); - - case TokenKind::Break(): - return ParseKeywordStatement(ParseNodeKind::BreakStatement(), - KeywordStatementArgument::None); - - case TokenKind::Return(): - return ParseKeywordStatement(ParseNodeKind::ReturnStatement(), - KeywordStatementArgument::Optional); - - default: - // A statement with no introducer token can only be an expression - // statement. - return ParseExpressionStatement(); - } -} - -} // namespace Carbon diff --git a/toolchain/parser/parser_impl.h b/toolchain/parser/parser_impl.h deleted file mode 100644 index 04cadefbd011..000000000000 --- a/toolchain/parser/parser_impl.h +++ /dev/null @@ -1,301 +0,0 @@ -// Part of the Carbon Language project, under the Apache License v2.0 with LLVM -// Exceptions. See /LICENSE for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -#ifndef CARBON_TOOLCHAIN_PARSER_PARSER_IMPL_H_ -#define CARBON_TOOLCHAIN_PARSER_PARSER_IMPL_H_ - -#include "llvm/ADT/Optional.h" -#include "toolchain/diagnostics/diagnostic_emitter.h" -#include "toolchain/lexer/token_kind.h" -#include "toolchain/lexer/tokenized_buffer.h" -#include "toolchain/parser/parse_node_kind.h" -#include "toolchain/parser/parse_tree.h" -#include "toolchain/parser/precedence.h" - -namespace Carbon { - -class ParseTree::Parser { - public: - // Parses the tokens into a parse tree, emitting any errors encountered. - // - // This is the entry point to the parser implementation. - static auto Parse(TokenizedBuffer& tokens, TokenDiagnosticEmitter& emitter) - -> ParseTree; - - private: - class ScopedStackStep; - struct SubtreeStart; - - explicit Parser(ParseTree& tree_arg, TokenizedBuffer& tokens_arg, - TokenDiagnosticEmitter& emitter); - - auto AtEndOfFile() -> bool { - return tokens_.GetKind(*position_) == TokenKind::EndOfFile(); - } - - // Gets the kind of the next token to be consumed. - [[nodiscard]] auto NextTokenKind() const -> TokenKind { - return tokens_.GetKind(*position_); - } - - // Tests whether the next token to be consumed is of the specified kind. - [[nodiscard]] auto NextTokenIs(TokenKind kind) const -> bool { - return NextTokenKind() == kind; - } - - // Tests whether the next token to be consumed is of any of the specified - // kinds. - [[nodiscard]] auto NextTokenIsOneOf( - std::initializer_list kinds) const -> bool { - return NextTokenKind().IsOneOf(kinds); - } - - // Requires (and asserts) that the current position matches the provided - // `Kind`. Returns the current token and advances to the next position. - auto Consume(TokenKind kind) -> TokenizedBuffer::Token; - - // If the current position's token matches this `Kind`, returns it and - // advances to the next position. Otherwise returns an empty optional. - auto ConsumeIf(TokenKind kind) -> llvm::Optional; - - // Adds a node to the parse tree that is fully parsed, has no children - // ("leaf"), and has a subsequent sibling. - // - // This sets up the next sibling of the node to be the next node in the parse - // tree's preorder sequence. - auto AddLeafNode(ParseNodeKind kind, TokenizedBuffer::Token token, - bool has_error = false) -> Node; - - // Composes `ConsumeIf` and `AddLeafNode`, propagating the failure case - // through the optional. - auto ConsumeAndAddLeafNodeIf(TokenKind t_kind, ParseNodeKind n_kind) - -> llvm::Optional; - - // Marks the node `n` as having some parse errors and that the tree contains - // a node with a parse error. - auto MarkNodeError(Node n) -> void; - - // Tracks the current location as a potential start of a subtree. - // - // This returns a marker representing the current position, which can later - // be used in a call to `AddNode` to mark all nodes created since this - // position as children of the added node. - auto GetSubtreeStartPosition() -> SubtreeStart; - - // Add a node to the parse tree that potentially has a subtree larger than - // itself. - // - // Requires a start marker be passed to compute the size of the subtree rooted - // at this node. - auto AddNode(ParseNodeKind n_kind, TokenizedBuffer::Token t, - SubtreeStart start, bool has_error = false) -> Node; - - // If the current token is an opening symbol for a matched group, skips - // forward to one past the matched closing symbol and returns true. Otherwise, - // returns false. - auto SkipMatchingGroup() -> bool; - - // Skip forward to the given token. - auto SkipTo(TokenizedBuffer::Token t) -> void; - - // Find the next token of any of the given kinds at the current bracketing - // level. - auto FindNextOf(std::initializer_list desired_kinds) - -> llvm::Optional; - - // Callback used if we find a semicolon when skipping to the end of a - // declaration or statement. - using SemiHandler = llvm::function_ref< - auto(TokenizedBuffer::Token semi)->llvm::Optional>; - - // Skips forward to move past the likely end of a declaration or statement. - // - // Looks forward, skipping over any matched symbol groups, to find the next - // position that is likely past the end of a declaration or statement. This - // is a heuristic and should only be called when skipping past parse errors. - // - // The strategy for recognizing when we have likely passed the end of a - // declaration or statement: - // - If we get to a close curly brace, we likely ended the entire context. - // - If we get to a semicolon, that should have ended the declaration or - // statement. - // - If we get to a new line from the `SkipRoot` token, but with the same or - // less indentation, there is likely a missing semicolon. Continued - // declarations or statements across multiple lines should be indented. - // - // If we find a semicolon based on this skipping, we return that token. - // Otherwise we will return an empty optional. - auto SkipPastLikelyEnd(TokenizedBuffer::Token skip_root) - -> llvm::Optional; - - // Parses a close paren token corresponding to the given open paren token, - // possibly skipping forward and diagnosing if necessary. Creates and returns - // a parse node of the specified kind if successful. - auto ParseCloseParen(TokenizedBuffer::Token open_paren, ParseNodeKind kind) - -> llvm::Optional; - - // Parses a comma-separated list with the given delimiters. - template - auto ParseList(TokenKind open, TokenKind close, - ListElementParser list_element_parser, - ParseNodeKind comma_kind, ListCompletionHandler list_handler, - bool allow_trailing_comma = false) -> llvm::Optional; - - // Parses a parenthesized, comma-separated list. - template - auto ParseParenList(ListElementParser list_element_parser, - ParseNodeKind comma_kind, - ListCompletionHandler list_handler, - bool allow_trailing_comma = false) - -> llvm::Optional { - return ParseList(TokenKind::OpenParen(), TokenKind::CloseParen(), - list_element_parser, comma_kind, list_handler, - allow_trailing_comma); - } - - // Parses a single function parameter declaration. - auto ParseFunctionParameter() -> llvm::Optional; - - // Parses the signature of the function, consisting of a parameter list and an - // optional return type. Returns the root node of the signature which must be - // based on the open parenthesis of the parameter list. - auto ParseFunctionSignature() -> bool; - - // Parses a block of code: `{ ... }`. - // - // These contain variable declarations and statements. - auto ParseCodeBlock() -> llvm::Optional; - - // Similar to ParseCodeBlock(), but supports different ParseNodeKinds because - // function definitions are represented differently from other code blocks. - // If subtree_start is before start_kind, earlier nodes will be treated as - // children of the start_kind node. - auto ParseCodeBlock(SubtreeStart subtree_start, ParseNodeKind start_kind, - ParseNodeKind end_kind) -> llvm::Optional; - - // Parses a function declaration with an optional definition. Returns the - // function parse node which is based on the `fn` introducer keyword. - auto ParseFunctionDeclaration() -> Node; - - // Parses a variable declaration with an optional initializer. - auto ParseVariableDeclaration() -> Node; - - // Parses and returns an empty declaration node from a single semicolon token. - auto ParseEmptyDeclaration() -> Node; - - // Parses a package directive. - auto ParsePackageDirective() -> Node; - - // Tries to parse a declaration. If a declaration, even an empty one after - // skipping errors, can be parsed, it is returned. There may be parse errors - // even when a node is returned. - auto ParseDeclaration() -> llvm::Optional; - - // Parses a parenthesized expression. - auto ParseParenExpression() -> llvm::Optional; - - // Parses a braced expression. - auto ParseBraceExpression() -> llvm::Optional; - - // Parses a primary expression, which is either a terminal portion of an - // expression tree, such as an identifier or literal, or a parenthesized - // expression. - auto ParsePrimaryExpression() -> llvm::Optional; - - // Parses a designator expression suffix starting with `.`. - auto ParseDesignatorExpression(SubtreeStart start, ParseNodeKind kind, - bool has_errors) -> llvm::Optional; - - // Parses a call expression suffix starting with `(`. - auto ParseCallExpression(SubtreeStart start, bool has_errors) - -> llvm::Optional; - - // Parses a postfix expression, which is a primary expression followed by - // zero or more of the following: - // - // - function applications - // - array indexes (TODO) - // - designators - auto ParsePostfixExpression() -> llvm::Optional; - - enum class OperatorFixity { Prefix, Infix, Postfix }; - - // Determines whether the current token satisfies the lexical validity rules - // for an infix operator. - auto IsLexicallyValidInfixOperator() -> bool; - - // Diagnoses if the current token is not written properly for the given - // fixity, for example because mandatory whitespace is missing. - auto DiagnoseOperatorFixity(OperatorFixity fixity) -> void; - - // Determines whether the current trailing operator should be treated as - // infix. - auto IsTrailingOperatorInfix() -> bool; - - // Parses an expression involving operators, in a context with the given - // precedence. - auto ParseOperatorExpression(PrecedenceGroup precedence) - -> llvm::Optional; - - // Parses an expression. - auto ParseExpression() -> llvm::Optional; - - // Parses a type expression. - auto ParseType() -> llvm::Optional; - - // Parses an expression statement: an expression followed by a semicolon. - auto ParseExpressionStatement() -> llvm::Optional; - - // Parses the parenthesized condition in an if-statement. - auto ParseParenCondition(TokenKind introducer) -> llvm::Optional; - - // Parses an if-statement. - auto ParseIfStatement() -> llvm::Optional; - - // Parses a while-statement. - auto ParseWhileStatement() -> llvm::Optional; - - // Parses a for-statement. - auto ParseForStatement() -> llvm::Optional; - - enum class KeywordStatementArgument { - None, - Optional, - Mandatory, - }; - - // Parses a statement of the form `keyword;` such as `break;` or `continue;`. - auto ParseKeywordStatement(ParseNodeKind kind, - KeywordStatementArgument argument) - -> llvm::Optional; - - // Parses a statement. - auto ParseStatement() -> llvm::Optional; - - enum class PatternKind { - Parameter, - Variable, - }; - - // Parses a pattern. - auto ParsePattern(PatternKind kind) -> llvm::Optional; - - ParseTree& tree_; - TokenizedBuffer& tokens_; - TokenDiagnosticEmitter& emitter_; - - // The current position within the token buffer. Never equal to `end`. - TokenizedBuffer::TokenIterator position_; - // The end position of the token buffer. There will always be an `EndOfFile` - // token between `position` (inclusive) and `end` (exclusive). - TokenizedBuffer::TokenIterator end_; - - // Managed through RETURN_IF_STACK_LIMITED, which should be invoked by all - // functions. - int stack_depth_ = 0; -}; - -} // namespace Carbon - -#endif // CARBON_TOOLCHAIN_PARSER_PARSER_IMPL_H_