diff --git a/toolchain/parse/handle_decl_scope_loop.cpp b/toolchain/parse/handle_decl_scope_loop.cpp index c3acf04c8719..427a22d9e107 100644 --- a/toolchain/parse/handle_decl_scope_loop.cpp +++ b/toolchain/parse/handle_decl_scope_loop.cpp @@ -2,11 +2,54 @@ // Exceptions. See /LICENSE for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +#include "toolchain/lex/token_kind.h" #include "toolchain/parse/context.h" +#include "toolchain/parse/node_kind.h" namespace Carbon::Parse { -static auto OutputInvalidParseSubtree(Context& context, int32_t subtree_start) +// Handles positions which are end of scope and packaging declarations. Returns +// true when either applies. When the position is neither, returns false, and +// may still update packaging state. +static auto TryHandleEndOrPackagingDecl(Context& context) -> bool { + switch (context.PositionKind()) { + case Lex::TokenKind::CloseCurlyBrace: + case Lex::TokenKind::FileEnd: { + // This is the end of the scope, so the loop state ends. + context.PopAndDiscardState(); + return true; + } + // `import`, `library`, and `package` manage their packaging state. + case Lex::TokenKind::Import: { + context.PushState(State::Import); + return true; + } + case Lex::TokenKind::Library: { + context.PushState(State::Library); + return true; + } + case Lex::TokenKind::Package: { + context.PushState(State::Package); + return true; + } + default: { + // Because a non-packaging keyword was encountered, packaging is complete. + // Misplaced packaging keywords may lead to this being re-triggered. + if (context.packaging_state() != + Context::PackagingState::AfterNonPackagingDecl) { + if (!context.first_non_packaging_token().is_valid()) { + context.set_first_non_packaging_token(*context.position()); + } + context.set_packaging_state( + Context::PackagingState::AfterNonPackagingDecl); + } + return false; + } + } +} + +// Finishes an invalid declaration, skipping past its end. +static auto FinishAndSkipInvalidDecl(Context& context, int32_t subtree_start) -> void { auto cursor = *context.position(); // Consume to the next `;` or end of line. We ignore the return value since @@ -25,34 +68,168 @@ static auto OutputInvalidParseSubtree(Context& context, int32_t subtree_start) /*has_error=*/true); } -// Handles an unrecognized declaration. +// Prints a diagnostic and calls FinishAndSkipInvalidDecl. static auto HandleUnrecognizedDecl(Context& context, int32_t subtree_start) -> void { CARBON_DIAGNOSTIC(UnrecognizedDecl, Error, "Unrecognized declaration introducer."); context.emitter().Emit(*context.position(), UnrecognizedDecl); - OutputInvalidParseSubtree(context, subtree_start); + FinishAndSkipInvalidDecl(context, subtree_start); } -static auto TokenIsModifierOrIntroducer(Lex::TokenKind token_kind) -> bool { - switch (token_kind) { - case Lex::TokenKind::Abstract: - case Lex::TokenKind::Base: - case Lex::TokenKind::Class: - case Lex::TokenKind::Constraint: - case Lex::TokenKind::Default: - case Lex::TokenKind::Extend: - case Lex::TokenKind::Final: - case Lex::TokenKind::Fn: - case Lex::TokenKind::Impl: - case Lex::TokenKind::Interface: - case Lex::TokenKind::Let: - case Lex::TokenKind::Namespace: - case Lex::TokenKind::Private: - case Lex::TokenKind::Protected: - case Lex::TokenKind::Var: - case Lex::TokenKind::Virtual: +// Handles `base` as a declaration. +static auto HandleBaseAsDecl(Context& context, Context::StateStackEntry state) + -> void { + // At this point, `base` has been ruled out as a modifier (`base class`). If + // it's followed by a colon, it's an introducer (`extend base: BaseType;`). + // Otherwise it's an error. + if (context.PositionIs(Lex::TokenKind::Colon, Lookahead::NextToken)) { + context.ReplacePlaceholderNode(state.subtree_start, + NodeKind::BaseIntroducer, context.Consume()); + // Reuse state here to retain its `subtree_start` + state.state = State::BaseDecl; + context.PushState(state); + context.PushState(State::Expr); + context.AddLeafNode(NodeKind::BaseColon, context.Consume()); + } else { + // TODO: If the next token isn't a colon or `class`, try to recover + // based on whether we're in a class, whether we have an `extend` + // modifier, and the following tokens. + context.AddLeafNode(NodeKind::InvalidParse, context.Consume(), + /*has_error=*/true); + CARBON_DIAGNOSTIC(ExpectedAfterBase, Error, + "`class` or `:` expected after `base`."); + context.emitter().Emit(*context.position(), ExpectedAfterBase); + FinishAndSkipInvalidDecl(context, state.subtree_start); + } +} + +// Returns true if the current position is a declaration. If we see a +// declaration introducer keyword token, replace the placeholder node and switch +// to a state to parse the rest of the declaration. +static auto TryHandleAsDecl(Context& context, Context::StateStackEntry state, + bool saw_modifier) -> bool { + auto introducer = [&](NodeKind node_kind, State next_state) { + context.ReplacePlaceholderNode(state.subtree_start, node_kind, + context.Consume()); + // Reuse state here to retain its `subtree_start`. + state.state = next_state; + context.PushState(state); + }; + + switch (context.PositionKind()) { + case Lex::TokenKind::Base: { + HandleBaseAsDecl(context, state); return true; + } + case Lex::TokenKind::Class: { + introducer(NodeKind::ClassIntroducer, State::TypeAfterIntroducerAsClass); + return true; + } + case Lex::TokenKind::Constraint: { + introducer(NodeKind::NamedConstraintIntroducer, + State::TypeAfterIntroducerAsNamedConstraint); + return true; + } + case Lex::TokenKind::Extend: { + // TODO: Treat this `extend` token as a declaration introducer + HandleUnrecognizedDecl(context, state.subtree_start); + return true; + } + case Lex::TokenKind::Fn: { + introducer(NodeKind::FunctionIntroducer, State::FunctionIntroducer); + return true; + } + case Lex::TokenKind::Impl: { + // TODO: Treat this `impl` token as a declaration introducer + HandleUnrecognizedDecl(context, state.subtree_start); + return true; + } + case Lex::TokenKind::Interface: { + introducer(NodeKind::InterfaceIntroducer, + State::TypeAfterIntroducerAsInterface); + return true; + } + case Lex::TokenKind::Namespace: { + introducer(NodeKind::NamespaceStart, State::Namespace); + return true; + } + case Lex::TokenKind::Let: { + introducer(NodeKind::LetIntroducer, State::Let); + return true; + } + case Lex::TokenKind::Var: { + introducer(NodeKind::VariableIntroducer, State::VarAsDecl); + return true; + } + + case Lex::TokenKind::Semi: { + if (saw_modifier) { + // Modifiers require an introducer keyword. + HandleUnrecognizedDecl(context, state.subtree_start); + } else { + context.ReplacePlaceholderNode(state.subtree_start, NodeKind::EmptyDecl, + context.Consume()); + } + return true; + } + + default: + return false; + } +} + +// Returns true if position_kind could be either an introducer or modifier, and +// should be treated as an introducer. +static auto ResolveAmbiguousTokenAsDeclaration(Context& context, + Lex::TokenKind position_kind) + -> bool { + switch (position_kind) { + case Lex::TokenKind::Base: + case Lex::TokenKind::Extend: + case Lex::TokenKind::Impl: + // This is an ambiguous token, so now we check what the next token is. + + // We use the macro for modifiers, including introducers which are + // also modifiers (such as `base`). Other introducer tokens need to be + // added by hand. + switch (context.PositionKind(Lookahead::NextToken)) { + case Lex::TokenKind::Class: + case Lex::TokenKind::Constraint: + case Lex::TokenKind::Fn: + case Lex::TokenKind::Interface: + case Lex::TokenKind::Let: + case Lex::TokenKind::Namespace: + case Lex::TokenKind::Var: +#define CARBON_PARSE_NODE_KIND(...) +#define CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER(Name, ...) \ + case Lex::TokenKind::Name: +#include "toolchain/parse/node_kind.def" + return false; + + default: + return true; + } + break; + default: + return false; + } +} + +// Returns true if the current position is a modifier, handling it if so. +static auto TryHandleAsModifier(Context& context) -> bool { + auto position_kind = context.PositionKind(); + if (ResolveAmbiguousTokenAsDeclaration(context, position_kind)) { + return false; + } + + switch (position_kind) { +#define CARBON_PARSE_NODE_KIND(...) +#define CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER(Name, ...) \ + case Lex::TokenKind::Name: \ + context.AddLeafNode(NodeKind::Name##Modifier, context.Consume()); \ + return true; +#include "toolchain/parse/node_kind.def" default: return false; @@ -62,45 +239,10 @@ static auto TokenIsModifierOrIntroducer(Lex::TokenKind token_kind) -> bool { auto HandleDeclScopeLoop(Context& context) -> void { // This maintains the current state unless we're at the end of the scope. - switch (context.PositionKind()) { - case Lex::TokenKind::CloseCurlyBrace: - case Lex::TokenKind::FileEnd: { - // This is the end of the scope, so the loop state ends. - context.PopAndDiscardState(); - return; - } - // `import`, `library`, and `package` manage their packaging state. - case Lex::TokenKind::Import: { - context.PushState(State::Import); - return; - } - case Lex::TokenKind::Library: { - context.PushState(State::Library); - return; - } - case Lex::TokenKind::Package: { - context.PushState(State::Package); - return; - } - default: { - break; - } + if (TryHandleEndOrPackagingDecl(context)) { + return; } - // Because a non-packaging keyword was encountered, packaging is complete. - // Misplaced packaging keywords may lead to this being re-triggered. - if (context.packaging_state() != - Context::PackagingState::AfterNonPackagingDecl) { - if (!context.first_non_packaging_token().is_valid()) { - context.set_first_non_packaging_token(*context.position()); - } - context.set_packaging_state(Context::PackagingState::AfterNonPackagingDecl); - } - - // Remaining keywords are only valid after imports are complete, and so all - // result in a `set_packaging_state` call. Note, this may not always be - // necessary but is probably cheaper than validating. - // Create a state with the correct starting position, with a dummy kind until // we see the declaration's introducer. context.PushState(State::DeclScopeLoop); @@ -110,165 +252,12 @@ auto HandleDeclScopeLoop(Context& context) -> void { // it is found. context.AddLeafNode(NodeKind::Placeholder, *context.position()); - auto introducer = [&](NodeKind node_kind, State next_state) { - context.ReplacePlaceholderNode(state.subtree_start, node_kind, - context.Consume()); - // Reuse state here to retain its `subtree_start` - state.state = next_state; - context.PushState(state); - }; - bool saw_modifier = false; - while (true) { - switch (context.PositionKind()) { - // If we see a modifier keyword token, add it as a leaf node and loop to - // the next token. - case Lex::TokenKind::Abstract: - context.AddLeafNode(NodeKind::AbstractModifier, context.Consume()); - saw_modifier = true; - break; - case Lex::TokenKind::Default: - context.AddLeafNode(NodeKind::DefaultModifier, context.Consume()); - saw_modifier = true; - break; - case Lex::TokenKind::Final: - context.AddLeafNode(NodeKind::FinalModifier, context.Consume()); - saw_modifier = true; - break; - case Lex::TokenKind::Private: - context.AddLeafNode(NodeKind::PrivateModifier, context.Consume()); - saw_modifier = true; - break; - case Lex::TokenKind::Protected: - context.AddLeafNode(NodeKind::ProtectedModifier, context.Consume()); - saw_modifier = true; - break; - case Lex::TokenKind::Virtual: - context.AddLeafNode(NodeKind::VirtualModifier, context.Consume()); - saw_modifier = true; - break; - - case Lex::TokenKind::Base: - // `base` may be followed by: - // - a colon - // => assume it is an introducer, as in `extend base: BaseType;`. - // - a modifier or an introducer - // => assume it is a modifier, as in `base class`; which is handled - // by falling through to the next case. - // Anything else is an error. - if (context.PositionIs(Lex::TokenKind::Colon, Lookahead::NextToken)) { - context.ReplacePlaceholderNode( - state.subtree_start, NodeKind::BaseIntroducer, context.Consume()); - // Reuse state here to retain its `subtree_start` - state.state = State::BaseDecl; - context.PushState(state); - context.PushState(State::Expr); - context.AddLeafNode(NodeKind::BaseColon, context.Consume()); - return; - } else if (!TokenIsModifierOrIntroducer( - context.PositionKind(Lookahead::NextToken))) { - // TODO: If the next token isn't a colon or `class`, try to recover - // based on whether we're in a class, whether we have an `extend` - // modifier, and the following tokens. - context.AddLeafNode(NodeKind::InvalidParse, context.Consume(), - /*has_error=*/true); - CARBON_DIAGNOSTIC(ExpectedAfterBase, Error, - "`class` or `:` expected after `base`."); - context.emitter().Emit(*context.position(), ExpectedAfterBase); - OutputInvalidParseSubtree(context, state.subtree_start); - return; - } - context.AddLeafNode(NodeKind::BaseModifier, context.Consume()); - saw_modifier = true; - break; - - case Lex::TokenKind::Impl: { - // `impl` is considered a declaration modifier if it is followed by - // another modifier or an introducer. - if (TokenIsModifierOrIntroducer( - context.PositionKind(Lookahead::NextToken))) { - context.AddLeafNode(NodeKind::ImplModifier, context.Consume()); - saw_modifier = true; - } else { - // TODO: Treat this `impl` token as a declaration introducer - HandleUnrecognizedDecl(context, state.subtree_start); - return; - } - break; - } - - case Lex::TokenKind::Extend: { - // `extend` is considered a declaration modifier if it is followed by - // another modifier or an introducer. - if (TokenIsModifierOrIntroducer( - context.PositionKind(Lookahead::NextToken))) { - context.AddLeafNode(NodeKind::ExtendModifier, context.Consume()); - saw_modifier = true; - } else { - // TODO: Treat this `extend` token as a declaration introducer - HandleUnrecognizedDecl(context, state.subtree_start); - return; - } - break; - } - - // If we see a declaration introducer keyword token, replace the - // placeholder node and switch to a state to parse the rest of the - // declaration. We don't allow namespace or empty declarations here since - // they can't have modifiers and don't use bracketing parse nodes that - // would allow a variable number of modifier nodes. - case Lex::TokenKind::Class: { - introducer(NodeKind::ClassIntroducer, - State::TypeAfterIntroducerAsClass); - return; - } - case Lex::TokenKind::Constraint: { - introducer(NodeKind::NamedConstraintIntroducer, - State::TypeAfterIntroducerAsNamedConstraint); - return; - } - case Lex::TokenKind::Fn: { - introducer(NodeKind::FunctionIntroducer, State::FunctionIntroducer); - return; - } - case Lex::TokenKind::Interface: { - introducer(NodeKind::InterfaceIntroducer, - State::TypeAfterIntroducerAsInterface); - return; - } - case Lex::TokenKind::Var: { - introducer(NodeKind::VariableIntroducer, State::VarAsDecl); - return; - } - case Lex::TokenKind::Let: { - introducer(NodeKind::LetIntroducer, State::Let); - return; - } - - // We don't allow namespace or empty declarations after a modifier since - // they can't have modifiers and don't use bracketing parse nodes that - // would allow a variable number of modifier nodes. - case Lex::TokenKind::Namespace: { - introducer(NodeKind::NamespaceStart, State::Namespace); - return; - } - case Lex::TokenKind::Semi: { - if (saw_modifier) { - HandleUnrecognizedDecl(context, state.subtree_start); - } else { - context.ReplacePlaceholderNode( - state.subtree_start, NodeKind::EmptyDecl, context.Consume()); - } - return; - } - - default: { - // For anything else, report an error and output an invalid parse node - // or subtree. - HandleUnrecognizedDecl(context, state.subtree_start); - return; - } - } + while (TryHandleAsModifier(context)) { + saw_modifier = true; + } + if (!TryHandleAsDecl(context, state, saw_modifier)) { + HandleUnrecognizedDecl(context, state.subtree_start); } }