diff --git a/toolchain/lex/BUILD b/toolchain/lex/BUILD index 35ef519535b6..7480e652d024 100644 --- a/toolchain/lex/BUILD +++ b/toolchain/lex/BUILD @@ -199,7 +199,10 @@ cc_library( cc_library( name = "token_index", hdrs = ["token_index.h"], - deps = ["//toolchain/base:index_base"], + deps = [ + ":token_kind", + "//toolchain/base:index_base", + ], ) cc_library( diff --git a/toolchain/lex/token_index.h b/toolchain/lex/token_index.h index 7588383b598d..ef131a27d90d 100644 --- a/toolchain/lex/token_index.h +++ b/toolchain/lex/token_index.h @@ -6,6 +6,7 @@ #define CARBON_TOOLCHAIN_LEX_TOKEN_INDEX_H_ #include "toolchain/base/index_base.h" +#include "toolchain/lex/token_kind.h" namespace Carbon::Lex { @@ -32,6 +33,21 @@ struct TokenIndex : public IndexBase { constexpr TokenIndex TokenIndex::Invalid(TokenIndex::InvalidIndex); constexpr TokenIndex TokenIndex::FirstNonCommentToken(1); +// A lightweight handle to a lexed token in a `TokenizedBuffer` whose kind is +// known to be `Kind`. +template +struct TokenIndexForKind : public TokenIndex { + // NOLINTNEXTLINE(readability-identifier-naming) + static const TokenKind& Kind; + constexpr explicit TokenIndexForKind(TokenIndex index) : TokenIndex(index) {} +}; +template +const TokenKind& TokenIndexForKind::Kind = K; + +#define CARBON_TOKEN(TokenName) \ + using TokenName##TokenIndex = TokenIndexForKind; +#include "toolchain/lex/token_kind.def" + } // namespace Carbon::Lex #endif // CARBON_TOOLCHAIN_LEX_TOKEN_INDEX_H_ diff --git a/toolchain/parse/extract.cpp b/toolchain/parse/extract.cpp index 364ac97fa3b4..2b7c2fac4779 100644 --- a/toolchain/parse/extract.cpp +++ b/toolchain/parse/extract.cpp @@ -330,10 +330,11 @@ auto NodeExtractor::MatchesTokenKind(Lex::TokenKind expected_kind) const // Extract the token corresponding to a node. template -struct Extractable> { - static auto Extract(NodeExtractor& extractor) -> std::optional> { +struct Extractable> { + static auto Extract(NodeExtractor& extractor) + -> std::optional> { if (extractor.MatchesTokenKind(Kind)) { - return Token{.index = extractor.token()}; + return static_cast>(extractor.token()); } else { return std::nullopt; } @@ -342,15 +343,16 @@ struct Extractable> { // Extract the token corresponding to a node. template <> -struct Extractable { - static auto Extract(NodeExtractor& extractor) -> std::optional { +struct Extractable { + static auto Extract(NodeExtractor& extractor) + -> std::optional { if (!extractor.has_token()) { if (auto* trace = extractor.trace()) { *trace << "Token expected but processing root node\n"; } return std::nullopt; } - return AnyToken{.index = extractor.token()}; + return extractor.token(); } }; diff --git a/toolchain/parse/node_ids.h b/toolchain/parse/node_ids.h index 14f915974149..3aaf3754ce78 100644 --- a/toolchain/parse/node_ids.h +++ b/toolchain/parse/node_ids.h @@ -111,32 +111,6 @@ struct NodeIdNot : public NodeId { : NodeId(NodeId::InvalidIndex) {} }; -// This class holds the token corresponding to a parse node, and defines the -// expected token kind. The specified token index will always have kind K if the -// enclosing node doesn't have errors. Note that we never try to extract a node -// that has errors, and there are no restrictions on the token kind in that -// case. -template -struct Token { - static constexpr const Lex::TokenKind& Kind = K; - Lex::TokenIndex index; -}; - -// This class holds the token corresponding to a parse node in the case where -// the parse node can correspond to any token. This should only be used when the -// node kind is either not used in a finished tree, such as `Placeholder`, or is -// always invalid, such as `InvalidParse`. -struct AnyToken { - Lex::TokenIndex index; -}; - -// This class holds the token corresponding to a parse node in the case of a -// virtual token. The parse node doesn't actually own the token in this case. -template -struct VirtualToken { - Token token; -}; - // Note that the support for extracting these types using the `Tree::Extract*` // functions is defined in `extract.cpp`. diff --git a/toolchain/parse/typed_nodes.h b/toolchain/parse/typed_nodes.h index 66cc5e6a2dfd..41a342e3bc81 100644 --- a/toolchain/parse/typed_nodes.h +++ b/toolchain/parse/typed_nodes.h @@ -7,7 +7,7 @@ #include -#include "toolchain/lex/token_kind.h" +#include "toolchain/lex/token_index.h" #include "toolchain/parse/node_ids.h" #include "toolchain/parse/node_kind.h" @@ -72,6 +72,17 @@ struct LeafNode { // - Any provided `Aggregate` type that is a simple aggregate type such as // `struct Aggregate { T x; U y; }`, // to match children with types `T` and `U`. +// +// In addition to the fields describing the child nodes, each parse node should +// also have exactly one field that describes the token corresponding to the +// parse node itself. This field should have the name `token`. The type of the +// field should be `Lex::*TokenIndex`, describing the kind of the token, such as +// `Lex::SemiTokenIndex` for a `;` token. If the parse node can correspond to +// any kind of token, `Lex::TokenIndex` can be used instead, but should only be +// used when the node kind is either not used in a finished tree, such as +// `Placeholder`, or is always invalid, such as `InvalidParse`. The location of +// the field relative to the child nodes indicates the location within the +// corresponding grammar production where the token appears. // ---------------------------------------------------------------------------- // Error nodes @@ -80,11 +91,12 @@ struct LeafNode { // An invalid parse. Used to balance the parse tree. This type is here only to // ensure we have a type for each parse node kind. This node kind always has an // error, so can never be extracted. -using InvalidParse = LeafNode; // An invalid subtree. Always has an error so can never be extracted. -using InvalidParseStart = LeafNode; +using InvalidParseStart = + LeafNode; struct InvalidParseSubtree { static constexpr auto Kind = NodeKind::InvalidParseSubtree.Define( {.category = NodeCategory::Decl, @@ -92,56 +104,55 @@ struct InvalidParseSubtree { InvalidParseStartId start; llvm::SmallVector> extra; - AnyToken token; + Lex::TokenIndex token; }; // A placeholder node to be replaced; it will never exist in a valid parse tree. // Its token kind is not enforced even when valid. -using Placeholder = LeafNode; +using Placeholder = LeafNode; // File nodes // ---------- // The start of the file. -using FileStart = - LeafNode>; +using FileStart = LeafNode; // The end of the file. -using FileEnd = LeafNode>; +using FileEnd = LeafNode; // General-purpose nodes // --------------------- // An empty declaration, such as `;`. -using EmptyDecl = LeafNode, +using EmptyDecl = LeafNode; // A name in a non-expression context, such as a declaration. using IdentifierName = - LeafNode, + LeafNode; // A name in an expression context. using IdentifierNameExpr = - LeafNode, + LeafNode; // The `self` value and `Self` type identifier keywords. Typically of the form // `self: Self`. -using SelfValueName = LeafNode>; +using SelfValueName = + LeafNode; using SelfValueNameExpr = - LeafNode, NodeCategory::Expr>; + LeafNode; using SelfTypeNameExpr = - LeafNode, NodeCategory::Expr>; + LeafNode; // The `base` value keyword, introduced by `base: B`. Typically referenced in // an expression, as in `x.base` or `{.base = ...}`, but can also be used as a // declared name, as in `{.base: partial B}`. -using BaseName = LeafNode, - NodeCategory::MemberName>; +using BaseName = + LeafNode; // An unqualified name and optionally a following sequence of parameters. // For example, `A`, `A(n: i32)`, or `A[T:! type](n: T)`. @@ -157,7 +168,7 @@ struct NameQualifier { NodeKind::NameQualifier.Define({.bracketed_by = IdentifierName::Kind}); NameAndParams name_and_params; - Token token; + Lex::PeriodTokenIndex token; }; // A complete name in a declaration: `A.C(T:! type).F(n: i32)`. @@ -172,26 +183,24 @@ struct DeclName { // The `package` keyword in an expression. using PackageExpr = - LeafNode, - NodeCategory::Expr>; + LeafNode; // The name of a package or library for `package`, `import`, and `library`. -using PackageName = - LeafNode>; +using PackageName = LeafNode; using LibraryName = - LeafNode>; + LeafNode; using DefaultLibrary = - LeafNode>; + LeafNode; using PackageIntroducer = - LeafNode>; + LeafNode; // `library` in `package` or `import`. struct LibrarySpecifier { static constexpr auto Kind = NodeKind::LibrarySpecifier.Define({.child_count = 1}); - Token token; + Lex::LibraryTokenIndex token; NodeIdOneOf name; }; @@ -206,12 +215,12 @@ struct PackageDecl { llvm::SmallVector modifiers; std::optional name; std::optional library; - Token token; + Lex::SemiTokenIndex token; }; // `import TheirPackage library "TheirLibrary";` using ImportIntroducer = - LeafNode>; + LeafNode; struct ImportDecl { static constexpr auto Kind = NodeKind::ImportDecl.Define( {.category = NodeCategory::Decl, .bracketed_by = ImportIntroducer::Kind}); @@ -220,12 +229,12 @@ struct ImportDecl { llvm::SmallVector modifiers; std::optional name; std::optional library; - Token token; + Lex::SemiTokenIndex token; }; // `library` as declaration. using LibraryIntroducer = - LeafNode>; + LeafNode; struct LibraryDecl { static constexpr auto Kind = NodeKind::LibraryDecl.Define({.category = NodeCategory::Decl, @@ -234,12 +243,12 @@ struct LibraryDecl { LibraryIntroducerId introducer; llvm::SmallVector modifiers; NodeIdOneOf library_name; - Token token; + Lex::SemiTokenIndex token; }; // `export` as a declaration. using ExportIntroducer = - LeafNode>; + LeafNode; struct ExportDecl { static constexpr auto Kind = NodeKind::ExportDecl.Define( {.category = NodeCategory::Decl, .bracketed_by = ExportIntroducer::Kind}); @@ -247,14 +256,14 @@ struct ExportDecl { ExportIntroducerId introducer; llvm::SmallVector modifiers; DeclName name; - Token token; + Lex::SemiTokenIndex token; }; // Namespace nodes // --------------- using NamespaceStart = - LeafNode>; + LeafNode; // A namespace: `namespace N;`. struct Namespace { @@ -264,7 +273,7 @@ struct Namespace { NamespaceStartId introducer; llvm::SmallVector modifiers; DeclName name; - Token token; + Lex::SemiTokenIndex token; }; // Pattern nodes @@ -276,7 +285,7 @@ struct BindingPattern { {.category = NodeCategory::Pattern, .child_count = 2}); NodeIdOneOf name; - Token token; + Lex::ColonTokenIndex token; AnyExprId type; }; @@ -286,7 +295,7 @@ struct CompileTimeBindingPattern { {.category = NodeCategory::Pattern, .child_count = 2}); NodeIdOneOf name; - Token token; + Lex::ColonExclaimTokenIndex token; AnyExprId type; }; @@ -295,7 +304,7 @@ struct Addr { static constexpr auto Kind = NodeKind::Addr.Define( {.category = NodeCategory::Pattern, .child_count = 1}); - Token token; + Lex::AddrTokenIndex token; AnyPatternId inner; }; @@ -304,16 +313,16 @@ struct Template { static constexpr auto Kind = NodeKind::Template.Define( {.category = NodeCategory::Pattern, .child_count = 1}); - Token token; + Lex::TemplateTokenIndex token; // This is a CompileTimeBindingPatternId in any valid program. // TODO: Should the parser enforce that? AnyPatternId inner; }; using TuplePatternStart = - LeafNode>; + LeafNode; using PatternListComma = - LeafNode>; + LeafNode; // A parameter list or tuple pattern: `(a: i32, b: i32)`. struct TuplePattern { @@ -323,12 +332,11 @@ struct TuplePattern { TuplePatternStartId left_paren; CommaSeparatedList params; - Token token; + Lex::CloseParenTokenIndex token; }; -using ImplicitParamListStart = - LeafNode>; +using ImplicitParamListStart = LeafNode; // An implicit parameter list: `[T:! type, self: Self]`. struct ImplicitParamList { @@ -337,20 +345,20 @@ struct ImplicitParamList { ImplicitParamListStartId left_square; CommaSeparatedList params; - Token token; + Lex::CloseSquareBracketTokenIndex token; }; // Function nodes // -------------- using FunctionIntroducer = - LeafNode>; + LeafNode; // A return type: `-> i32`. struct ReturnType { static constexpr auto Kind = NodeKind::ReturnType.Define({.child_count = 1}); - Token token; + Lex::MinusGreaterTokenIndex token; AnyExprId type; }; @@ -368,13 +376,11 @@ struct FunctionSignature { TokenKind token; }; -using FunctionDecl = - FunctionSignature, - NodeCategory::Decl>; +using FunctionDecl = FunctionSignature; using FunctionDefinitionStart = FunctionSignature, - NodeCategory::None>; + Lex::OpenCurlyBraceTokenIndex, NodeCategory::None>; // A function definition: `fn F() -> i32 { ... }`. struct FunctionDefinition { @@ -384,14 +390,14 @@ struct FunctionDefinition { FunctionDefinitionStartId signature; llvm::SmallVector body; - Token token; + Lex::CloseCurlyBraceTokenIndex token; }; using BuiltinFunctionDefinitionStart = FunctionSignature, NodeCategory::None>; + Lex::EqualTokenIndex, NodeCategory::None>; using BuiltinName = - LeafNode>; + LeafNode; // A builtin function definition: `fn F() -> i32 = "builtin name";` struct BuiltinFunctionDefinition { @@ -401,16 +407,16 @@ struct BuiltinFunctionDefinition { BuiltinFunctionDefinitionStartId signature; BuiltinNameId builtin_name; - Token token; + Lex::SemiTokenIndex token; }; // `alias` nodes // ------------- using AliasIntroducer = - LeafNode>; + LeafNode; using AliasInitializer = - LeafNode>; + LeafNode; // An `alias` declaration: `alias a = b;`. struct Alias { @@ -423,16 +429,14 @@ struct Alias { DeclName name; AliasInitializerId equals; AnyExprId initializer; - Token token; + Lex::SemiTokenIndex token; }; // `let` nodes // ----------- -using LetIntroducer = - LeafNode>; -using LetInitializer = - LeafNode>; +using LetIntroducer = LeafNode; +using LetInitializer = LeafNode; // A `let` declaration: `let a: i32 = 5;`. struct LetDecl { @@ -449,18 +453,18 @@ struct LetDecl { AnyExprId initializer; }; std::optional initializer; - Token token; + Lex::SemiTokenIndex token; }; // `var` nodes // ----------- using VariableIntroducer = - LeafNode>; + LeafNode; using ReturnedModifier = - LeafNode>; + LeafNode; using VariableInitializer = - LeafNode>; + LeafNode; // A `var` declaration: `var a: i32;` or `var a: i32 = 5;`. struct VariableDecl { @@ -478,14 +482,14 @@ struct VariableDecl { AnyExprId value; }; std::optional initializer; - Token token; + Lex::SemiTokenIndex token; }; // Statement nodes // --------------- using CodeBlockStart = - LeafNode>; + LeafNode; // A code block: `{ statement; statement; ... }`. struct CodeBlock { @@ -494,7 +498,7 @@ struct CodeBlock { CodeBlockStartId left_brace; llvm::SmallVector statements; - Token token; + Lex::CloseCurlyBraceTokenIndex token; }; // An expression statement: `F(x);`. @@ -503,11 +507,11 @@ struct ExprStatement { {.category = NodeCategory::Statement, .child_count = 1}); AnyExprId expr; - Token token; + Lex::SemiTokenIndex token; }; using BreakStatementStart = - LeafNode>; + LeafNode; // A break statement: `break;`. struct BreakStatement { @@ -517,11 +521,11 @@ struct BreakStatement { .child_count = 1}); BreakStatementStartId introducer; - Token token; + Lex::SemiTokenIndex token; }; using ContinueStatementStart = - LeafNode>; + LeafNode; // A continue statement: `continue;`. struct ContinueStatement { @@ -531,13 +535,13 @@ struct ContinueStatement { .child_count = 1}); ContinueStatementStartId introducer; - Token token; + Lex::SemiTokenIndex token; }; using ReturnStatementStart = - LeafNode>; + LeafNode; using ReturnVarModifier = - LeafNode>; + LeafNode; // A return statement: `return;` or `return expr;` or `return var;`. struct ReturnStatement { @@ -548,11 +552,11 @@ struct ReturnStatement { ReturnStatementStartId introducer; std::optional expr; std::optional var; - Token token; + Lex::SemiTokenIndex token; }; using ForHeaderStart = - LeafNode>; + LeafNode; // The `var ... in` portion of a `for` statement. struct ForIn { @@ -560,7 +564,7 @@ struct ForIn { {.bracketed_by = VariableIntroducer::Kind, .child_count = 2}); VariableIntroducerId introducer; - Token token; + Lex::InTokenIndex token; AnyPatternId pattern; }; @@ -572,7 +576,7 @@ struct ForHeader { ForHeaderStartId introducer; ForInId var; AnyExprId range; - Token token; + Lex::CloseParenTokenIndex token; }; // A complete `for (...) { ... }` statement. @@ -582,13 +586,13 @@ struct ForStatement { .bracketed_by = ForHeader::Kind, .child_count = 2}); - Token token; + Lex::ForTokenIndex token; ForHeaderId header; CodeBlockId body; }; using IfConditionStart = - LeafNode>; + LeafNode; // The condition portion of an `if` statement: `(expr)`. struct IfCondition { @@ -597,18 +601,18 @@ struct IfCondition { IfConditionStartId left_paren; AnyExprId condition; - Token token; + Lex::CloseParenTokenIndex token; }; using IfStatementElse = - LeafNode>; + LeafNode; // An `if` statement: `if (expr) { ... } else { ... }`. struct IfStatement { static constexpr auto Kind = NodeKind::IfStatement.Define( {.category = NodeCategory::Statement, .bracketed_by = IfCondition::Kind}); - Token token; + Lex::IfTokenIndex token; IfConditionId head; CodeBlockId then; @@ -620,7 +624,7 @@ struct IfStatement { }; using WhileConditionStart = - LeafNode>; + LeafNode; // The condition portion of a `while` statement: `(expr)`. struct WhileCondition { @@ -629,7 +633,7 @@ struct WhileCondition { WhileConditionStartId left_paren; AnyExprId condition; - Token token; + Lex::CloseParenTokenIndex token; }; // A `while` statement: `while (expr) { ... }`. @@ -639,13 +643,13 @@ struct WhileStatement { .bracketed_by = WhileCondition::Kind, .child_count = 2}); - Token token; + Lex::WhileTokenIndex token; WhileConditionId head; CodeBlockId body; }; using MatchConditionStart = - LeafNode>; + LeafNode; struct MatchCondition { static constexpr auto Kind = NodeKind::MatchCondition.Define( @@ -653,26 +657,26 @@ struct MatchCondition { MatchConditionStartId left_paren; AnyExprId condition; - Token token; + Lex::CloseParenTokenIndex token; }; using MatchIntroducer = - LeafNode>; + LeafNode; struct MatchStatementStart { static constexpr auto Kind = NodeKind::MatchStatementStart.Define( {.bracketed_by = MatchIntroducer::Kind, .child_count = 2}); MatchIntroducerId introducer; MatchConditionId condition; - Token token; + Lex::OpenCurlyBraceTokenIndex token; }; using MatchCaseIntroducer = - LeafNode>; + LeafNode; using MatchCaseGuardIntroducer = - LeafNode>; + LeafNode; using MatchCaseGuardStart = - LeafNode>; + LeafNode; struct MatchCaseGuard { static constexpr auto Kind = NodeKind::MatchCaseGuard.Define( @@ -681,11 +685,11 @@ struct MatchCaseGuard { MatchCaseGuardIntroducerId introducer; MatchCaseGuardStartId left_paren; AnyExprId condition; - Token token; + Lex::CloseParenTokenIndex token; }; -using MatchCaseEqualGreater = LeafNode>; +using MatchCaseEqualGreater = + LeafNode; struct MatchCaseStart { static constexpr auto Kind = NodeKind::MatchCaseStart.Define( @@ -695,7 +699,7 @@ struct MatchCaseStart { AnyPatternId pattern; std::optional guard; MatchCaseEqualGreaterId equal_greater_token; - Token token; + Lex::OpenCurlyBraceTokenIndex token; }; struct MatchCase { @@ -704,13 +708,13 @@ struct MatchCase { MatchCaseStartId head; llvm::SmallVector statements; - Token token; + Lex::CloseCurlyBraceTokenIndex token; }; using MatchDefaultIntroducer = - LeafNode>; -using MatchDefaultEqualGreater = LeafNode>; + LeafNode; +using MatchDefaultEqualGreater = + LeafNode; struct MatchDefaultStart { static constexpr auto Kind = NodeKind::MatchDefaultStart.Define( @@ -718,7 +722,7 @@ struct MatchDefaultStart { MatchDefaultIntroducerId introducer; MatchDefaultEqualGreaterId equal_greater_token; - Token token; + Lex::OpenCurlyBraceTokenIndex token; }; struct MatchDefault { @@ -727,7 +731,7 @@ struct MatchDefault { MatchDefaultStartId introducer; llvm::SmallVector statements; - Token token; + Lex::CloseCurlyBraceTokenIndex token; }; // A `match` statement: `match (expr) { case (...) => {...} default => {...}}`. @@ -740,14 +744,14 @@ struct MatchStatement { llvm::SmallVector cases; std::optional default_case; - Token token; + Lex::CloseCurlyBraceTokenIndex token; }; // Expression nodes // ---------------- -using ArrayExprStart = LeafNode>; +using ArrayExprStart = + LeafNode; // The start of an array type, `[i32;`. // @@ -758,7 +762,7 @@ struct ArrayExprSemi { ArrayExprStartId left_square; AnyExprId type; - Token token; + Lex::SemiTokenIndex token; }; // An array type, such as `[i32; 3]` or `[i32;]`. @@ -768,7 +772,7 @@ struct ArrayExpr { ArrayExprSemiId start; std::optional bound; - Token token; + Lex::CloseSquareBracketTokenIndex token; }; // The opening portion of an indexing expression: `a[`. @@ -779,7 +783,7 @@ struct IndexExprStart { NodeKind::IndexExprStart.Define({.child_count = 1}); AnyExprId sequence; - Token token; + Lex::OpenSquareBracketTokenIndex token; }; // An indexing expression, such as `a[1]`. @@ -791,11 +795,11 @@ struct IndexExpr { IndexExprStartId start; AnyExprId index; - Token token; + Lex::CloseSquareBracketTokenIndex token; }; using ParenExprStart = - LeafNode>; + LeafNode; // A parenthesized expression: `(a)`. struct ParenExpr { @@ -806,13 +810,13 @@ struct ParenExpr { ParenExprStartId start; AnyExprId expr; - Token token; + Lex::CloseParenTokenIndex token; }; using TupleLiteralStart = - LeafNode>; + LeafNode; using TupleLiteralComma = - LeafNode>; + LeafNode; // A tuple literal: `()`, `(a, b, c)`, or `(a,)`. struct TupleLiteral { @@ -822,7 +826,7 @@ struct TupleLiteral { TupleLiteralStartId start; CommaSeparatedList elements; - Token token; + Lex::CloseParenTokenIndex token; }; // The opening portion of a call expression: `F(`. @@ -833,11 +837,10 @@ struct CallExprStart { NodeKind::CallExprStart.Define({.child_count = 1}); AnyExprId callee; - Token token; + Lex::OpenParenTokenIndex token; }; -using CallExprComma = - LeafNode>; +using CallExprComma = LeafNode; // A call expression: `F(a, b, c)`. struct CallExpr { @@ -846,7 +849,7 @@ struct CallExpr { CallExprStartId start; CommaSeparatedList arguments; - Token token; + Lex::CloseParenTokenIndex token; }; // A member access expression: `a.b` or `a.(b)`. @@ -855,7 +858,7 @@ struct MemberAccessExpr { {.category = NodeCategory::Expr, .child_count = 2}); AnyExprId lhs; - Token token; + Lex::PeriodTokenIndex token; AnyMemberNameOrMemberExprId rhs; }; @@ -865,7 +868,7 @@ struct PointerMemberAccessExpr { {.category = NodeCategory::Expr, .child_count = 2}); AnyExprId lhs; - Token token; + Lex::MinusGreaterTokenIndex token; AnyMemberNameOrMemberExprId rhs; }; @@ -903,23 +906,22 @@ struct PostfixOperator { // Literals, operators, and modifiers #define CARBON_PARSE_NODE_KIND(...) -#define CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(Name, LexTokenKind) \ - using Name = LeafNode, \ +#define CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(Name, LexTokenKind) \ + using Name = LeafNode; -#define CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER(Name) \ - using Name##Modifier = \ - LeafNode, \ +#define CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER(Name) \ + using Name##Modifier = \ + LeafNode; -#define CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Name) \ - using PrefixOperator##Name = PrefixOperator>; -#define CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Name) \ - using InfixOperator##Name = InfixOperator>; -#define CARBON_PARSE_NODE_KIND_POSTFIX_OPERATOR(Name) \ - using PostfixOperator##Name = \ - PostfixOperator>; +#define CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Name) \ + using PrefixOperator##Name = \ + PrefixOperator; +#define CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Name) \ + using InfixOperator##Name = \ + InfixOperator; +#define CARBON_PARSE_NODE_KIND_POSTFIX_OPERATOR(Name) \ + using PostfixOperator##Name = \ + PostfixOperator; #include "toolchain/parse/node_kind.def" // The first operand of a short-circuiting infix operator: `a and` or `a or`. @@ -932,7 +934,9 @@ struct ShortCircuitOperandAnd { NodeKind::ShortCircuitOperandAnd.Define({.child_count = 1}); AnyExprId operand; - VirtualToken token; + // This is a virtual token. The `and` token is owned by the + // ShortCircuitOperatorAnd node. + Lex::AndTokenIndex token; }; struct ShortCircuitOperandOr { @@ -940,7 +944,9 @@ struct ShortCircuitOperandOr { NodeKind::ShortCircuitOperandOr.Define({.child_count = 1}); AnyExprId operand; - VirtualToken token; + // This is a virtual token. The `or` token is owned by the + // ShortCircuitOperatorOr node. + Lex::OrTokenIndex token; }; struct ShortCircuitOperatorAnd { @@ -950,7 +956,7 @@ struct ShortCircuitOperatorAnd { .child_count = 2}); ShortCircuitOperandAndId lhs; - Token token; + Lex::AndTokenIndex token; AnyExprId rhs; }; @@ -961,7 +967,7 @@ struct ShortCircuitOperatorOr { .child_count = 2}); ShortCircuitOperandOrId lhs; - Token token; + Lex::OrTokenIndex token; AnyExprId rhs; }; @@ -969,7 +975,7 @@ struct ShortCircuitOperatorOr { struct IfExprIf { static constexpr auto Kind = NodeKind::IfExprIf.Define({.child_count = 1}); - Token token; + Lex::IfTokenIndex token; AnyExprId condition; }; @@ -977,7 +983,7 @@ struct IfExprIf { struct IfExprThen { static constexpr auto Kind = NodeKind::IfExprThen.Define({.child_count = 1}); - Token token; + Lex::ThenTokenIndex token; AnyExprId result; }; @@ -990,7 +996,7 @@ struct IfExprElse { IfExprIfId start; IfExprThenId then; - Token token; + Lex::ElseTokenIndex token; AnyExprId else_result; }; @@ -998,7 +1004,7 @@ struct IfExprElse { // ------------ using ChoiceIntroducer = - LeafNode>; + LeafNode; struct ChoiceSignature { static constexpr auto Kind = NodeKind::ChoiceDefinitionStart.Define( @@ -1007,14 +1013,13 @@ struct ChoiceSignature { ChoiceIntroducerId introducer; llvm::SmallVector modifiers; DeclName name; - Token token; + Lex::OpenCurlyBraceTokenIndex token; }; using ChoiceDefinitionStart = ChoiceSignature; using ChoiceAlternativeListComma = - LeafNode>; + LeafNode; struct ChoiceDefinition { static constexpr auto Kind = NodeKind::ChoiceDefinition.Define( @@ -1027,27 +1032,26 @@ struct ChoiceDefinition { std::optional parameters; }; CommaSeparatedList alternatives; - Token token; + Lex::CloseCurlyBraceTokenIndex token; }; // Struct type and value literals // ---------------------------------------- // `{` -using StructLiteralStart = LeafNode>; -using StructTypeLiteralStart = LeafNode>; +using StructLiteralStart = + LeafNode; +using StructTypeLiteralStart = + LeafNode; // `,` -using StructComma = - LeafNode>; +using StructComma = LeafNode; // `.a` struct StructFieldDesignator { static constexpr auto Kind = NodeKind::StructFieldDesignator.Define({.child_count = 1}); - Token token; + Lex::PeriodTokenIndex token; NodeIdOneOf name; }; @@ -1057,7 +1061,7 @@ struct StructField { {.bracketed_by = StructFieldDesignator::Kind, .child_count = 2}); StructFieldDesignatorId designator; - Token token; + Lex::EqualTokenIndex token; AnyExprId expr; }; @@ -1067,7 +1071,7 @@ struct StructTypeField { {.bracketed_by = StructFieldDesignator::Kind, .child_count = 2}); StructFieldDesignatorId designator; - Token token; + Lex::ColonTokenIndex token; AnyExprId type_expr; }; @@ -1079,7 +1083,7 @@ struct StructLiteral { StructLiteralStartId start; CommaSeparatedList fields; - Token token; + Lex::CloseCurlyBraceTokenIndex token; }; // Struct type literals, such as `{.a: i32}`. @@ -1090,7 +1094,7 @@ struct StructTypeLiteral { StructTypeLiteralStartId start; CommaSeparatedList fields; - Token token; + Lex::CloseCurlyBraceTokenIndex token; }; // `class` declarations and definitions @@ -1098,7 +1102,7 @@ struct StructTypeLiteral { // `class` using ClassIntroducer = - LeafNode>; + LeafNode; // A class signature `class C` template , - NodeCategory::Decl>; +using ClassDecl = ClassSignature; // `class C {` using ClassDefinitionStart = ClassSignature, NodeCategory::None>; + Lex::OpenCurlyBraceTokenIndex, NodeCategory::None>; // `class C { ... }` struct ClassDefinition { @@ -1130,7 +1133,7 @@ struct ClassDefinition { ClassDefinitionStartId signature; llvm::SmallVector members; - Token token; + Lex::CloseCurlyBraceTokenIndex token; }; // Adapter declaration @@ -1138,7 +1141,7 @@ struct ClassDefinition { // `adapt` using AdaptIntroducer = - LeafNode>; + LeafNode; // `adapt SomeType;` struct AdaptDecl { static constexpr auto Kind = NodeKind::AdaptDecl.Define( @@ -1147,16 +1150,15 @@ struct AdaptDecl { AdaptIntroducerId introducer; llvm::SmallVector modifiers; AnyExprId adapted_type; - Token token; + Lex::SemiTokenIndex token; }; // Base class declaration // ---------------------- // `base` -using BaseIntroducer = - LeafNode>; -using BaseColon = LeafNode>; +using BaseIntroducer = LeafNode; +using BaseColon = LeafNode; // `extend base: BaseClass;` struct BaseDecl { static constexpr auto Kind = NodeKind::BaseDecl.Define( @@ -1166,7 +1168,7 @@ struct BaseDecl { llvm::SmallVector modifiers; BaseColonId colon; AnyExprId base_class; - Token token; + Lex::SemiTokenIndex token; }; // Interface declarations and definitions @@ -1174,7 +1176,7 @@ struct BaseDecl { // `interface` using InterfaceIntroducer = - LeafNode>; + LeafNode; // `interface I` template , + InterfaceSignature; // `interface I {` using InterfaceDefinitionStart = InterfaceSignature, - NodeCategory::None>; + Lex::OpenCurlyBraceTokenIndex, NodeCategory::None>; // `interface I { ... }` struct InterfaceDefinition { @@ -1207,28 +1208,26 @@ struct InterfaceDefinition { InterfaceDefinitionStartId signature; llvm::SmallVector members; - Token token; + Lex::CloseCurlyBraceTokenIndex token; }; // `impl`...`as` declarations and definitions // ------------------------------------------ // `impl` -using ImplIntroducer = - LeafNode>; +using ImplIntroducer = LeafNode; // `forall [...]` struct ImplForall { static constexpr auto Kind = NodeKind::ImplForall.Define({.child_count = 1}); - Token token; + Lex::ForallTokenIndex token; ImplicitParamListId params; }; // `as` with no type before it -using DefaultSelfImplAs = - LeafNode, - NodeCategory::ImplAs>; +using DefaultSelfImplAs = LeafNode; // ` as` struct TypeImplAs { @@ -1236,7 +1235,7 @@ struct TypeImplAs { {.category = NodeCategory::ImplAs, .child_count = 1}); AnyExprId type_expr; - Token token; + Lex::AsTokenIndex token; }; // `impl T as I` @@ -1255,12 +1254,12 @@ struct ImplSignature { }; // `impl T as I;` -using ImplDecl = ImplSignature, - NodeCategory::Decl>; +using ImplDecl = + ImplSignature; // `impl T as I {` using ImplDefinitionStart = - ImplSignature, NodeCategory::None>; + ImplSignature; // `impl T as I { ... }` struct ImplDefinition { @@ -1270,15 +1269,15 @@ struct ImplDefinition { ImplDefinitionStartId signature; llvm::SmallVector members; - Token token; + Lex::CloseCurlyBraceTokenIndex token; }; // Named constraint declarations and definitions // --------------------------------------------- // `constraint` -using NamedConstraintIntroducer = LeafNode>; +using NamedConstraintIntroducer = + LeafNode; // `constraint NC` template , NodeCategory::Decl>; + NamedConstraintSignature; // `constraint NC {` using NamedConstraintDefinitionStart = NamedConstraintSignature, - NodeCategory::None>; + Lex::OpenCurlyBraceTokenIndex, NodeCategory::None>; // `constraint NC { ... }` struct NamedConstraintDefinition { @@ -1311,7 +1309,7 @@ struct NamedConstraintDefinition { NamedConstraintDefinitionStartId signature; llvm::SmallVector members; - Token token; + Lex::CloseCurlyBraceTokenIndex token; }; // --------------------------------------------------------------------------- diff --git a/toolchain/parse/typed_nodes_test.cpp b/toolchain/parse/typed_nodes_test.cpp index 67e53a51a0de..594aa24695f0 100644 --- a/toolchain/parse/typed_nodes_test.cpp +++ b/toolchain/parse/typed_nodes_test.cpp @@ -315,15 +315,15 @@ TEST_F(TypedNodeTest, Token) { auto n_var = tree->ExtractAs(file.decls[0]); ASSERT_TRUE(n_var.has_value()); - EXPECT_EQ(tokens->GetKind(n_var->token.index), Lex::TokenKind::Semi); + EXPECT_EQ(tokens->GetKind(n_var->token), Lex::TokenKind::Semi); auto n_intro = tree->ExtractAs(n_var->introducer); ASSERT_TRUE(n_intro.has_value()); - EXPECT_EQ(tokens->GetKind(n_intro->token.index), Lex::TokenKind::Var); + EXPECT_EQ(tokens->GetKind(n_intro->token), Lex::TokenKind::Var); auto n_patt = tree->ExtractAs(n_var->pattern); ASSERT_TRUE(n_patt.has_value()); - EXPECT_EQ(tokens->GetKind(n_patt->token.index), Lex::TokenKind::Colon); + EXPECT_EQ(tokens->GetKind(n_patt->token), Lex::TokenKind::Colon); } TEST_F(TypedNodeTest, VerifyInvalid) {