mirror of
https://github.com/carbon-language/carbon-lang.git
synced 2026-10-04 22:02:52 +01:00
Chandler Carruth
a46ca6bf7a
This removes a (very) hot branch in the lexer where we need to special case when a token is the first token and can't look at its previous token. It also seems like a generally nice change to the structure of both the token buffer and parse tree as there are now bracketing elements for both ends and we should be able to avoid similar branching in the future. Mostly mechanical updates to the lexer and parser code to handle this, but also needed to special case the location information in the autoupdate code. And then the usual large body of auto-updated tests. No benchmark data for this change alone as in isolation and in the current lexer structure it doesn't make a big difference. But this branch was particularly difficult to handle when trying to update the whitespace skipping code to be faster, and so I think it is worth systematically avoiding the special case here.
419 lines
15 KiB
Modula-2
419 lines
15 KiB
Modula-2
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
|
|
// Exceptions. See /LICENSE for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
// This is an X-macro header. It does not use `#include` guards, and instead is
|
|
// designed to be `#include`ed after the x-macro is defined in order for its
|
|
// inclusion to expand to the desired output. Macro definitions are cleaned up
|
|
// at the end of this file.
|
|
//
|
|
// Supported x-macros are:
|
|
// - CARBON_PARSE_NODE_KIND(Name)
|
|
// Used as a fallback if other macros are missing. No kinds should use this
|
|
// directly.
|
|
// - CARBON_PARSE_NODE_KIND_BRACKET(Name, BracketName)
|
|
// Defines a bracketed node kind. BracketName should refer to the node
|
|
// kind that is the _start_ of the bracketed range.
|
|
// - CARBON_PARSE_NODE_KIND_CHILD_COUNT(Name, ChildCount)
|
|
// Defines a parse node with a set number of children, often 0. This count
|
|
// must be correct even when the node contains errors.
|
|
//
|
|
// This tree represents the subset relationship between these macros, where if a
|
|
// specific x-macro isn't defined, it'll fall back to the parent macro.
|
|
//
|
|
// Parse nodes are clustered based on language feature. Comments will show their
|
|
// relationship in postorder, using indentation for child node relationships.
|
|
|
|
#if !(defined(CARBON_PARSE_NODE_KIND) || \
|
|
(defined(CARBON_PARSE_NODE_KIND_BRACKET) && \
|
|
defined(CARBON_PARSE_NODE_KIND_CHILD_COUNT)))
|
|
#error "Must define CARBON_PARSE_NODE_KIND family x-macros to use this file."
|
|
#endif
|
|
|
|
// The BRACKET and CHILD_COUNT macros will use CARBON_PARSE_NODE_KIND by default
|
|
// when undefined.
|
|
#ifndef CARBON_PARSE_NODE_KIND_BRACKET
|
|
#define CARBON_PARSE_NODE_KIND_BRACKET(Name, ...) CARBON_PARSE_NODE_KIND(Name)
|
|
#endif
|
|
#ifndef CARBON_PARSE_NODE_KIND_CHILD_COUNT
|
|
#define CARBON_PARSE_NODE_KIND_CHILD_COUNT(Name, ...) \
|
|
CARBON_PARSE_NODE_KIND(Name)
|
|
#endif
|
|
|
|
// The start of the file.
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(FileStart, 0)
|
|
|
|
// The end of the file.
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(FileEnd, 0)
|
|
|
|
// An invalid parse. Used to balance the parse tree. Always has an error.
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(InvalidParse, 0)
|
|
|
|
// An empty declaration, such as `;`.
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(EmptyDeclaration, 0)
|
|
|
|
// A name in a non-expression context, such as a declaration.
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(Name, 0)
|
|
|
|
// A name in an expression context.
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(NameExpression, 0)
|
|
|
|
// `package`:
|
|
// PackageIntroducer
|
|
// _external_: Name
|
|
// _external_: Literal
|
|
// PackageLibrary
|
|
// PackageApi or PackageImpl
|
|
// PackageDirective
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(PackageIntroducer, 0)
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(PackageApi, 0)
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(PackageImpl, 0)
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(PackageLibrary, 1)
|
|
CARBON_PARSE_NODE_KIND_BRACKET(PackageDirective, PackageIntroducer)
|
|
|
|
// `namespace`:
|
|
// NamespaceStart
|
|
// _external_: Name or QualifiedDeclaration
|
|
// Namespace
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(NamespaceStart, 0)
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(Namespace, 2)
|
|
|
|
// A code block:
|
|
// CodeBlockStart
|
|
// _external_: statements
|
|
// CodeBlock
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(CodeBlockStart, 0)
|
|
CARBON_PARSE_NODE_KIND_BRACKET(CodeBlock, CodeBlockStart)
|
|
|
|
// `fn`:
|
|
// FunctionIntroducer
|
|
// _external_: Name or QualifiedDeclaration
|
|
// _external_: ParameterList
|
|
// _external_: type expression
|
|
// ReturnType
|
|
// FunctionDefinitionStart
|
|
// _external_: statements
|
|
// FunctionDefinition
|
|
//
|
|
// The above is the structure for a definition; for a declaration,
|
|
// FunctionDefinitionStart and later nodes are removed and replaced by
|
|
// FunctionDeclaration.
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(FunctionIntroducer, 0)
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(ReturnType, 1)
|
|
CARBON_PARSE_NODE_KIND_BRACKET(FunctionDefinitionStart, FunctionIntroducer)
|
|
CARBON_PARSE_NODE_KIND_BRACKET(FunctionDefinition, FunctionDefinitionStart)
|
|
CARBON_PARSE_NODE_KIND_BRACKET(FunctionDeclaration, FunctionIntroducer)
|
|
|
|
// A parameter list, possibly deduced:
|
|
// [Deduced]ParamertListStart
|
|
// _external_: [Generic]PatternBinding
|
|
// ParameterListComma
|
|
// [Deduced]ParameterList
|
|
//
|
|
// Expressions and ParameterListComma may repeat with ParameterListComma as a
|
|
// separator.
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(ParameterListStart, 0)
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(DeducedParameterListStart, 0)
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(ParameterListComma, 0)
|
|
CARBON_PARSE_NODE_KIND_BRACKET(ParameterList, ParameterListStart)
|
|
CARBON_PARSE_NODE_KIND_BRACKET(DeducedParameterList, DeducedParameterListStart)
|
|
|
|
// An array type, such as `[i32; 3]` or `[i32;]`:
|
|
// ArrayExpressionStart
|
|
// _external_: type expression
|
|
// ArrayExpressionSemi
|
|
// _external_: expression
|
|
// ArrayExpression
|
|
//
|
|
// The bound expression is optional.
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(ArrayExpressionStart, 0)
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(ArrayExpressionSemi, 2)
|
|
CARBON_PARSE_NODE_KIND_BRACKET(ArrayExpression, ArrayExpressionSemi)
|
|
|
|
// A pattern binding, such as `name: Type`:
|
|
// Name
|
|
// _external_: type expression
|
|
// [Generic]PatternBinding
|
|
//
|
|
// Address and Template may be parents to [Generic]PatternBinding, in that
|
|
// order.
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(PatternBinding, 2)
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(GenericPatternBinding, 2)
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(Address, 1)
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(Template, 1)
|
|
|
|
// `var`:
|
|
// VariableIntroducer
|
|
// _external_: PatternBinding
|
|
// optional VariableInitializer
|
|
// optional _external_: expression
|
|
// VariableDeclaration
|
|
//
|
|
// The VariableInitializer and following expression are paired: either both will
|
|
// be present, or neither will.
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(VariableIntroducer, 0)
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(VariableInitializer, 0)
|
|
CARBON_PARSE_NODE_KIND_BRACKET(VariableDeclaration, VariableIntroducer)
|
|
|
|
// An expression statement:
|
|
// _external_: expression
|
|
// ExpressionStatement
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(ExpressionStatement, 1)
|
|
|
|
// `break`:
|
|
// BreakStatementStart
|
|
// BreakStatement
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(BreakStatementStart, 0)
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(BreakStatement, 1)
|
|
|
|
// `continue`:
|
|
// ContinueStatementStart
|
|
// ContinueStatement
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(ContinueStatementStart, 0)
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(ContinueStatement, 1)
|
|
|
|
// `return`:
|
|
// ReturnStatementStart
|
|
// _external_: expression
|
|
// ReturnStatement
|
|
//
|
|
// The child expression is optional.
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(ReturnStatementStart, 0)
|
|
CARBON_PARSE_NODE_KIND_BRACKET(ReturnStatement, ReturnStatementStart)
|
|
|
|
// `for`:
|
|
// ForHeaderStart
|
|
// VariableIntroducer
|
|
// _external_: PatternBinding
|
|
// _external_: type expression
|
|
// ForIn
|
|
// _external_: expression
|
|
// ForHeader
|
|
// _external_: CodeBlock
|
|
// ForStatement
|
|
//
|
|
// Versus a normal `var`, ForIn replaces VariableDeclaration.
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(ForHeaderStart, 0)
|
|
CARBON_PARSE_NODE_KIND_BRACKET(ForIn, VariableIntroducer)
|
|
CARBON_PARSE_NODE_KIND_BRACKET(ForHeader, ForHeaderStart)
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(ForStatement, 2)
|
|
|
|
// `if` statement + `else`:
|
|
// IfConditionStart
|
|
// _external_: expression
|
|
// IfCondition
|
|
// _external_: CodeBlock
|
|
// IfStatementElse
|
|
// _external_: CodeBlock or IfStatement
|
|
// IfStatement
|
|
//
|
|
// IfStatementElse and the following node are optional based on `else` presence.
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(IfConditionStart, 0)
|
|
CARBON_PARSE_NODE_KIND_BRACKET(IfCondition, IfConditionStart)
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(IfStatementElse, 0)
|
|
CARBON_PARSE_NODE_KIND_BRACKET(IfStatement, IfCondition)
|
|
|
|
// `while`:
|
|
// WhileConditionStart
|
|
// _external_: expression
|
|
// WhileCondition
|
|
// _external_: CodeBlock
|
|
// WhileStatement
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(WhileConditionStart, 0)
|
|
CARBON_PARSE_NODE_KIND_BRACKET(WhileCondition, WhileConditionStart)
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(WhileStatement, 2)
|
|
|
|
// Index expressions, such as `a[1]`:
|
|
// _external_: expression
|
|
// IndexExpressionStart
|
|
// _external_: expression
|
|
// IndexExpression
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(IndexExpressionStart, 1)
|
|
CARBON_PARSE_NODE_KIND_BRACKET(IndexExpression, IndexExpressionStart)
|
|
|
|
// Parenthesized expressions, such as `(2)`:
|
|
// ParenExpressionOrTupleLiteralStart
|
|
// _external_: expression
|
|
// ParenExpression
|
|
//
|
|
// Tuples, such as `(1, 2)`:
|
|
// ParenExpressionOrTupleLiteralStart
|
|
// _external_: expression
|
|
// TupleLiteralComma
|
|
// TupleLiteral
|
|
//
|
|
// Expressions and TupleLiteralComma may repeat with TupleLiteralComma as a
|
|
// separator.
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(ParenExpressionOrTupleLiteralStart, 0)
|
|
CARBON_PARSE_NODE_KIND_BRACKET(ParenExpression,
|
|
ParenExpressionOrTupleLiteralStart)
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(TupleLiteralComma, 0)
|
|
CARBON_PARSE_NODE_KIND_BRACKET(TupleLiteral, ParenExpressionOrTupleLiteralStart)
|
|
|
|
// Call expressions, such as `a()`:
|
|
// _external_: expression
|
|
// CallExpressionStart
|
|
// _external_: expression
|
|
// CallExpressionComma
|
|
// CallExpression
|
|
//
|
|
// Expressions and CallExpressionComma may repeat with CallExpressionComma as a
|
|
// separator.
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(CallExpressionStart, 1)
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(CallExpressionComma, 0)
|
|
CARBON_PARSE_NODE_KIND_BRACKET(CallExpression, CallExpressionStart)
|
|
|
|
// A qualified declaration, such as `a.b`:
|
|
// _external_: NameExpression or QualifiedDeclaration
|
|
// _external_: Name
|
|
// QualifiedDeclaration
|
|
//
|
|
// TODO: This will eventually more general expressions, for example with
|
|
// `GenericType(type_args).ChildType(child_type_args).Name`.
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(QualifiedDeclaration, 2)
|
|
|
|
// A member access expression, such as `a.b` or
|
|
// `GetObject().(Interface.member)`:
|
|
// _external_: lhs expression
|
|
// _external_: rhs expression
|
|
// QualifiedExpression
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(MemberAccessExpression, 2)
|
|
|
|
// A pointer member access expression, such as `a->b` or
|
|
// `GetObject()->(Interface.member)`:
|
|
// _external_: lhs expression
|
|
// _external_: rhs expression
|
|
// QualifiedExpression
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(PointerMemberAccessExpression, 2)
|
|
|
|
// A literal.
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(Literal, 0)
|
|
|
|
// A prefix operator:
|
|
// _external_: expression
|
|
// PrefixOperator
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(PrefixOperator, 1)
|
|
|
|
// An infix operator:
|
|
// _external_: lhs expression
|
|
// _external_: rhs expression
|
|
// InfixOperator
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(InfixOperator, 2)
|
|
|
|
// The first operand of a short-circuiting infix operator:
|
|
// _external_: expression
|
|
// ShortCircuitOperand
|
|
// _external_: expression
|
|
// _external_: InfixOperator
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(ShortCircuitOperand, 1)
|
|
|
|
// A postfix operator:
|
|
// _external_: expression
|
|
// PostfixOperator
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(PostfixOperator, 1)
|
|
|
|
// `if` expression + `then` + `else`:
|
|
// _external_: expression
|
|
// IfExpressionIf
|
|
// _external_: expression
|
|
// IfExpressionThen
|
|
// _external_: expression
|
|
// IfExpressionElse
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(IfExpressionIf, 1)
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(IfExpressionThen, 1)
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(IfExpressionElse, 3)
|
|
|
|
// Struct literals, such as `{.a = 0}`:
|
|
// StructLiteralOrStructTypeLiteralStart
|
|
// _external_: Name
|
|
// StructFieldDesignator
|
|
// _external_: expression
|
|
// StructFieldValue
|
|
// StructComma
|
|
// StructLiteral
|
|
//
|
|
// Struct type literals, such as `{.a: i32}`:
|
|
// _external_: Name
|
|
// StructFieldDesignator
|
|
// _external_: type expression
|
|
// StructFieldType
|
|
// StructComma
|
|
// StructTypeLiteral
|
|
//
|
|
// Elements (StructFieldValue and StructFieldType, respectively) and StructComma
|
|
// may repeat with StructComma as a separator.
|
|
//
|
|
// When a valid StructFieldType or StructFieldValue cannot be formed, elements
|
|
// may be replaced by StructFieldUnknown, which may have a preceding sibling
|
|
// StructFieldDesignator if one was successfully parsed.
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(StructLiteralOrStructTypeLiteralStart, 0)
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(StructFieldDesignator, 1)
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(StructFieldValue, 2)
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(StructFieldType, 2)
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(StructFieldUnknown, 0)
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(StructComma, 0)
|
|
CARBON_PARSE_NODE_KIND_BRACKET(StructLiteral,
|
|
StructLiteralOrStructTypeLiteralStart)
|
|
CARBON_PARSE_NODE_KIND_BRACKET(StructTypeLiteral,
|
|
StructLiteralOrStructTypeLiteralStart)
|
|
|
|
// `class`:
|
|
// ClassIntroducer
|
|
// _external_: Name or QualifiedDeclaration
|
|
// ClassDefinitionStart
|
|
// _external_: declarations
|
|
// ClassDefinition
|
|
//
|
|
// The above is the structure for a definition; for a declaration,
|
|
// ClassDefinitionStart and later nodes are removed and replaced by
|
|
// ClassDeclaration.
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(ClassIntroducer, 0)
|
|
CARBON_PARSE_NODE_KIND_BRACKET(ClassDefinitionStart, ClassIntroducer)
|
|
CARBON_PARSE_NODE_KIND_BRACKET(ClassDefinition, ClassDefinitionStart)
|
|
CARBON_PARSE_NODE_KIND_BRACKET(ClassDeclaration, ClassIntroducer)
|
|
|
|
// `interface`:
|
|
// InterfaceIntroducer
|
|
// _external_: Name or QualifiedDeclaration
|
|
// InterfaceDefinitionStart
|
|
// _external_: declarations
|
|
// InterfaceDefinition
|
|
//
|
|
// The above is the structure for a definition; for a declaration,
|
|
// InterfaceDefinitionStart and later nodes are removed and replaced by
|
|
// InterfaceDeclaration.
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(InterfaceIntroducer, 0)
|
|
CARBON_PARSE_NODE_KIND_BRACKET(InterfaceDefinitionStart, InterfaceIntroducer)
|
|
CARBON_PARSE_NODE_KIND_BRACKET(InterfaceDefinition, InterfaceDefinitionStart)
|
|
CARBON_PARSE_NODE_KIND_BRACKET(InterfaceDeclaration, InterfaceIntroducer)
|
|
|
|
// `constraint`:
|
|
// NamedConstraintIntroducer
|
|
// _external_: Name or QualifiedDeclaration
|
|
// NamedConstraintDefinitionStart
|
|
// _external_: declarations
|
|
// NamedConstraintDefinition
|
|
//
|
|
// The above is the structure for a definition; for a declaration,
|
|
// NamedConstraintDefinitionStart and later nodes are removed and replaced by
|
|
// NamedConstraintDeclaration.
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(NamedConstraintIntroducer, 0)
|
|
CARBON_PARSE_NODE_KIND_BRACKET(NamedConstraintDefinitionStart,
|
|
NamedConstraintIntroducer)
|
|
CARBON_PARSE_NODE_KIND_BRACKET(NamedConstraintDefinition,
|
|
NamedConstraintDefinitionStart)
|
|
CARBON_PARSE_NODE_KIND_BRACKET(NamedConstraintDeclaration,
|
|
NamedConstraintIntroducer)
|
|
|
|
// The `self` value and `Self` type identifier keywords. Typically of the form
|
|
// `self: Self`:
|
|
// SelfValueName
|
|
// SelfTypeNameExpression
|
|
// PatternBinding
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(SelfValueName, 0)
|
|
CARBON_PARSE_NODE_KIND_CHILD_COUNT(SelfTypeNameExpression, 0)
|
|
|
|
#undef CARBON_PARSE_NODE_KIND
|
|
#undef CARBON_PARSE_NODE_KIND_BRACKET
|
|
#undef CARBON_PARSE_NODE_KIND_CHILD_COUNT
|