From 3c01ee69ed13185f0477c75189283aaebe68ab17 Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Fri, 31 May 2024 16:11:51 -0700 Subject: [PATCH] Move information on the token associated with a parse node from the `.def` file into the typed node. (#4001) Instead of tracking the token associated with a parse node in the `.def` file macro, track it on the typed node instead. List the token as a field inside the node structure to show the order of the token relative to the other components of the grammar production, and to allow the token index to be accessed when the node is extracted. Remove the corresponding information from the `.def` file, leaving behind just a list of parse node kinds in the majority of cases. This also removes the checking of the token kind associated with a parse node in the case where the parse node has errors. Previously we had a flag on the node kind to indicate whether we should check this, but per [discord discussion](https://discord.com/channels/655572317891461132/655578254970716160/1246214418979881052), we have decided to remove this. --------- Co-authored-by: Jon Ross-Perkins --- common/struct_reflection.h | 29 +- toolchain/lex/BUILD | 7 + toolchain/lex/token_index.h | 37 ++ toolchain/lex/tokenized_buffer.h | 24 +- toolchain/parse/BUILD | 1 + toolchain/parse/context.cpp | 2 - toolchain/parse/extract.cpp | 337 ++++++++---- toolchain/parse/node_ids.h | 27 + toolchain/parse/node_kind.cpp | 27 +- toolchain/parse/node_kind.def | 796 ++++++--------------------- toolchain/parse/tree.cpp | 14 +- toolchain/parse/tree.h | 50 +- toolchain/parse/typed_nodes.h | 374 +++++++++---- toolchain/parse/typed_nodes_test.cpp | 123 ++++- 14 files changed, 921 insertions(+), 927 deletions(-) create mode 100644 toolchain/lex/token_index.h diff --git a/common/struct_reflection.h b/common/struct_reflection.h index 6d32b3e938d6..9bb113819817 100644 --- a/common/struct_reflection.h +++ b/common/struct_reflection.h @@ -21,7 +21,7 @@ // - Only simple aggregate structs are supported. Types with base classes, // non-public data members, constructors, or virtual functions are not // supported. -// - Structs with more than 6 fields are not supported. This limit is easy to +// - Structs with more than 8 fields are not supported. This limit is easy to // increase if needed, but removing it entirely is hard. // - Structs containing a reference to the same type are not supported. @@ -76,7 +76,7 @@ constexpr auto CountFields() -> int { return CountFields>(); } else if constexpr (AnyWorkedSoFar) { constexpr int NumFields = sizeof...(Fields) - 1; - static_assert(NumFields <= 6, "Unsupported: too many fields in struct"); + static_assert(NumFields <= 8, "Unsupported: too many fields in struct"); return NumFields; } else if constexpr (sizeof...(Fields) > 32) { // If we go too far without finding a working initializer, something @@ -161,6 +161,31 @@ struct FieldAccessor<6> { } }; +template <> +struct FieldAccessor<7> { + template + static auto Get(T& value) -> auto { + auto& [field0, field1, field2, field3, field4, field5, field6] = value; + return std::tuple(field0, field1, field2, field3, field4, + field5, field6); + } +}; + +template <> +struct FieldAccessor<8> { + template + static auto Get(T& value) -> auto { + auto& [field0, field1, field2, field3, field4, field5, field6, field7] = + value; + return std::tuple( + field0, field1, field2, field3, field4, field5, field6, field7); + } +}; + } // namespace Internal // Get the fields of the struct `T` as a tuple. diff --git a/toolchain/lex/BUILD b/toolchain/lex/BUILD index d956c3ad0e1a..35ef519535b6 100644 --- a/toolchain/lex/BUILD +++ b/toolchain/lex/BUILD @@ -196,6 +196,12 @@ cc_library( ], ) +cc_library( + name = "token_index", + hdrs = ["token_index.h"], + deps = ["//toolchain/base:index_base"], +) + cc_library( name = "tokenized_buffer", srcs = ["tokenized_buffer.cpp"], @@ -205,6 +211,7 @@ cc_library( ":helpers", ":numeric_literal", ":string_literal", + ":token_index", ":token_kind", "//common:check", "//common:ostream", diff --git a/toolchain/lex/token_index.h b/toolchain/lex/token_index.h new file mode 100644 index 000000000000..7588383b598d --- /dev/null +++ b/toolchain/lex/token_index.h @@ -0,0 +1,37 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef CARBON_TOOLCHAIN_LEX_TOKEN_INDEX_H_ +#define CARBON_TOOLCHAIN_LEX_TOKEN_INDEX_H_ + +#include "toolchain/base/index_base.h" + +namespace Carbon::Lex { + +// A lightweight handle to a lexed token in a `TokenizedBuffer`. +// +// `TokenIndex` objects are designed to be passed by value, not reference or +// pointer. They are also designed to be small and efficient to store in data +// structures. +// +// `TokenIndex` objects from the same `TokenizedBuffer` can be compared with +// each other, both for being the same token within the buffer, and to establish +// relative position within the token stream that has been lexed out of the +// buffer. `TokenIndex` objects from different `TokenizedBuffer`s cannot be +// meaningfully compared. +// +// All other APIs to query a `TokenIndex` are on the `TokenizedBuffer`. +struct TokenIndex : public IndexBase { + static const TokenIndex Invalid; + // Comments aren't tokenized, so this is the first token after FileStart. + static const TokenIndex FirstNonCommentToken; + using IndexBase::IndexBase; +}; + +constexpr TokenIndex TokenIndex::Invalid(TokenIndex::InvalidIndex); +constexpr TokenIndex TokenIndex::FirstNonCommentToken(1); + +} // namespace Carbon::Lex + +#endif // CARBON_TOOLCHAIN_LEX_TOKEN_INDEX_H_ diff --git a/toolchain/lex/tokenized_buffer.h b/toolchain/lex/tokenized_buffer.h index ae43a201fbbd..f4ebd3dfe34d 100644 --- a/toolchain/lex/tokenized_buffer.h +++ b/toolchain/lex/tokenized_buffer.h @@ -20,6 +20,7 @@ #include "toolchain/base/index_base.h" #include "toolchain/base/value_store.h" #include "toolchain/diagnostics/diagnostic_emitter.h" +#include "toolchain/lex/token_index.h" #include "toolchain/lex/token_kind.h" #include "toolchain/source/source_buffer.h" @@ -27,29 +28,6 @@ namespace Carbon::Lex { class TokenizedBuffer; -// A lightweight handle to a lexed token in a `TokenizedBuffer`. -// -// `TokenIndex` objects are designed to be passed by value, not reference or -// pointer. They are also designed to be small and efficient to store in data -// structures. -// -// `TokenIndex` objects from the same `TokenizedBuffer` can be compared with -// each other, both for being the same token within the buffer, and to establish -// relative position within the token stream that has been lexed out of the -// buffer. `TokenIndex` objects from different `TokenizedBuffer`s cannot be -// meaningfully compared. -// -// All other APIs to query a `TokenIndex` are on the `TokenizedBuffer`. -struct TokenIndex : public IndexBase { - static const TokenIndex Invalid; - // Comments aren't tokenized, so this is the first token after FileStart. - static const TokenIndex FirstNonCommentToken; - using IndexBase::IndexBase; -}; - -constexpr TokenIndex TokenIndex::Invalid(TokenIndex::InvalidIndex); -constexpr TokenIndex TokenIndex::FirstNonCommentToken(1); - // A lightweight handle to a lexed line in a `TokenizedBuffer`. // // `LineIndex` objects are designed to be passed by value, not reference or diff --git a/toolchain/parse/BUILD b/toolchain/parse/BUILD index 32d49ab8a33f..c3d45e2156d7 100644 --- a/toolchain/parse/BUILD +++ b/toolchain/parse/BUILD @@ -27,6 +27,7 @@ cc_library( "//common:enum_base", "//common:ostream", "//toolchain/base:index_base", + "//toolchain/lex:token_index", "//toolchain/lex:token_kind", "@llvm-project//llvm:Support", ], diff --git a/toolchain/parse/context.cpp b/toolchain/parse/context.cpp index 76f1c70a04d3..237008b6cd97 100644 --- a/toolchain/parse/context.cpp +++ b/toolchain/parse/context.cpp @@ -68,7 +68,6 @@ Context::Context(Tree& tree, Lex::TokenizedBuffer& tokens, auto Context::AddLeafNode(NodeKind kind, Lex::TokenIndex token, bool has_error) -> void { - kind.CheckMatchesTokenKind(tokens_->GetKind(token), has_error); tree_->node_impls_.push_back( Tree::NodeImpl(kind, has_error, token, /*subtree_size=*/1)); if (has_error) { @@ -78,7 +77,6 @@ auto Context::AddLeafNode(NodeKind kind, Lex::TokenIndex token, bool has_error) auto Context::AddNode(NodeKind kind, Lex::TokenIndex token, int subtree_start, bool has_error) -> void { - kind.CheckMatchesTokenKind(tokens_->GetKind(token), has_error); int subtree_size = tree_->size() - subtree_start + 1; tree_->node_impls_.push_back( Tree::NodeImpl(kind, has_error, token, subtree_size)); diff --git a/toolchain/parse/extract.cpp b/toolchain/parse/extract.cpp index ab519dc9e93b..364ac97fa3b4 100644 --- a/toolchain/parse/extract.cpp +++ b/toolchain/parse/extract.cpp @@ -13,6 +13,76 @@ namespace Carbon::Parse { +namespace { +// Implementation of the process of extracting a typed node structure from the +// parse tree. The extraction process uses the class `Extractable`, defined +// below, to extract individual fields of type `T`. +class NodeExtractor { + public: + struct CheckpointState { + Tree::SiblingIterator it; + }; + + NodeExtractor(const Tree* tree, Lex::TokenizedBuffer* tokens, + ErrorBuilder* trace, NodeId node_id, + llvm::iterator_range children) + : tree_(tree), + tokens_(tokens), + trace_(trace), + node_id_(node_id), + it_(children.begin()), + end_(children.end()) {} + + auto at_end() const -> bool { return it_ == end_; } + auto kind() const -> NodeKind { return tree_->node_kind(*it_); } + auto has_token() const -> bool { return node_id_.is_valid(); } + auto token() const -> Lex::TokenIndex { return tree_->node_token(node_id_); } + auto token_kind() const -> Lex::TokenKind { + return tokens_->GetKind(token()); + } + auto trace() const -> ErrorBuilder* { return trace_; } + + // Saves a checkpoint of our current position so we can return later if + // extraction of a child node fails. + auto Checkpoint() const -> CheckpointState { return {.it = it_}; } + auto RestoreCheckpoint(CheckpointState checkpoint) { it_ = checkpoint.it; } + + // Determines whether the current position matches the specified node kind. If + // not, produces a suitable trace message. + auto MatchesNodeIdForKind(NodeKind kind) const -> bool; + + // Determines whether the current position matches the specified node + // category. If not, produces a suitable trace message. + auto MatchesNodeIdInCategory(NodeCategory category) const -> bool; + + // Determines whether the current position matches any of the specified node + // kinds. If not, produces a suitable trace message. + auto MatchesNodeIdOneOf(std::initializer_list kinds) const -> bool; + + // Determines whether the token corresponding to the enclosing node is of the + // specified kind. If not, produces a suitable trace message. + auto MatchesTokenKind(Lex::TokenKind expected_kind) const -> bool; + + // Extracts the next node from the tree. + auto ExtractNode() -> NodeId { return *it_++; } + + // Extracts a tuple-like type `T` by extracting its components and then + // assembling a `T` value. + template + auto ExtractTupleLikeType(std::index_sequence /*indices*/, + std::tuple* /*type*/) -> std::optional; + + private: + const Tree* tree_; + Lex::TokenizedBuffer* tokens_; + ErrorBuilder* trace_; + NodeId node_id_; + Tree::SiblingIterator it_; + Tree::SiblingIterator end_; +}; +} // namespace + +namespace { // A trait type that should be specialized by types that can be extracted // from a parse tree. A specialization should provide the following API: // @@ -23,9 +93,7 @@ namespace Carbon::Parse { // // `it`, and increment `it` past this type. Returns `std::nullopt` if // // the tree is malformed. If `trace != nullptr`, writes what actions // // were taken to `*trace`. -// static auto Extract(Tree* tree, Tree::SiblingIterator& it, -// Tree::SiblingIterator end, -// ErrorBuilder* trace) -> std::optional; +// static auto Extract(NodeExtractor* extractor) -> std::optional; // }; // ``` // @@ -35,44 +103,40 @@ namespace Carbon::Parse { // This class is only in this file. template struct Extractable; +} // namespace // Extract a `NodeId` as a single child. template <> struct Extractable { - static auto Extract(const Tree* tree, Tree::SiblingIterator& it, - Tree::SiblingIterator end, ErrorBuilder* trace) - -> std::optional { - if (it == end) { - if (trace) { + static auto Extract(NodeExtractor& extractor) -> std::optional { + if (extractor.at_end()) { + if (auto* trace = extractor.trace()) { *trace << "NodeId error: no more children\n"; } return std::nullopt; } - if (trace) { - *trace << "NodeId: " << tree->node_kind(*it) << " consumed\n"; + if (auto* trace = extractor.trace()) { + *trace << "NodeId: " << extractor.kind() << " consumed\n"; } - return *it++; + return extractor.ExtractNode(); } }; -static auto NodeIdForKindAccept(const NodeKind& kind, const Tree* tree, - const Tree::SiblingIterator& it, - Tree::SiblingIterator end, ErrorBuilder* trace) - -> bool { - if (it == end || tree->node_kind(*it) != kind) { - if (trace) { - if (it == end) { - *trace << "NodeIdForKind error: no more children, expected " << kind - << "\n"; +auto NodeExtractor::MatchesNodeIdForKind(NodeKind expected_kind) const -> bool { + if (at_end() || kind() != expected_kind) { + if (trace_) { + if (at_end()) { + *trace_ << "NodeIdForKind error: no more children, expected " + << expected_kind << "\n"; } else { - *trace << "NodeIdForKind error: wrong kind " << tree->node_kind(*it) - << ", expected " << kind << "\n"; + *trace_ << "NodeIdForKind error: wrong kind " << kind() << ", expected " + << expected_kind << "\n"; } } return false; } - if (trace) { - *trace << "NodeIdForKind: " << kind << " consumed\n"; + if (trace_) { + *trace_ << "NodeIdForKind: " << expected_kind << " consumed\n"; } return true; } @@ -81,35 +145,32 @@ static auto NodeIdForKindAccept(const NodeKind& kind, const Tree* tree, // as a single required child. template struct Extractable> { - static auto Extract(const Tree* tree, Tree::SiblingIterator& it, - Tree::SiblingIterator end, ErrorBuilder* trace) + static auto Extract(NodeExtractor& extractor) -> std::optional> { - if (NodeIdForKindAccept(Kind, tree, it, end, trace)) { - return NodeIdForKind(*it++); + if (extractor.MatchesNodeIdForKind(Kind)) { + return NodeIdForKind(extractor.ExtractNode()); } else { return std::nullopt; } } }; -static auto NodeIdInCategoryAccept(NodeCategory category, const Tree* tree, - const Tree::SiblingIterator& it, - Tree::SiblingIterator end, - ErrorBuilder* trace) -> bool { - if (it == end || !tree->node_kind(*it).category().HasAnyOf(category)) { - if (trace) { - *trace << "NodeIdInCategory " << category << " error: "; - if (it == end) { - *trace << "no more children\n"; +auto NodeExtractor::MatchesNodeIdInCategory(NodeCategory category) const + -> bool { + if (at_end() || !kind().category().HasAnyOf(category)) { + if (trace_) { + *trace_ << "NodeIdInCategory " << category << " error: "; + if (at_end()) { + *trace_ << "no more children\n"; } else { - *trace << "kind " << tree->node_kind(*it) << " doesn't match\n"; + *trace_ << "kind " << kind() << " doesn't match\n"; } } return false; } - if (trace) { - *trace << "NodeIdInCategory " << category << ": kind " - << tree->node_kind(*it) << " consumed\n"; + if (trace_) { + *trace_ << "NodeIdInCategory " << category << ": kind " << kind() + << " consumed\n"; } return true; } @@ -117,47 +178,45 @@ static auto NodeIdInCategoryAccept(NodeCategory category, const Tree* tree, // Extract a `NodeIdInCategory` as a single child. template struct Extractable> { - static auto Extract(const Tree* tree, Tree::SiblingIterator& it, - Tree::SiblingIterator end, ErrorBuilder* trace) + static auto Extract(NodeExtractor& extractor) -> std::optional> { - if (NodeIdInCategoryAccept(Category, tree, it, end, trace)) { - return NodeIdInCategory(*it++); + if (extractor.MatchesNodeIdInCategory(Category)) { + return NodeIdInCategory(extractor.ExtractNode()); } else { return std::nullopt; } } }; -static auto NodeIdOneOfAccept(std::initializer_list kinds, - const Tree* tree, const Tree::SiblingIterator& it, - Tree::SiblingIterator end, ErrorBuilder* trace) - -> bool { +auto NodeExtractor::MatchesNodeIdOneOf( + std::initializer_list kinds) const -> bool { auto trace_kinds = [&] { llvm::ListSeparator sep(" or "); for (auto kind : kinds) { - *trace << sep << kind; + *trace_ << sep << kind; } }; - auto kind = tree->node_kind(*it); - if (it == end || std::find(kinds.begin(), kinds.end(), kind) == kinds.end()) { - if (trace) { - if (it == end) { - *trace << "NodeIdOneOf error: no more children, expected "; + auto node_kind = kind(); + if (at_end() || + std::find(kinds.begin(), kinds.end(), node_kind) == kinds.end()) { + if (trace_) { + if (at_end()) { + *trace_ << "NodeIdOneOf error: no more children, expected "; trace_kinds(); - *trace << "\n"; + *trace_ << "\n"; } else { - *trace << "NodeIdOneOf error: wrong kind " << tree->node_kind(*it) - << ", expected "; + *trace_ << "NodeIdOneOf error: wrong kind " << node_kind + << ", expected "; trace_kinds(); - *trace << "\n"; + *trace_ << "\n"; } } return false; } - if (trace) { - *trace << "NodeIdOneOf "; + if (trace_) { + *trace_ << "NodeIdOneOf "; trace_kinds(); - *trace << ": " << tree->node_kind(*it) << " consumed\n"; + *trace_ << ": " << node_kind << " consumed\n"; } return true; } @@ -165,11 +224,10 @@ static auto NodeIdOneOfAccept(std::initializer_list kinds, // Extract a `NodeIdOneOf` as a single required child. template struct Extractable> { - static auto Extract(const Tree* tree, Tree::SiblingIterator& it, - Tree::SiblingIterator end, ErrorBuilder* trace) + static auto Extract(NodeExtractor& extractor) -> std::optional> { - if (NodeIdOneOfAccept({T::Kind...}, tree, it, end, trace)) { - return NodeIdOneOf(*it++); + if (extractor.MatchesNodeIdOneOf({T::Kind...})) { + return NodeIdOneOf(extractor.ExtractNode()); } else { return std::nullopt; } @@ -180,12 +238,10 @@ struct Extractable> { // Note: this is only instantiated once, so no need to create a helper function. template struct Extractable> { - static auto Extract(const Tree* tree, Tree::SiblingIterator& it, - Tree::SiblingIterator end, ErrorBuilder* trace) - -> std::optional> { - if (it == end || tree->node_kind(*it) == T::Kind) { - if (trace) { - if (it == end) { + static auto Extract(NodeExtractor& extractor) -> std::optional> { + if (extractor.at_end() || extractor.kind() == T::Kind) { + if (auto* trace = extractor.trace()) { + if (extractor.at_end()) { *trace << "NodeIdNot " << T::Kind << " error: no more children\n"; } else { *trace << "NodeIdNot error: unexpected " << T::Kind << "\n"; @@ -193,35 +249,34 @@ struct Extractable> { } return std::nullopt; } - if (trace) { - *trace << "NodeIdNot " << T::Kind << ": " << tree->node_kind(*it) + if (auto* trace = extractor.trace()) { + *trace << "NodeIdNot " << T::Kind << ": " << extractor.kind() << " consumed\n"; } - return NodeIdNot(*it++); + return NodeIdNot(extractor.ExtractNode()); } }; // Extract an `llvm::SmallVector` by extracting `T`s until we can't. template struct Extractable> { - static auto Extract(const Tree* tree, Tree::SiblingIterator& it, - Tree::SiblingIterator end, ErrorBuilder* trace) + static auto Extract(NodeExtractor& extractor) -> std::optional> { - if (trace) { + if (auto* trace = extractor.trace()) { *trace << "Vector: begin\n"; } llvm::SmallVector result; - while (it != end) { - auto old_it = it; - auto item = Extractable::Extract(tree, it, end, trace); + while (!extractor.at_end()) { + auto checkpoint = extractor.Checkpoint(); + auto item = Extractable::Extract(extractor); if (!item.has_value()) { - it = old_it; + extractor.RestoreCheckpoint(checkpoint); break; } result.push_back(*item); } std::reverse(result.begin(), result.end()); - if (trace) { + if (auto* trace = extractor.trace()) { *trace << "Vector: end\n"; } return result; @@ -232,91 +287,131 @@ struct Extractable> { // a `T`, and extracting nothing if that fails. template struct Extractable> { - static auto Extract(const Tree* tree, Tree::SiblingIterator& it, - Tree::SiblingIterator end, ErrorBuilder* trace) + static auto Extract(NodeExtractor& extractor) -> std::optional> { - if (trace) { + if (auto* trace = extractor.trace()) { *trace << "Optional " << typeid(T).name() << ": begin\n"; } - auto old_it = it; - std::optional value = Extractable::Extract(tree, it, end, trace); + auto checkpoint = extractor.Checkpoint(); + std::optional value = Extractable::Extract(extractor); if (value) { - if (trace) { + if (auto* trace = extractor.trace()) { *trace << "Optional " << typeid(T).name() << ": found\n"; } return value; } - if (trace) { + if (auto* trace = extractor.trace()) { *trace << "Optional " << typeid(T).name() << ": missing\n"; } - it = old_it; + extractor.RestoreCheckpoint(checkpoint); return value; } }; +auto NodeExtractor::MatchesTokenKind(Lex::TokenKind expected_kind) const + -> bool { + if (!node_id_.is_valid()) { + if (trace_) { + *trace_ << "Token " << expected_kind + << " expected but processing root node\n"; + } + return false; + } + if (token_kind() != expected_kind) { + if (trace_) { + *trace_ << "Token " << expected_kind << " expected for " + << tree_->node_kind(node_id_) << ", found " << token_kind() + << "\n"; + } + return false; + } + return true; +} + +// Extract the token corresponding to a node. +template +struct Extractable> { + static auto Extract(NodeExtractor& extractor) -> std::optional> { + if (extractor.MatchesTokenKind(Kind)) { + return Token{.index = extractor.token()}; + } else { + return std::nullopt; + } + } +}; + +// Extract the token corresponding to a node. +template <> +struct Extractable { + static auto Extract(NodeExtractor& extractor) -> std::optional { + if (!extractor.has_token()) { + if (auto* trace = extractor.trace()) { + *trace << "Token expected but processing root node\n"; + } + return std::nullopt; + } + return AnyToken{.index = extractor.token()}; + } +}; + template -static auto ExtractTupleLikeType(const Tree* tree, Tree::SiblingIterator& it, - Tree::SiblingIterator end, ErrorBuilder* trace, - std::index_sequence /*indices*/, - std::tuple* /*type*/) +auto NodeExtractor::ExtractTupleLikeType( + std::index_sequence /*indices*/, std::tuple* /*type*/) -> std::optional { std::tuple...> fields; - if (trace) { - *trace << "Aggregate " << typeid(T).name() << ": begin\n"; + if (trace_) { + *trace_ << "Aggregate " << typeid(T).name() << ": begin\n"; } // Use a fold over the `=` operator to parse fields from right to left. [[maybe_unused]] int unused; bool ok = true; static_cast( - ((ok && (ok = (std::get(fields) = - Extractable::Extract(tree, it, end, trace)) + ((ok && (ok = (std::get(fields) = Extractable::Extract(*this)) .has_value()), unused) = ... = 0)); if (!ok) { - if (trace) { - *trace << "Aggregate " << typeid(T).name() << ": error\n"; + if (trace_) { + *trace_ << "Aggregate " << typeid(T).name() << ": error\n"; } return std::nullopt; } - if (trace) { - *trace << "Aggregate " << typeid(T).name() << ": success\n"; + if (trace_) { + *trace_ << "Aggregate " << typeid(T).name() << ": success\n"; } return T{std::move(std::get(fields).value())...}; } +namespace { // Extract the fields of a simple aggregate type. template struct Extractable { static_assert(std::is_aggregate_v, "Unsupported child type"); - static auto ExtractImpl(const Tree* tree, Tree::SiblingIterator& it, - Tree::SiblingIterator end, ErrorBuilder* trace) - -> std::optional { + static auto ExtractImpl(NodeExtractor& extractor) -> std::optional { // Compute the corresponding tuple type. using TupleType = decltype(StructReflection::AsTuple(std::declval())); - return ExtractTupleLikeType( - tree, it, end, trace, + return extractor.ExtractTupleLikeType( std::make_index_sequence>(), static_cast(nullptr)); } - static auto Extract(const Tree* tree, Tree::SiblingIterator& it, - Tree::SiblingIterator end, ErrorBuilder* trace) - -> std::optional { + static auto Extract(NodeExtractor& extractor) -> std::optional { static_assert(!HasKindMember, "Missing Id suffix"); - return ExtractImpl(tree, it, end, trace); + return ExtractImpl(extractor); } }; +} // namespace template auto Tree::TryExtractNodeFromChildren( - llvm::iterator_range children, + NodeId node_id, llvm::iterator_range children, ErrorBuilder* trace) const -> std::optional { - auto it = children.begin(); - auto result = Extractable::ExtractImpl(this, it, children.end(), trace); - if (it != children.end()) { + NodeExtractor extractor(this, tokens_, trace, node_id, children); + auto result = Extractable::ExtractImpl(extractor); + if (!extractor.at_end()) { if (trace) { - *trace << "Error: " << node_kind(*it) << " node left unconsumed."; + *trace << "Error: " << node_kind(extractor.ExtractNode()) + << " node left unconsumed."; } return std::nullopt; } @@ -324,9 +419,9 @@ auto Tree::TryExtractNodeFromChildren( } // Manually instantiate Tree::TryExtractNodeFromChildren -#define CARBON_PARSE_NODE_KIND(KindName) \ - template auto Tree::TryExtractNodeFromChildren( \ - llvm::iterator_range children, \ +#define CARBON_PARSE_NODE_KIND(KindName) \ + template auto Tree::TryExtractNodeFromChildren( \ + NodeId node_id, llvm::iterator_range children, \ ErrorBuilder * trace) const -> std::optional; // Also instantiate for `File`, even though it isn't a parse node. @@ -334,7 +429,7 @@ CARBON_PARSE_NODE_KIND(File) #include "toolchain/parse/node_kind.def" auto Tree::ExtractFile() const -> File { - return ExtractNodeFromChildren(roots()); + return ExtractNodeFromChildren(NodeId::Invalid, roots()); } } // namespace Carbon::Parse diff --git a/toolchain/parse/node_ids.h b/toolchain/parse/node_ids.h index dc69358ff839..14f915974149 100644 --- a/toolchain/parse/node_ids.h +++ b/toolchain/parse/node_ids.h @@ -6,6 +6,7 @@ #define CARBON_TOOLCHAIN_PARSE_NODE_IDS_H_ #include "toolchain/base/index_base.h" +#include "toolchain/lex/token_index.h" #include "toolchain/parse/node_kind.h" namespace Carbon::Parse { @@ -110,6 +111,32 @@ struct NodeIdNot : public NodeId { : NodeId(NodeId::InvalidIndex) {} }; +// This class holds the token corresponding to a parse node, and defines the +// expected token kind. The specified token index will always have kind K if the +// enclosing node doesn't have errors. Note that we never try to extract a node +// that has errors, and there are no restrictions on the token kind in that +// case. +template +struct Token { + static constexpr const Lex::TokenKind& Kind = K; + Lex::TokenIndex index; +}; + +// This class holds the token corresponding to a parse node in the case where +// the parse node can correspond to any token. This should only be used when the +// node kind is either not used in a finished tree, such as `Placeholder`, or is +// always invalid, such as `InvalidParse`. +struct AnyToken { + Lex::TokenIndex index; +}; + +// This class holds the token corresponding to a parse node in the case of a +// virtual token. The parse node doesn't actually own the token in this case. +template +struct VirtualToken { + Token token; +}; + // Note that the support for extracting these types using the `Tree::Extract*` // functions is defined in `extract.cpp`. diff --git a/toolchain/parse/node_kind.cpp b/toolchain/parse/node_kind.cpp index d49a8f0d50d7..626ebb9fa80d 100644 --- a/toolchain/parse/node_kind.cpp +++ b/toolchain/parse/node_kind.cpp @@ -4,7 +4,6 @@ #include "toolchain/parse/node_kind.h" -#include "common/check.h" #include "llvm/ADT/StringExtras.h" #include "toolchain/parse/typed_nodes.h" @@ -37,30 +36,10 @@ CARBON_DEFINE_ENUM_CLASS_NAMES(NodeKind) = { #include "toolchain/parse/node_kind.def" }; -auto NodeKind::CheckMatchesTokenKind(Lex::TokenKind token_kind, bool has_error) - -> void { - static constexpr Lex::TokenKind TokenIfValid[] = { -#define CARBON_IF_VALID(LexTokenKind) LexTokenKind -#define CARBON_PARSE_NODE_KIND_WITH_TOKEN(Name, LexTokenKind) \ - Lex::TokenKind::LexTokenKind, +// Check that each typed node defines a `token` member. +#define CARBON_PARSE_NODE_KIND(Name) \ + static_assert(requires(Name node) { node.token; }); #include "toolchain/parse/node_kind.def" - }; - static constexpr Lex::TokenKind TokenIfError[] = { -#define CARBON_IF_VALID(LexTokenKind) Error -#define CARBON_PARSE_NODE_KIND_WITH_TOKEN(Name, LexTokenKind) \ - Lex::TokenKind::LexTokenKind, -#include "toolchain/parse/node_kind.def" - }; - - Lex::TokenKind expected_token_kind = - has_error ? TokenIfError[AsInt()] : TokenIfValid[AsInt()]; - // Error indicates that the kind shouldn't be enforced. - CARBON_CHECK(Lex::TokenKind::Error == expected_token_kind || - token_kind == expected_token_kind) - << "Created parse node with NodeKind " << *this << " and has_error " - << has_error << " for lexical token kind " << token_kind - << ", but expected token kind " << expected_token_kind; -} auto NodeKind::has_bracket() const -> bool { return definition().has_bracket(); diff --git a/toolchain/parse/node_kind.def b/toolchain/parse/node_kind.def index aaa8752ac040..dbb8b9e79805 100644 --- a/toolchain/parse/node_kind.def +++ b/toolchain/parse/node_kind.def @@ -9,47 +9,30 @@ // // Supported x-macros are: // - CARBON_PARSE_NODE_KIND(Name) -// Used as a fallback if other macros are missing. -// - CARBON_PARSE_NODE_KIND_WITH_TOKEN(Name, LexTokenKind) -// Defines a parse node kind and specifies its corresponding token kind. -// - CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Name) -// Defines a parse node for a prefix operator, with the Name as token. -// - CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Name) -// Defines a parse node for an infix operator, with the Name as token. -// - CARBON_PARSE_NODE_KIND_POSTFIX_OPERATOR(Name) -// Defines a parse node for a postfix operator, with the Name as token. -// - CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(Name, LexTokenKind) -// Defines a parse node that corresponds to a token that is a single-token -// literal. The token is wrapped for LexTokenKinds. -// - CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER(Name) -// A token-based modifier. The Name is the TokenKind, and will be appended -// with "Modifier" for the parse kind. +// Used as a fallback if other macros are missing. Also, used directly +// when none of the below category-oriented macros apply. +// - CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Name) +// Defines a parse node for a prefix operator, with the Name as token. +// - CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Name) +// Defines a parse node for an infix operator, with the Name as token. +// - CARBON_PARSE_NODE_KIND_POSTFIX_OPERATOR(Name) +// Defines a parse node for a postfix operator, with the Name as token. +// - CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(Name, LexTokenKind) +// Defines a parse node that corresponds to a token that is a single-token +// literal. The token is wrapped for LexTokenKinds. +// - CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER(Name) +// A token-based modifier. The Name is the TokenKind, and will be appended +// with "Modifier" for the parse kind. // // LexTokenKind indicates the token kind required on a valid node. -// It will be either `kind_name` to indicate it's required even if the node is -// an error, or `CARBON_IF_VALID(kind_name)` to indicate that the kind is -// not enforced on error. A token kind of `Error` may be used if the kind should -// never be enforced; this should only be used when the node is either not in -// trees (`Placeholder`) or is always invalid (such as `InvalidParse`). -// -// This tree represents the subset relationship between these macros, where if a -// specific x-macro isn't defined, it'll fall back to the parent macro. // // Parse nodes are clustered based on language feature. See typed_nodes.h for // the expected tree structure under each node kind. -#if !(defined(CARBON_PARSE_NODE_KIND) || \ - defined(CARBON_PARSE_NODE_KIND_WITH_TOKEN)) +#if !defined(CARBON_PARSE_NODE_KIND) #error "Must define CARBON_PARSE_NODE_KIND family x-macros to use this file." #endif -// The WITH_TOKEN macro will use CARBON_PARSE_NODE_KIND by default when -// undefined. -#ifndef CARBON_PARSE_NODE_KIND_WITH_TOKEN -#define CARBON_PARSE_NODE_KIND_WITH_TOKEN(Name, LexTokenKind) \ - CARBON_PARSE_NODE_KIND(Name) -#endif - // This is expected to be used with something like: // // // Use x-macros to handle modifier cases. @@ -57,7 +40,7 @@ // #define CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Name, ...) #ifndef CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR #define CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Name) \ - CARBON_PARSE_NODE_KIND_WITH_TOKEN(PrefixOperator##Name, Name) + CARBON_PARSE_NODE_KIND(PrefixOperator##Name) #endif // This is expected to be used with something like: @@ -67,7 +50,7 @@ // #define CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Name, ...) #ifndef CARBON_PARSE_NODE_KIND_INFIX_OPERATOR #define CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Name) \ - CARBON_PARSE_NODE_KIND_WITH_TOKEN(InfixOperator##Name, Name) + CARBON_PARSE_NODE_KIND(InfixOperator##Name) #endif // This is expected to be used with something like: @@ -77,7 +60,7 @@ // #define CARBON_PARSE_NODE_KIND_POSTFIX_OPERATOR(Name, ...) #ifndef CARBON_PARSE_NODE_KIND_POSTFIX_OPERATOR #define CARBON_PARSE_NODE_KIND_POSTFIX_OPERATOR(Name) \ - CARBON_PARSE_NODE_KIND_WITH_TOKEN(PostfixOperator##Name, Name) + CARBON_PARSE_NODE_KIND(PostfixOperator##Name) #endif // This is expected to be used with something like: @@ -87,7 +70,7 @@ // #define CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(Name, ...) #ifndef CARBON_PARSE_NODE_KIND_TOKEN_LITERAL #define CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(Name, LexTokenKinds) \ - CARBON_PARSE_NODE_KIND_WITH_TOKEN(Name, LexTokenKinds) + CARBON_PARSE_NODE_KIND(Name) #endif // This is expected to be used with something like: @@ -97,405 +80,144 @@ // #define CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER(Name, ...) #ifndef CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER #define CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER(Name) \ - CARBON_PARSE_NODE_KIND_WITH_TOKEN(Name##Modifier, Name) + CARBON_PARSE_NODE_KIND(Name##Modifier) #endif -// The start of the file. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(FileStart, FileStart) +CARBON_PARSE_NODE_KIND(FileStart) +CARBON_PARSE_NODE_KIND(FileEnd) -// The end of the file. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(FileEnd, FileEnd) +CARBON_PARSE_NODE_KIND(InvalidParse) +CARBON_PARSE_NODE_KIND(InvalidParseStart) +CARBON_PARSE_NODE_KIND(InvalidParseSubtree) -// An invalid parse. Used to balance the parse tree. Always has an error. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(InvalidParse, Error) +CARBON_PARSE_NODE_KIND(Placeholder) -// An invalid subtree. Always has an error. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(InvalidParseStart, Error) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(InvalidParseSubtree, Error) +CARBON_PARSE_NODE_KIND(EmptyDecl) -// A placeholder node to be replaced; it will never exist in a valid parse tree. -// Its token kind is not enforced even when valid. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(Placeholder, Error) +CARBON_PARSE_NODE_KIND(IdentifierName) -// An empty declaration, such as `;`. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(EmptyDecl, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(IdentifierNameExpr) -// An identifier name in a non-expression context, such as a declaration. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(IdentifierName, CARBON_IF_VALID(Identifier)) +CARBON_PARSE_NODE_KIND(SelfValueName) +CARBON_PARSE_NODE_KIND(SelfValueNameExpr) +CARBON_PARSE_NODE_KIND(SelfTypeNameExpr) -// An identifier name in an expression context. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(IdentifierNameExpr, Identifier) +CARBON_PARSE_NODE_KIND(BaseName) +CARBON_PARSE_NODE_KIND(PackageExpr) +CARBON_PARSE_NODE_KIND(PackageName) +CARBON_PARSE_NODE_KIND(LibraryName) -// The `self` value and `Self` type identifier keywords. Typically of the form -// `self: Self`. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(SelfValueName, SelfValueIdentifier) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(SelfValueNameExpr, SelfValueIdentifier) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(SelfTypeNameExpr, SelfTypeIdentifier) +CARBON_PARSE_NODE_KIND(PackageIntroducer) +CARBON_PARSE_NODE_KIND(PackageDecl) -// The `base` value keyword, introduced by `base: B`. Typically referenced in -// an expression, as in `x.base` or `{.base = ...}`, but can also be used as a -// declared name, as in `{.base: partial B}`. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(BaseName, Base) +CARBON_PARSE_NODE_KIND(ImportIntroducer) +CARBON_PARSE_NODE_KIND(ImportDecl) -// The `package` keyword in an expression. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(PackageExpr, Package) +CARBON_PARSE_NODE_KIND(DefaultLibrary) +CARBON_PARSE_NODE_KIND(LibraryIntroducer) +CARBON_PARSE_NODE_KIND(LibraryDecl) -// ---------------------------------------------------------------------------- +CARBON_PARSE_NODE_KIND(LibrarySpecifier) -// The comments below follow this pattern: -// -// // Descriptive heading: -// // Child1 -// // Child2 -// // Parent -// -// In this, `Child1`, `Child2`, and `Parent` are all kinds of parse nodes, which -// are then defined using the `CARBON_PARSE_NODE_KIND_*` macros. They are -// written in postorder, with the indentation showing the tree structure. See -// tree.h for more information. -// -// A parse node kind may be preceded by: -// - `_optional_` if this node (or nodes) may be present or omitted in valid -// parses, depending on which tokens are in the source code. -// - `_repeated_` if this node (or nodes) may be repeated or omitted in valid -// parses, depending on which tokens are in the source code. -// - `_external_:` if this node is the child of multiple kinds of nodes and -// is documented separately. -// -// There is generally a close correspondence between handling of tokens and the -// creation of non-external nodes in a given block. +CARBON_PARSE_NODE_KIND(NameQualifier) -// ---------------------------------------------------------------------------- +CARBON_PARSE_NODE_KIND(ExportIntroducer) +CARBON_PARSE_NODE_KIND(ExportDecl) -// The name of a package or library for `package`, `import`, and `library`. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(PackageName, Identifier) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(LibraryName, StringLiteral) +CARBON_PARSE_NODE_KIND(NamespaceStart) +CARBON_PARSE_NODE_KIND(Namespace) -// `package`: -// PackageIntroducer -// _repeated_ _external_: modifier -// _optional_ _external_: PackageName -// _optional_ _external_: LibrarySpecifier -// PackageDecl -CARBON_PARSE_NODE_KIND_WITH_TOKEN(PackageIntroducer, Package) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(PackageDecl, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(CodeBlockStart) +CARBON_PARSE_NODE_KIND(CodeBlock) -// `import`: -// ImportIntroducer -// _repeated_ _external_: modifier -// _optional_ _external_: PackageName -// _optional_ _external_: LibrarySpecifier -// ImportDecl -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ImportIntroducer, Import) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ImportDecl, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(FunctionIntroducer) +CARBON_PARSE_NODE_KIND(ReturnType) +CARBON_PARSE_NODE_KIND(FunctionDefinitionStart) +CARBON_PARSE_NODE_KIND(FunctionDefinition) +CARBON_PARSE_NODE_KIND(FunctionDecl) +CARBON_PARSE_NODE_KIND(BuiltinFunctionDefinitionStart) +CARBON_PARSE_NODE_KIND(BuiltinName) +CARBON_PARSE_NODE_KIND(BuiltinFunctionDefinition) -// `library` as declaration: -// LibraryIntroducer -// _repeated_ _external_: modifier -// DefaultLibrary or _external_: LibraryName -// LibraryDecl -CARBON_PARSE_NODE_KIND_WITH_TOKEN(DefaultLibrary, Default) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(LibraryIntroducer, Library) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(LibraryDecl, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(AliasIntroducer) +CARBON_PARSE_NODE_KIND(AliasInitializer) +CARBON_PARSE_NODE_KIND(Alias) -// `library` in `package` or `import`: -// _external_: LibraryName or DefaultLibrary -// LibrarySpecifier -CARBON_PARSE_NODE_KIND_WITH_TOKEN(LibrarySpecifier, Library) +CARBON_PARSE_NODE_KIND(TuplePatternStart) +CARBON_PARSE_NODE_KIND(PatternListComma) +CARBON_PARSE_NODE_KIND(TuplePattern) -// Declaration names. -// -// _repeated_: NameQualifier -// _external_: IdentifierName -// _optional_ _external_: ImplicitParamList -// _optional_ _external_: TuplePattern -// _declaration name_ -// -// _external_: IdentifierName -// _optional_ _external_: ImplicitParamList -// _optional_ _external_: TuplePattern -// NameQualifier -CARBON_PARSE_NODE_KIND_WITH_TOKEN(NameQualifier, CARBON_IF_VALID(Period)) +CARBON_PARSE_NODE_KIND(ImplicitParamListStart) +CARBON_PARSE_NODE_KIND(ImplicitParamList) -// `export`: -// ExportIntroducer -// _external_: _declaration name_ -// ExportDecl -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ExportIntroducer, Export) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ExportDecl, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(ArrayExprStart) +CARBON_PARSE_NODE_KIND(ArrayExprSemi) +CARBON_PARSE_NODE_KIND(ArrayExpr) -// `namespace`: -// NamespaceStart -// _repeated_ _external_: modifier -// _external_: _declaration name_ -// Namespace -CARBON_PARSE_NODE_KIND_WITH_TOKEN(NamespaceStart, Namespace) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(Namespace, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(BindingPattern) +CARBON_PARSE_NODE_KIND(CompileTimeBindingPattern) +CARBON_PARSE_NODE_KIND(Addr) +CARBON_PARSE_NODE_KIND(Template) -// A code block: -// CodeBlockStart -// _repeated_ _external_: statement -// CodeBlock -CARBON_PARSE_NODE_KIND_WITH_TOKEN(CodeBlockStart, - CARBON_IF_VALID(OpenCurlyBrace)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(CodeBlock, CARBON_IF_VALID(CloseCurlyBrace)) +CARBON_PARSE_NODE_KIND(LetIntroducer) +CARBON_PARSE_NODE_KIND(LetInitializer) +CARBON_PARSE_NODE_KIND(LetDecl) -// `fn` declarations start with a function signature: -// -// FunctionIntroducer -// _repeated_ _external_: modifier -// _external_: _declaration name_ -// ReturnType -// _function signature_ -// -// There are three forms of function declaration: -// -// _function signature_ -// FunctionDecl -// -// _function signature_ -// FunctionDefinitionStart -// _repeated_ _external_: statement -// FunctionDefinition -// -// _function signature_ -// BuiltinFunctionDefinitionStart -// BuiltinName -// BuiltinFunctionDefinition -CARBON_PARSE_NODE_KIND_WITH_TOKEN(FunctionIntroducer, Fn) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ReturnType, MinusGreater) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(FunctionDefinitionStart, OpenCurlyBrace) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(FunctionDefinition, CloseCurlyBrace) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(FunctionDecl, CARBON_IF_VALID(Semi)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(BuiltinFunctionDefinitionStart, Equal) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(BuiltinName, StringLiteral) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(BuiltinFunctionDefinition, - CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(VariableIntroducer) +CARBON_PARSE_NODE_KIND(ReturnedModifier) +CARBON_PARSE_NODE_KIND(VariableInitializer) +CARBON_PARSE_NODE_KIND(VariableDecl) -// `alias`: -// AliasIntroducer -// _repeated_ _external_: modifier -// _external_: _declaration name_ -// AliasInitializer -// _external_: expression -// Alias -CARBON_PARSE_NODE_KIND_WITH_TOKEN(AliasIntroducer, Alias) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(AliasInitializer, Equal) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(Alias, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(ExprStatement) -// A tuple pattern: -// TuplePatternStart -// _external_: [Generic]BindingPattern -// PatternListComma -// _repeated_ -// TuplePattern -// -// Patterns and PatternListComma may repeat with PatternListComma as a -// separator. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(TuplePatternStart, OpenParen) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(PatternListComma, Comma) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(TuplePattern, CloseParen) +CARBON_PARSE_NODE_KIND(BreakStatementStart) +CARBON_PARSE_NODE_KIND(BreakStatement) -// An implicit parameter list: -// ImplicitParamListStart -// _external_: [Generic]BindingPattern -// PatternListComma -// _repeated_ -// ImplicitParamList -// -// Patterns and PatternListComma may repeat with PatternListComma as a -// separator. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ImplicitParamListStart, OpenSquareBracket) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ImplicitParamList, CloseSquareBracket) +CARBON_PARSE_NODE_KIND(ContinueStatementStart) +CARBON_PARSE_NODE_KIND(ContinueStatement) -// An array type, such as `[i32; 3]` or `[i32;]`: -// ArrayExprStart -// _external_: type expression -// ArrayExprSemi -// _optional_ _external_: expression -// ArrayExpr -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ArrayExprStart, OpenSquareBracket) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ArrayExprSemi, CARBON_IF_VALID(Semi)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ArrayExpr, CloseSquareBracket) +CARBON_PARSE_NODE_KIND(ReturnStatementStart) +CARBON_PARSE_NODE_KIND(ReturnVarModifier) +CARBON_PARSE_NODE_KIND(ReturnStatement) -// A binding pattern, such as `name: Type`: -// IdentifierName or SelfValueName -// _external_: type expression -// [Generic]BindingPattern -// _optional_ Addr -// _optional_ Template -CARBON_PARSE_NODE_KIND_WITH_TOKEN(BindingPattern, CARBON_IF_VALID(Colon)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(CompileTimeBindingPattern, ColonExclaim) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(Addr, Addr) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(Template, Template) +CARBON_PARSE_NODE_KIND(ForHeaderStart) +CARBON_PARSE_NODE_KIND(ForIn) +CARBON_PARSE_NODE_KIND(ForHeader) +CARBON_PARSE_NODE_KIND(ForStatement) -// `let` declarations, including associated constant declarations: -// LetIntroducer -// _repeated_ _external_: modifier -// _external_: BindingPattern or TuplePattern -// LetInitializer -// _external_: expression -// _optional_ -// LetDecl -// -// Modifier keywords only appear for `let` declarations, not `let` statements. -// -// The LetInitializer and following expression are paired: either both will be -// present, or neither will. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(LetIntroducer, Let) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(LetInitializer, Equal) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(LetDecl, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(IfConditionStart) +CARBON_PARSE_NODE_KIND(IfCondition) +CARBON_PARSE_NODE_KIND(IfStatementElse) +CARBON_PARSE_NODE_KIND(IfStatement) -// `var` and `returned var`: -// VariableIntroducer -// _repeated_ _external_: modifier -// _optional_ ReturnedModifier -// _external_: BindingPattern or TuplePattern -// VariableInitializer -// _external_: expression -// _optional_ -// VariableDecl -// -// Access and declaration modifier keywords only appear for `var` declarations, -// whereas the returned modifier only appears on `var` statements. -// -// The VariableInitializer and following expression are paired: either both will -// be present, or neither will. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(VariableIntroducer, CARBON_IF_VALID(Var)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ReturnedModifier, Returned) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(VariableInitializer, Equal) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(VariableDecl, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(WhileConditionStart) +CARBON_PARSE_NODE_KIND(WhileCondition) +CARBON_PARSE_NODE_KIND(WhileStatement) -// An expression statement: -// _external_: expression -// ExprStatement -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ExprStatement, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(IndexExprStart) +CARBON_PARSE_NODE_KIND(IndexExpr) -// `break`: -// BreakStatementStart -// BreakStatement -CARBON_PARSE_NODE_KIND_WITH_TOKEN(BreakStatementStart, Break) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(BreakStatement, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(ParenExprStart) +CARBON_PARSE_NODE_KIND(ParenExpr) -// `continue`: -// ContinueStatementStart -// ContinueStatement -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ContinueStatementStart, Continue) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ContinueStatement, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(TupleLiteralStart) +CARBON_PARSE_NODE_KIND(TupleLiteralComma) +CARBON_PARSE_NODE_KIND(TupleLiteral) -// `return`: -// ReturnStatementStart -// _optional_ ReturnVarModifier or _external_: expression -// ReturnStatement -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ReturnStatementStart, Return) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ReturnVarModifier, Var) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ReturnStatement, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(CallExprStart) +CARBON_PARSE_NODE_KIND(CallExprComma) +CARBON_PARSE_NODE_KIND(CallExpr) -// `for`: -// ForHeaderStart -// VariableIntroducer -// _external_: BindingPattern -// ForIn -// _external_: expression -// ForHeader -// _external_: CodeBlock -// ForStatement -// -// Versus a normal `var`, ForIn replaces VariableDecl. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ForHeaderStart, CARBON_IF_VALID(OpenParen)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ForIn, CARBON_IF_VALID(In)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ForHeader, CARBON_IF_VALID(CloseParen)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ForStatement, For) +CARBON_PARSE_NODE_KIND(MemberAccessExpr) -// `if` statement + `else`: -// IfConditionStart -// _external_: expression -// IfCondition -// _external_: CodeBlock -// IfStatementElse -// _external_: CodeBlock or IfStatement -// IfStatement -// -// IfStatementElse and the following node are optional based on `else` presence. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(IfConditionStart, CARBON_IF_VALID(OpenParen)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(IfCondition, CARBON_IF_VALID(CloseParen)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(IfStatementElse, Else) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(IfStatement, If) +CARBON_PARSE_NODE_KIND(PointerMemberAccessExpr) -// `while`: -// WhileConditionStart -// _external_: expression -// WhileCondition -// _external_: CodeBlock -// WhileStatement -CARBON_PARSE_NODE_KIND_WITH_TOKEN(WhileConditionStart, - CARBON_IF_VALID(OpenParen)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(WhileCondition, CARBON_IF_VALID(CloseParen)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(WhileStatement, While) - -// Index expressions, such as `a[1]`: -// _external_: expression -// IndexExprStart -// _external_: expression -// IndexExpr -CARBON_PARSE_NODE_KIND_WITH_TOKEN(IndexExprStart, OpenSquareBracket) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(IndexExpr, CloseSquareBracket) - -// Parenthesized single expressions, such as `(2)`: -// ParenExprStart -// _external_: expression -// ParenExpr -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ParenExprStart, OpenParen) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ParenExpr, CloseParen) - -// Tuples, such as `(1, 2)`: -// TupleLiteralStart -// _external_: expression -// TupleLiteralComma -// _repeated_ -// TupleLiteral -// -// Expressions and TupleLiteralComma may repeat with TupleLiteralComma as a -// separator. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(TupleLiteralStart, OpenParen) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(TupleLiteralComma, Comma) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(TupleLiteral, CloseParen) - -// Call expressions, such as `a()`: -// _external_: expression -// CallExprStart -// _external_: expression -// CallExprComma -// _repeated_ -// CallExpr -// -// Exprs and CallExprComma may repeat with CallExprComma as a separator. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(CallExprStart, OpenParen) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(CallExprComma, Comma) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(CallExpr, CloseParen) - -// A member access expression, such as `a.b` or -// `GetObject().(Interface.member)`: -// _external_: lhs expression -// _external_: rhs expression -// MemberAccessExpr -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MemberAccessExpr, Period) - -// A pointer member access expression, such as `a->b` or -// `GetObject()->(Interface.member)`: -// _external_: lhs expression -// _external_: rhs expression -// PointerMemberAccessExpr -CARBON_PARSE_NODE_KIND_WITH_TOKEN(PointerMemberAccessExpr, MinusGreater) - -// A value literal. CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(BoolLiteralFalse, False) CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(BoolLiteralTrue, True) CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(IntLiteral, IntLiteral) CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(RealLiteral, RealLiteral) CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(StringLiteral, StringLiteral) -// A type literal. CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(BoolTypeLiteral, Bool) CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(IntTypeLiteral, IntTypeLiteral) CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(UnsignedIntTypeLiteral, @@ -505,9 +227,6 @@ CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(StringTypeLiteral, StringTypeLiteral) CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(TypeTypeLiteral, Type) CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(AutoTypeLiteral, Auto) -// A prefix operator, such as `not`: -// _external_: expression -// PrefixOperator CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Amp) CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Caret) CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Const) @@ -517,10 +236,6 @@ CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(MinusMinus) CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(PlusPlus) CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Star) -// An infix operator, such as `+`: -// _external_: lhs expression -// _external_: rhs expression -// InfixOperator CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Amp) CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(AmpEqual) CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(As) @@ -551,68 +266,26 @@ CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(SlashEqual) CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Star) CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(StarEqual) -// A postfix operator, currently only `*`: -// _external_: expression -// PostfixOperator CARBON_PARSE_NODE_KIND_POSTFIX_OPERATOR(Star) -// A short-circuiting infix operator, such as `and`: -// _external_: expression -// ShortCircuitOperand(And|Or) -// _external_: expression -// ShortCircuitOperand(And|Or) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ShortCircuitOperandAnd, And) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ShortCircuitOperandOr, Or) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ShortCircuitOperatorAnd, And) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ShortCircuitOperatorOr, Or) +CARBON_PARSE_NODE_KIND(ShortCircuitOperandAnd) +CARBON_PARSE_NODE_KIND(ShortCircuitOperandOr) +CARBON_PARSE_NODE_KIND(ShortCircuitOperatorAnd) +CARBON_PARSE_NODE_KIND(ShortCircuitOperatorOr) -// `if` expression + `then` + `else`: -// _external_: expression -// IfExprIf -// _external_: expression -// IfExprThen -// _external_: expression -// IfExprElse -CARBON_PARSE_NODE_KIND_WITH_TOKEN(IfExprIf, If) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(IfExprThen, Then) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(IfExprElse, CARBON_IF_VALID(Else)) +CARBON_PARSE_NODE_KIND(IfExprIf) +CARBON_PARSE_NODE_KIND(IfExprThen) +CARBON_PARSE_NODE_KIND(IfExprElse) -// Struct literals, such as `{.a = 0}`: -// StructLiteralStart -// _external_: IdentifierName or BaseName -// StructFieldDesignator -// _external_: expression -// StructField -// StructComma -// _repeated_ -// StructLiteral -// -// Struct type literals, such as `{.a: i32}`: -// StructTypeLiteralStart -// _external_: IdentifierName or BaseName -// StructFieldDesignator -// _external_: type expression -// StructTypeField -// StructComma -// _repeated_ -// StructTypeLiteral -// -// Elements (StructField and StructTypeField, respectively) and StructComma -// may repeat with StructComma as a separator. -// -// When a valid StructTypeField or StructField cannot be formed, elements -// may be replaced by InvalidParse, which may have a preceding sibling -// StructFieldDesignator if one was successfully parsed. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(StructLiteralStart, OpenCurlyBrace) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(StructTypeLiteralStart, OpenCurlyBrace) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(StructFieldDesignator, Period) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(StructField, Equal) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(StructTypeField, Colon) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(StructComma, Comma) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(StructLiteral, CloseCurlyBrace) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(StructTypeLiteral, CloseCurlyBrace) +CARBON_PARSE_NODE_KIND(StructLiteralStart) +CARBON_PARSE_NODE_KIND(StructTypeLiteralStart) +CARBON_PARSE_NODE_KIND(StructFieldDesignator) +CARBON_PARSE_NODE_KIND(StructField) +CARBON_PARSE_NODE_KIND(StructTypeField) +CARBON_PARSE_NODE_KIND(StructComma) +CARBON_PARSE_NODE_KIND(StructLiteral) +CARBON_PARSE_NODE_KIND(StructTypeLiteral) -// Various modifiers. These are all a single token. CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER(Abstract) CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER(Base) CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER(Default) @@ -625,186 +298,59 @@ CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER(Private) CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER(Protected) CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER(Virtual) -// `class`: -// ClassIntroducer -// _repeated_ _external_: modifier -// _external_: _declaration name_ -// _optional_ _external_: ImplicitParamList -// _optional_ _external_: TuplePattern -// ClassDefinitionStart -// _external_: declarations -// ClassDefinition -// -// The above is the structure for a definition; for a declaration, -// ClassDefinitionStart and later nodes are removed and replaced by -// ClassDecl. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ClassIntroducer, Class) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ClassDefinitionStart, OpenCurlyBrace) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ClassDefinition, CloseCurlyBrace) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ClassDecl, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(ClassIntroducer) +CARBON_PARSE_NODE_KIND(ClassDefinitionStart) +CARBON_PARSE_NODE_KIND(ClassDefinition) +CARBON_PARSE_NODE_KIND(ClassDecl) -// `adapt`: -// AdaptIntroducer -// _repeated_ _external_: modifier -// _external_: expression -// AdaptDecl -CARBON_PARSE_NODE_KIND_WITH_TOKEN(AdaptIntroducer, Adapt) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(AdaptDecl, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(AdaptIntroducer) +CARBON_PARSE_NODE_KIND(AdaptDecl) -// `base`: -// BaseIntroducer -// _repeated_ _external_: modifier -// BaseColon -// _external_: expression -// BaseDecl -CARBON_PARSE_NODE_KIND_WITH_TOKEN(BaseIntroducer, Base) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(BaseColon, Colon) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(BaseDecl, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(BaseIntroducer) +CARBON_PARSE_NODE_KIND(BaseColon) +CARBON_PARSE_NODE_KIND(BaseDecl) -// `interface`: -// InterfaceIntroducer -// _repeated_ _external_: modifier -// _external_: _declaration name_ -// _optional_ _external_: ImplicitParamList -// _optional_ _external_: TuplePattern -// InterfaceDefinitionStart -// _external_: declarations -// InterfaceDefinition -// -// The above is the structure for a definition; for a declaration, -// InterfaceDefinitionStart and later nodes are removed and replaced by -// InterfaceDecl. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(InterfaceIntroducer, Interface) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(InterfaceDefinitionStart, OpenCurlyBrace) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(InterfaceDefinition, CloseCurlyBrace) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(InterfaceDecl, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(InterfaceIntroducer) +CARBON_PARSE_NODE_KIND(InterfaceDefinitionStart) +CARBON_PARSE_NODE_KIND(InterfaceDefinition) +CARBON_PARSE_NODE_KIND(InterfaceDecl) -// `impl ... as`: -// ImplIntroducer -// _repeated_ _external_: modifier -// _optional_ _external_: ImplForall -// _optional_ _external_: expression -// _external_: DefaultSelfImplAs or TypeImplAs -// _external_: expression -// ImplDefinitionStart -// _external_: declarations -// ImplDefinition -// -// The above is the structure for a definition; for a declaration, -// ImplDefinitionStart and later nodes are removed and replaced by -// ImplDecl. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ImplIntroducer, Impl) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ImplDefinitionStart, OpenCurlyBrace) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ImplDefinition, CloseCurlyBrace) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ImplDecl, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(ImplIntroducer) +CARBON_PARSE_NODE_KIND(ImplDefinitionStart) +CARBON_PARSE_NODE_KIND(ImplDefinition) +CARBON_PARSE_NODE_KIND(ImplDecl) +CARBON_PARSE_NODE_KIND(ImplForall) +CARBON_PARSE_NODE_KIND(TypeImplAs) +CARBON_PARSE_NODE_KIND(DefaultSelfImplAs) -// `forall ...`: -// _external_: ImplicitParamList -// ImplForall -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ImplForall, Forall) +CARBON_PARSE_NODE_KIND(NamedConstraintIntroducer) +CARBON_PARSE_NODE_KIND(NamedConstraintDefinitionStart) +CARBON_PARSE_NODE_KIND(NamedConstraintDefinition) +CARBON_PARSE_NODE_KIND(NamedConstraintDecl) -// `... as`: -// _external_: expression -// TypeImplAs -CARBON_PARSE_NODE_KIND_WITH_TOKEN(TypeImplAs, As) -// `as` without a type before it -CARBON_PARSE_NODE_KIND_WITH_TOKEN(DefaultSelfImplAs, As) +CARBON_PARSE_NODE_KIND(ChoiceIntroducer) +CARBON_PARSE_NODE_KIND(ChoiceDefinitionStart) +CARBON_PARSE_NODE_KIND(ChoiceDefinition) +CARBON_PARSE_NODE_KIND(ChoiceAlternativeListComma) -// `constraint`: -// NamedConstraintIntroducer -// _repeated_ _external_: modifier -// _external_: _declaration name_ -// _optional_ _external_: ImplicitParamList -// _optional_ _external_: TuplePattern -// NamedConstraintDefinitionStart -// _external_: declarations -// NamedConstraintDefinition -// -// The above is the structure for a definition; for a declaration, -// NamedConstraintDefinitionStart and later nodes are removed and replaced by -// NamedConstraintDecl. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(NamedConstraintIntroducer, Constraint) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(NamedConstraintDefinitionStart, - OpenCurlyBrace) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(NamedConstraintDefinition, CloseCurlyBrace) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(NamedConstraintDecl, CARBON_IF_VALID(Semi)) - -// `choice`: -// ChoiceIntroducer -// _external_: _declaration name_ -// ChoiceDefinitionStart -// _optional_ _external_: ChoiceAlternativeList -// ChoiceDefinition -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ChoiceIntroducer, Choice) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ChoiceDefinitionStart, - CARBON_IF_VALID(OpenCurlyBrace)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ChoiceDefinition, - CARBON_IF_VALID(CloseCurlyBrace)) - -// Choice alternative list: -// _external_: IdentifierName -// _optional_ _external_ : TuplePattern -// _optional_: ChoiceAlternativeListComma -// _repeated_ -// ChoiceAlternativeList -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ChoiceAlternativeListComma, Comma) - -// `match`: -// MatchIntroducer -// MatchConditionStart -// _external_: expression -// MatchCondition -// MatchStatementStart -// _repeated_ _external_: MatchCase -// _optional_ _external_: MatchStatementDefault -// MatchStatement -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchIntroducer, Match) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchConditionStart, - CARBON_IF_VALID(OpenParen)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchCondition, CARBON_IF_VALID(CloseParen)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchStatementStart, - CARBON_IF_VALID(OpenCurlyBrace)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchStatement, - CARBON_IF_VALID(CloseCurlyBrace)) - -// `case`: -// MatchCaseIntroducer -// _external_: Pattern -// MatchCaseGuardIntroducer -// MatchCaseGuardStart -// _external_: expression -// MatchCaseGuard -// MatchCaseEqualGreater -// MatchCaseStart -// _repeated_ _external_: statement -// MatchCase -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchCaseIntroducer, Case) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchCaseGuardIntroducer, If) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchCaseGuardStart, - CARBON_IF_VALID(OpenParen)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchCaseGuard, CARBON_IF_VALID(CloseParen)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchCaseEqualGreater, - CARBON_IF_VALID(EqualGreater)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchCaseStart, - CARBON_IF_VALID(OpenCurlyBrace)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchCase, CARBON_IF_VALID(CloseCurlyBrace)) - -// `default`: -// MatchDefaultIntroducer -// MatchDefaultEqualGreater -// MatchDefaultStart -// _repeated_ _external_: statement -// MatchDefault -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchDefaultIntroducer, Default) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchDefaultEqualGreater, - CARBON_IF_VALID(EqualGreater)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchDefaultStart, - CARBON_IF_VALID(OpenCurlyBrace)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchDefault, - CARBON_IF_VALID(CloseCurlyBrace)) +CARBON_PARSE_NODE_KIND(MatchIntroducer) +CARBON_PARSE_NODE_KIND(MatchConditionStart) +CARBON_PARSE_NODE_KIND(MatchCondition) +CARBON_PARSE_NODE_KIND(MatchStatementStart) +CARBON_PARSE_NODE_KIND(MatchStatement) +CARBON_PARSE_NODE_KIND(MatchCaseIntroducer) +CARBON_PARSE_NODE_KIND(MatchCaseGuardIntroducer) +CARBON_PARSE_NODE_KIND(MatchCaseGuardStart) +CARBON_PARSE_NODE_KIND(MatchCaseGuard) +CARBON_PARSE_NODE_KIND(MatchCaseEqualGreater) +CARBON_PARSE_NODE_KIND(MatchCaseStart) +CARBON_PARSE_NODE_KIND(MatchCase) +CARBON_PARSE_NODE_KIND(MatchDefaultIntroducer) +CARBON_PARSE_NODE_KIND(MatchDefaultEqualGreater) +CARBON_PARSE_NODE_KIND(MatchDefaultStart) +CARBON_PARSE_NODE_KIND(MatchDefault) #undef CARBON_PARSE_NODE_KIND -#undef CARBON_PARSE_NODE_KIND_WITH_TOKEN #undef CARBON_PARSE_NODE_KIND_INFIX_OPERATOR #undef CARBON_PARSE_NODE_KIND_POSTFIX_OPERATOR #undef CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR diff --git a/toolchain/parse/tree.cpp b/toolchain/parse/tree.cpp index 2d15f9bc0613..f610f87cc6e7 100644 --- a/toolchain/parse/tree.cpp +++ b/toolchain/parse/tree.cpp @@ -170,12 +170,12 @@ auto Tree::Print(llvm::raw_ostream& output, bool preorder) const -> void { output << " ]\n"; } -static auto TestExtract(const Tree* tree, NodeId node_id, NodeKind kind, - ErrorBuilder* trace) -> bool { +auto Tree::VerifyExtract(NodeId node_id, NodeKind kind, + ErrorBuilder* trace) const -> bool { switch (kind) { #define CARBON_PARSE_NODE_KIND(Name) \ case NodeKind::Name: \ - return tree->VerifyExtractAs(node_id, trace).has_value(); + return VerifyExtractAs(node_id, trace).has_value(); #include "toolchain/parse/node_kind.def" } } @@ -200,12 +200,12 @@ auto Tree::Verify() const -> ErrorOr { // Without this code, a 10 mloc test case of lex & parse takes // 4.129 s ± 0.041 s. With this additional verification, it takes // 5.768 s ± 0.036 s. - if (!n_impl.has_error && !TestExtract(this, n, n_impl.kind, nullptr)) { + if (!n_impl.has_error && !VerifyExtract(n, n_impl.kind, nullptr)) { ErrorBuilder trace; trace << llvm::formatv( "NodeId #{0} couldn't be extracted as a {1}. Trace:\n", n, n_impl.kind); - TestExtract(this, n, n_impl.kind, &trace); + VerifyExtract(n, n_impl.kind, &trace); return trace; } @@ -273,10 +273,10 @@ auto Tree::Verify() const -> ErrorOr { } // Validate the roots, ensures Tree::ExtractFile() doesn't CHECK-fail. - if (!TryExtractNodeFromChildren(roots(), nullptr)) { + if (!TryExtractNodeFromChildren(NodeId::Invalid, roots(), nullptr)) { ErrorBuilder trace; trace << "Roots of tree couldn't be extracted as a `File`. Trace:\n"; - TryExtractNodeFromChildren(roots(), &trace); + TryExtractNodeFromChildren(NodeId::Invalid, roots(), &trace); return trace; } diff --git a/toolchain/parse/tree.h b/toolchain/parse/tree.h index e80cf7117745..b7017e12307e 100644 --- a/toolchain/parse/tree.h +++ b/toolchain/parse/tree.h @@ -255,7 +255,8 @@ class Tree : public Printable { // Extract a `File` object representing the parse tree for the whole file. // #include "toolchain/parse/typed_nodes.h" to get the definition of `File` - // and the types representing its children nodes. + // and the types representing its children nodes. This is implemented in + // extract.cpp. auto ExtractFile() const -> File; // Converts this node_id to a typed node of a specified type, if it is a valid @@ -276,14 +277,9 @@ class Tree : public Printable { // debugger. auto Verify() const -> ErrorOr; - // Like ExtractAs(), but malformed tree errors are not fatal. Should only be - // used by `Verify()`. - template - auto VerifyExtractAs(NodeId node_id, ErrorBuilder* trace) const - -> std::optional; - private: friend class Context; + friend class TypedNodesTestPeer; template struct ConvertTo; @@ -342,6 +338,24 @@ class Tree : public Printable { static_assert(sizeof(NodeImpl) == 12, "Unexpected size of node implementation!"); + // Like ExtractAs(), but malformed tree errors are not fatal. Should only be + // used by `Verify()` or by tests. + template + auto VerifyExtractAs(NodeId node_id, ErrorBuilder* trace) const + -> std::optional; + + // Wrapper around `VerifyExtractAs` to dispatch based on a runtime node kind. + // Returns true if extraction was successful. + auto VerifyExtract(NodeId node_id, NodeKind kind, ErrorBuilder* trace) const + -> bool; + + // Sets the kind of a node. This is intended to allow putting the tree into a + // state where verification can fail, in order to make the failure path of + // `Verify` testable. + auto SetNodeKindForTesting(NodeId node_id, NodeKind kind) -> void { + node_impls_[node_id.index].kind = kind; + } + // Prints a single node for Print(). Returns true when preorder and there are // children. auto PrintNode(llvm::raw_ostream& output, NodeId n, int depth, @@ -349,16 +363,17 @@ class Tree : public Printable { // Extract a node of type `T` from a sibling range. This is expected to // consume the complete sibling range. Malformed tree errors are written - // to `*trace`, if `trace != nullptr`. + // to `*trace`, if `trace != nullptr`. This is implemented in extract.cpp. template auto TryExtractNodeFromChildren( - llvm::iterator_range children, + NodeId node_id, llvm::iterator_range children, ErrorBuilder* trace) const -> std::optional; // Extract a node of type `T` from a sibling range. This is expected to // consume the complete sibling range. Malformed tree errors are fatal. template auto ExtractNodeFromChildren( + NodeId node_id, llvm::iterator_range children) const -> T; // Depth-first postorder sequence of node implementation data. @@ -476,12 +491,13 @@ class Tree::SiblingIterator template auto Tree::ExtractNodeFromChildren( - llvm::iterator_range children) const -> T { - auto result = TryExtractNodeFromChildren(children, nullptr); + NodeId node_id, llvm::iterator_range children) const + -> T { + auto result = TryExtractNodeFromChildren(node_id, children, nullptr); if (!result.has_value()) { // On error try again, this time capturing a trace. ErrorBuilder trace; - TryExtractNodeFromChildren(children, &trace); + TryExtractNodeFromChildren(node_id, children, &trace); CARBON_FATAL() << "Malformed parse node:\n" << static_cast(trace).message(); } @@ -495,7 +511,7 @@ auto Tree::ExtractAs(NodeId node_id) const -> std::optional { return std::nullopt; } - return ExtractNodeFromChildren(children(node_id)); + return ExtractNodeFromChildren(node_id, children(node_id)); } template @@ -503,10 +519,14 @@ auto Tree::VerifyExtractAs(NodeId node_id, ErrorBuilder* trace) const -> std::optional { static_assert(HasKindMember, "Not a parse node type"); if (!IsValid(node_id)) { + if (trace) { + *trace << "VerifyExtractAs error: wrong kind " << node_kind(node_id) + << ", expected " << T::Kind << "\n"; + } return std::nullopt; } - return TryExtractNodeFromChildren(children(node_id), trace); + return TryExtractNodeFromChildren(node_id, children(node_id), trace); } template @@ -517,7 +537,7 @@ auto Tree::Extract(IdT id) const } using T = typename NodeForId::TypedNode; - return ExtractNodeFromChildren(children(id)); + return ExtractNodeFromChildren(id, children(id)); } template diff --git a/toolchain/parse/typed_nodes.h b/toolchain/parse/typed_nodes.h index 319678cd0ee8..66cc5e6a2dfd 100644 --- a/toolchain/parse/typed_nodes.h +++ b/toolchain/parse/typed_nodes.h @@ -7,6 +7,7 @@ #include +#include "toolchain/lex/token_kind.h" #include "toolchain/parse/node_ids.h" #include "toolchain/parse/node_kind.h" @@ -27,11 +28,13 @@ template using CommaSeparatedList = llvm::SmallVector>; // This class provides a shorthand for defining parse node kinds for leaf nodes. -template struct LeafNode { static constexpr auto Kind = KindT.Define({.category = Category, .child_count = 0}); + + TokenKind token; }; // ---------------------------------------------------------------------------- @@ -77,11 +80,11 @@ struct LeafNode { // An invalid parse. Used to balance the parse tree. This type is here only to // ensure we have a type for each parse node kind. This node kind always has an // error, so can never be extracted. -using InvalidParse = - LeafNode; +using InvalidParse = LeafNode; // An invalid subtree. Always has an error so can never be extracted. -using InvalidParseStart = LeafNode; +using InvalidParseStart = LeafNode; struct InvalidParseSubtree { static constexpr auto Kind = NodeKind::InvalidParseSubtree.Define( {.category = NodeCategory::Decl, @@ -89,48 +92,56 @@ struct InvalidParseSubtree { InvalidParseStartId start; llvm::SmallVector> extra; + AnyToken token; }; // A placeholder node to be replaced; it will never exist in a valid parse tree. // Its token kind is not enforced even when valid. -using Placeholder = LeafNode; +using Placeholder = LeafNode; // File nodes // ---------- // The start of the file. -using FileStart = LeafNode; +using FileStart = + LeafNode>; // The end of the file. -using FileEnd = LeafNode; +using FileEnd = LeafNode>; // General-purpose nodes // --------------------- // An empty declaration, such as `;`. -using EmptyDecl = - LeafNode; +using EmptyDecl = LeafNode, + NodeCategory::Decl | NodeCategory::Statement>; // A name in a non-expression context, such as a declaration. using IdentifierName = - LeafNode; + LeafNode, + NodeCategory::MemberName>; // A name in an expression context. using IdentifierNameExpr = - LeafNode; + LeafNode, + NodeCategory::Expr>; // The `self` value and `Self` type identifier keywords. Typically of the form // `self: Self`. -using SelfValueName = LeafNode; +using SelfValueName = LeafNode>; using SelfValueNameExpr = - LeafNode; + LeafNode, NodeCategory::Expr>; using SelfTypeNameExpr = - LeafNode; + LeafNode, NodeCategory::Expr>; // The `base` value keyword, introduced by `base: B`. Typically referenced in // an expression, as in `x.base` or `{.base = ...}`, but can also be used as a // declared name, as in `{.base: partial B}`. -using BaseName = LeafNode; +using BaseName = LeafNode, + NodeCategory::MemberName>; // An unqualified name and optionally a following sequence of parameters. // For example, `A`, `A(n: i32)`, or `A[T:! type](n: T)`. @@ -146,6 +157,7 @@ struct NameQualifier { NodeKind::NameQualifier.Define({.bracketed_by = IdentifierName::Kind}); NameAndParams name_and_params; + Token token; }; // A complete name in a declaration: `A.C(T:! type).F(n: i32)`. @@ -159,20 +171,27 @@ struct DeclName { // -------------------------------- // The `package` keyword in an expression. -using PackageExpr = LeafNode; +using PackageExpr = + LeafNode, + NodeCategory::Expr>; // The name of a package or library for `package`, `import`, and `library`. -using PackageName = LeafNode; -using LibraryName = LeafNode; -using DefaultLibrary = LeafNode; +using PackageName = + LeafNode>; +using LibraryName = + LeafNode>; +using DefaultLibrary = + LeafNode>; -using PackageIntroducer = LeafNode; +using PackageIntroducer = + LeafNode>; // `library` in `package` or `import`. struct LibrarySpecifier { static constexpr auto Kind = NodeKind::LibrarySpecifier.Define({.child_count = 1}); + Token token; NodeIdOneOf name; }; @@ -187,10 +206,12 @@ struct PackageDecl { llvm::SmallVector modifiers; std::optional name; std::optional library; + Token token; }; // `import TheirPackage library "TheirLibrary";` -using ImportIntroducer = LeafNode; +using ImportIntroducer = + LeafNode>; struct ImportDecl { static constexpr auto Kind = NodeKind::ImportDecl.Define( {.category = NodeCategory::Decl, .bracketed_by = ImportIntroducer::Kind}); @@ -199,10 +220,12 @@ struct ImportDecl { llvm::SmallVector modifiers; std::optional name; std::optional library; + Token token; }; // `library` as declaration. -using LibraryIntroducer = LeafNode; +using LibraryIntroducer = + LeafNode>; struct LibraryDecl { static constexpr auto Kind = NodeKind::LibraryDecl.Define({.category = NodeCategory::Decl, @@ -211,10 +234,12 @@ struct LibraryDecl { LibraryIntroducerId introducer; llvm::SmallVector modifiers; NodeIdOneOf library_name; + Token token; }; // `export` as a declaration. -using ExportIntroducer = LeafNode; +using ExportIntroducer = + LeafNode>; struct ExportDecl { static constexpr auto Kind = NodeKind::ExportDecl.Define( {.category = NodeCategory::Decl, .bracketed_by = ExportIntroducer::Kind}); @@ -222,12 +247,14 @@ struct ExportDecl { ExportIntroducerId introducer; llvm::SmallVector modifiers; DeclName name; + Token token; }; // Namespace nodes // --------------- -using NamespaceStart = LeafNode; +using NamespaceStart = + LeafNode>; // A namespace: `namespace N;`. struct Namespace { @@ -237,6 +264,7 @@ struct Namespace { NamespaceStartId introducer; llvm::SmallVector modifiers; DeclName name; + Token token; }; // Pattern nodes @@ -248,6 +276,7 @@ struct BindingPattern { {.category = NodeCategory::Pattern, .child_count = 2}); NodeIdOneOf name; + Token token; AnyExprId type; }; @@ -257,6 +286,7 @@ struct CompileTimeBindingPattern { {.category = NodeCategory::Pattern, .child_count = 2}); NodeIdOneOf name; + Token token; AnyExprId type; }; @@ -265,6 +295,7 @@ struct Addr { static constexpr auto Kind = NodeKind::Addr.Define( {.category = NodeCategory::Pattern, .child_count = 1}); + Token token; AnyPatternId inner; }; @@ -273,13 +304,16 @@ struct Template { static constexpr auto Kind = NodeKind::Template.Define( {.category = NodeCategory::Pattern, .child_count = 1}); + Token token; // This is a CompileTimeBindingPatternId in any valid program. // TODO: Should the parser enforce that? AnyPatternId inner; }; -using TuplePatternStart = LeafNode; -using PatternListComma = LeafNode; +using TuplePatternStart = + LeafNode>; +using PatternListComma = + LeafNode>; // A parameter list or tuple pattern: `(a: i32, b: i32)`. struct TuplePattern { @@ -289,9 +323,12 @@ struct TuplePattern { TuplePatternStartId left_paren; CommaSeparatedList params; + Token token; }; -using ImplicitParamListStart = LeafNode; +using ImplicitParamListStart = + LeafNode>; // An implicit parameter list: `[T:! type, self: Self]`. struct ImplicitParamList { @@ -300,22 +337,26 @@ struct ImplicitParamList { ImplicitParamListStartId left_square; CommaSeparatedList params; + Token token; }; // Function nodes // -------------- -using FunctionIntroducer = LeafNode; +using FunctionIntroducer = + LeafNode>; // A return type: `-> i32`. struct ReturnType { static constexpr auto Kind = NodeKind::ReturnType.Define({.child_count = 1}); + Token token; AnyExprId type; }; // A function signature: `fn F() -> i32`. -template +template struct FunctionSignature { static constexpr auto Kind = KindT.Define( {.category = Category, .bracketed_by = FunctionIntroducer::Kind}); @@ -324,12 +365,16 @@ struct FunctionSignature { llvm::SmallVector modifiers; DeclName name; std::optional return_type; + TokenKind token; }; using FunctionDecl = - FunctionSignature; + FunctionSignature, + NodeCategory::Decl>; using FunctionDefinitionStart = - FunctionSignature; + FunctionSignature, + NodeCategory::None>; // A function definition: `fn F() -> i32 { ... }`. struct FunctionDefinition { @@ -339,12 +384,14 @@ struct FunctionDefinition { FunctionDefinitionStartId signature; llvm::SmallVector body; + Token token; }; using BuiltinFunctionDefinitionStart = FunctionSignature; -using BuiltinName = LeafNode; + Token, NodeCategory::None>; +using BuiltinName = + LeafNode>; // A builtin function definition: `fn F() -> i32 = "builtin name";` struct BuiltinFunctionDefinition { @@ -354,13 +401,16 @@ struct BuiltinFunctionDefinition { BuiltinFunctionDefinitionStartId signature; BuiltinNameId builtin_name; + Token token; }; // `alias` nodes // ------------- -using AliasIntroducer = LeafNode; -using AliasInitializer = LeafNode; +using AliasIntroducer = + LeafNode>; +using AliasInitializer = + LeafNode>; // An `alias` declaration: `alias a = b;`. struct Alias { @@ -373,13 +423,16 @@ struct Alias { DeclName name; AliasInitializerId equals; AnyExprId initializer; + Token token; }; // `let` nodes // ----------- -using LetIntroducer = LeafNode; -using LetInitializer = LeafNode; +using LetIntroducer = + LeafNode>; +using LetInitializer = + LeafNode>; // A `let` declaration: `let a: i32 = 5;`. struct LetDecl { @@ -396,14 +449,18 @@ struct LetDecl { AnyExprId initializer; }; std::optional initializer; + Token token; }; // `var` nodes // ----------- -using VariableIntroducer = LeafNode; -using ReturnedModifier = LeafNode; -using VariableInitializer = LeafNode; +using VariableIntroducer = + LeafNode>; +using ReturnedModifier = + LeafNode>; +using VariableInitializer = + LeafNode>; // A `var` declaration: `var a: i32;` or `var a: i32 = 5;`. struct VariableDecl { @@ -421,12 +478,14 @@ struct VariableDecl { AnyExprId value; }; std::optional initializer; + Token token; }; // Statement nodes // --------------- -using CodeBlockStart = LeafNode; +using CodeBlockStart = + LeafNode>; // A code block: `{ statement; statement; ... }`. struct CodeBlock { @@ -435,6 +494,7 @@ struct CodeBlock { CodeBlockStartId left_brace; llvm::SmallVector statements; + Token token; }; // An expression statement: `F(x);`. @@ -443,9 +503,11 @@ struct ExprStatement { {.category = NodeCategory::Statement, .child_count = 1}); AnyExprId expr; + Token token; }; -using BreakStatementStart = LeafNode; +using BreakStatementStart = + LeafNode>; // A break statement: `break;`. struct BreakStatement { @@ -455,9 +517,11 @@ struct BreakStatement { .child_count = 1}); BreakStatementStartId introducer; + Token token; }; -using ContinueStatementStart = LeafNode; +using ContinueStatementStart = + LeafNode>; // A continue statement: `continue;`. struct ContinueStatement { @@ -467,10 +531,13 @@ struct ContinueStatement { .child_count = 1}); ContinueStatementStartId introducer; + Token token; }; -using ReturnStatementStart = LeafNode; -using ReturnVarModifier = LeafNode; +using ReturnStatementStart = + LeafNode>; +using ReturnVarModifier = + LeafNode>; // A return statement: `return;` or `return expr;` or `return var;`. struct ReturnStatement { @@ -481,9 +548,11 @@ struct ReturnStatement { ReturnStatementStartId introducer; std::optional expr; std::optional var; + Token token; }; -using ForHeaderStart = LeafNode; +using ForHeaderStart = + LeafNode>; // The `var ... in` portion of a `for` statement. struct ForIn { @@ -491,6 +560,7 @@ struct ForIn { {.bracketed_by = VariableIntroducer::Kind, .child_count = 2}); VariableIntroducerId introducer; + Token token; AnyPatternId pattern; }; @@ -502,6 +572,7 @@ struct ForHeader { ForHeaderStartId introducer; ForInId var; AnyExprId range; + Token token; }; // A complete `for (...) { ... }` statement. @@ -511,11 +582,13 @@ struct ForStatement { .bracketed_by = ForHeader::Kind, .child_count = 2}); + Token token; ForHeaderId header; CodeBlockId body; }; -using IfConditionStart = LeafNode; +using IfConditionStart = + LeafNode>; // The condition portion of an `if` statement: `(expr)`. struct IfCondition { @@ -524,15 +597,18 @@ struct IfCondition { IfConditionStartId left_paren; AnyExprId condition; + Token token; }; -using IfStatementElse = LeafNode; +using IfStatementElse = + LeafNode>; // An `if` statement: `if (expr) { ... } else { ... }`. struct IfStatement { static constexpr auto Kind = NodeKind::IfStatement.Define( {.category = NodeCategory::Statement, .bracketed_by = IfCondition::Kind}); + Token token; IfConditionId head; CodeBlockId then; @@ -543,7 +619,8 @@ struct IfStatement { std::optional else_clause; }; -using WhileConditionStart = LeafNode; +using WhileConditionStart = + LeafNode>; // The condition portion of a `while` statement: `(expr)`. struct WhileCondition { @@ -552,6 +629,7 @@ struct WhileCondition { WhileConditionStartId left_paren; AnyExprId condition; + Token token; }; // A `while` statement: `while (expr) { ... }`. @@ -561,11 +639,13 @@ struct WhileStatement { .bracketed_by = WhileCondition::Kind, .child_count = 2}); + Token token; WhileConditionId head; CodeBlockId body; }; -using MatchConditionStart = LeafNode; +using MatchConditionStart = + LeafNode>; struct MatchCondition { static constexpr auto Kind = NodeKind::MatchCondition.Define( @@ -573,20 +653,26 @@ struct MatchCondition { MatchConditionStartId left_paren; AnyExprId condition; + Token token; }; -using MatchIntroducer = LeafNode; +using MatchIntroducer = + LeafNode>; struct MatchStatementStart { static constexpr auto Kind = NodeKind::MatchStatementStart.Define( {.bracketed_by = MatchIntroducer::Kind, .child_count = 2}); MatchIntroducerId introducer; - MatchConditionId left_brace; + MatchConditionId condition; + Token token; }; -using MatchCaseIntroducer = LeafNode; -using MatchCaseGuardIntroducer = LeafNode; -using MatchCaseGuardStart = LeafNode; +using MatchCaseIntroducer = + LeafNode>; +using MatchCaseGuardIntroducer = + LeafNode>; +using MatchCaseGuardStart = + LeafNode>; struct MatchCaseGuard { static constexpr auto Kind = NodeKind::MatchCaseGuard.Define( @@ -595,9 +681,11 @@ struct MatchCaseGuard { MatchCaseGuardIntroducerId introducer; MatchCaseGuardStartId left_paren; AnyExprId condition; + Token token; }; -using MatchCaseEqualGreater = LeafNode; +using MatchCaseEqualGreater = LeafNode>; struct MatchCaseStart { static constexpr auto Kind = NodeKind::MatchCaseStart.Define( @@ -607,6 +695,7 @@ struct MatchCaseStart { AnyPatternId pattern; std::optional guard; MatchCaseEqualGreaterId equal_greater_token; + Token token; }; struct MatchCase { @@ -615,10 +704,13 @@ struct MatchCase { MatchCaseStartId head; llvm::SmallVector statements; + Token token; }; -using MatchDefaultIntroducer = LeafNode; -using MatchDefaultEqualGreater = LeafNode; +using MatchDefaultIntroducer = + LeafNode>; +using MatchDefaultEqualGreater = LeafNode>; struct MatchDefaultStart { static constexpr auto Kind = NodeKind::MatchDefaultStart.Define( @@ -626,6 +718,7 @@ struct MatchDefaultStart { MatchDefaultIntroducerId introducer; MatchDefaultEqualGreaterId equal_greater_token; + Token token; }; struct MatchDefault { @@ -634,6 +727,7 @@ struct MatchDefault { MatchDefaultStartId introducer; llvm::SmallVector statements; + Token token; }; // A `match` statement: `match (expr) { case (...) => {...} default => {...}}`. @@ -646,12 +740,14 @@ struct MatchStatement { llvm::SmallVector cases; std::optional default_case; + Token token; }; // Expression nodes // ---------------- -using ArrayExprStart = LeafNode; +using ArrayExprStart = LeafNode>; // The start of an array type, `[i32;`. // @@ -662,6 +758,7 @@ struct ArrayExprSemi { ArrayExprStartId left_square; AnyExprId type; + Token token; }; // An array type, such as `[i32; 3]` or `[i32;]`. @@ -671,6 +768,7 @@ struct ArrayExpr { ArrayExprSemiId start; std::optional bound; + Token token; }; // The opening portion of an indexing expression: `a[`. @@ -681,6 +779,7 @@ struct IndexExprStart { NodeKind::IndexExprStart.Define({.child_count = 1}); AnyExprId sequence; + Token token; }; // An indexing expression, such as `a[1]`. @@ -692,9 +791,11 @@ struct IndexExpr { IndexExprStartId start; AnyExprId index; + Token token; }; -using ParenExprStart = LeafNode; +using ParenExprStart = + LeafNode>; // A parenthesized expression: `(a)`. struct ParenExpr { @@ -705,10 +806,13 @@ struct ParenExpr { ParenExprStartId start; AnyExprId expr; + Token token; }; -using TupleLiteralStart = LeafNode; -using TupleLiteralComma = LeafNode; +using TupleLiteralStart = + LeafNode>; +using TupleLiteralComma = + LeafNode>; // A tuple literal: `()`, `(a, b, c)`, or `(a,)`. struct TupleLiteral { @@ -718,6 +822,7 @@ struct TupleLiteral { TupleLiteralStartId start; CommaSeparatedList elements; + Token token; }; // The opening portion of a call expression: `F(`. @@ -728,9 +833,11 @@ struct CallExprStart { NodeKind::CallExprStart.Define({.child_count = 1}); AnyExprId callee; + Token token; }; -using CallExprComma = LeafNode; +using CallExprComma = + LeafNode>; // A call expression: `F(a, b, c)`. struct CallExpr { @@ -739,6 +846,7 @@ struct CallExpr { CallExprStartId start; CommaSeparatedList arguments; + Token token; }; // A member access expression: `a.b` or `a.(b)`. @@ -747,6 +855,7 @@ struct MemberAccessExpr { {.category = NodeCategory::Expr, .child_count = 2}); AnyExprId lhs; + Token token; AnyMemberNameOrMemberExprId rhs; }; @@ -756,52 +865,61 @@ struct PointerMemberAccessExpr { {.category = NodeCategory::Expr, .child_count = 2}); AnyExprId lhs; + Token token; AnyMemberNameOrMemberExprId rhs; }; // A prefix operator expression. -template +template struct PrefixOperator { static constexpr auto Kind = KindT.Define({.category = NodeCategory::Expr, .child_count = 1}); + TokenKind token; AnyExprId operand; }; // An infix operator expression. -template +template struct InfixOperator { static constexpr auto Kind = KindT.Define({.category = NodeCategory::Expr, .child_count = 2}); AnyExprId lhs; + TokenKind token; AnyExprId rhs; }; // A postfix operator expression. -template +template struct PostfixOperator { static constexpr auto Kind = KindT.Define({.category = NodeCategory::Expr, .child_count = 1}); AnyExprId operand; + TokenKind token; }; // Literals, operators, and modifiers #define CARBON_PARSE_NODE_KIND(...) -#define CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(Name, ...) \ - using Name = LeafNode; -#define CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER(Name, ...) \ - using Name##Modifier = \ - LeafNode; -#define CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Name, ...) \ - using PrefixOperator##Name = PrefixOperator; -#define CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Name, ...) \ - using InfixOperator##Name = InfixOperator; -#define CARBON_PARSE_NODE_KIND_POSTFIX_OPERATOR(Name, ...) \ - using PostfixOperator##Name = \ - PostfixOperator; +#define CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(Name, LexTokenKind) \ + using Name = LeafNode, \ + NodeCategory::Expr>; +#define CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER(Name) \ + using Name##Modifier = \ + LeafNode, \ + NodeCategory::Modifier>; +#define CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Name) \ + using PrefixOperator##Name = PrefixOperator>; +#define CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Name) \ + using InfixOperator##Name = InfixOperator>; +#define CARBON_PARSE_NODE_KIND_POSTFIX_OPERATOR(Name) \ + using PostfixOperator##Name = \ + PostfixOperator>; #include "toolchain/parse/node_kind.def" // The first operand of a short-circuiting infix operator: `a and` or `a or`. @@ -814,6 +932,7 @@ struct ShortCircuitOperandAnd { NodeKind::ShortCircuitOperandAnd.Define({.child_count = 1}); AnyExprId operand; + VirtualToken token; }; struct ShortCircuitOperandOr { @@ -821,6 +940,7 @@ struct ShortCircuitOperandOr { NodeKind::ShortCircuitOperandOr.Define({.child_count = 1}); AnyExprId operand; + VirtualToken token; }; struct ShortCircuitOperatorAnd { @@ -830,6 +950,7 @@ struct ShortCircuitOperatorAnd { .child_count = 2}); ShortCircuitOperandAndId lhs; + Token token; AnyExprId rhs; }; @@ -840,6 +961,7 @@ struct ShortCircuitOperatorOr { .child_count = 2}); ShortCircuitOperandOrId lhs; + Token token; AnyExprId rhs; }; @@ -847,6 +969,7 @@ struct ShortCircuitOperatorOr { struct IfExprIf { static constexpr auto Kind = NodeKind::IfExprIf.Define({.child_count = 1}); + Token token; AnyExprId condition; }; @@ -854,6 +977,7 @@ struct IfExprIf { struct IfExprThen { static constexpr auto Kind = NodeKind::IfExprThen.Define({.child_count = 1}); + Token token; AnyExprId result; }; @@ -866,13 +990,15 @@ struct IfExprElse { IfExprIfId start; IfExprThenId then; + Token token; AnyExprId else_result; }; // Choice nodes // ------------ -using ChoiceIntroducer = LeafNode; +using ChoiceIntroducer = + LeafNode>; struct ChoiceSignature { static constexpr auto Kind = NodeKind::ChoiceDefinitionStart.Define( @@ -881,12 +1007,14 @@ struct ChoiceSignature { ChoiceIntroducerId introducer; llvm::SmallVector modifiers; DeclName name; + Token token; }; using ChoiceDefinitionStart = ChoiceSignature; using ChoiceAlternativeListComma = - LeafNode; + LeafNode>; struct ChoiceDefinition { static constexpr auto Kind = NodeKind::ChoiceDefinition.Define( @@ -899,22 +1027,27 @@ struct ChoiceDefinition { std::optional parameters; }; CommaSeparatedList alternatives; + Token token; }; // Struct type and value literals // ---------------------------------------- // `{` -using StructLiteralStart = LeafNode; -using StructTypeLiteralStart = LeafNode; +using StructLiteralStart = LeafNode>; +using StructTypeLiteralStart = LeafNode>; // `,` -using StructComma = LeafNode; +using StructComma = + LeafNode>; // `.a` struct StructFieldDesignator { static constexpr auto Kind = NodeKind::StructFieldDesignator.Define({.child_count = 1}); + Token token; NodeIdOneOf name; }; @@ -924,6 +1057,7 @@ struct StructField { {.bracketed_by = StructFieldDesignator::Kind, .child_count = 2}); StructFieldDesignatorId designator; + Token token; AnyExprId expr; }; @@ -933,6 +1067,7 @@ struct StructTypeField { {.bracketed_by = StructFieldDesignator::Kind, .child_count = 2}); StructFieldDesignatorId designator; + Token token; AnyExprId type_expr; }; @@ -944,6 +1079,7 @@ struct StructLiteral { StructLiteralStartId start; CommaSeparatedList fields; + Token token; }; // Struct type literals, such as `{.a: i32}`. @@ -954,16 +1090,19 @@ struct StructTypeLiteral { StructTypeLiteralStartId start; CommaSeparatedList fields; + Token token; }; // `class` declarations and definitions // ------------------------------------ // `class` -using ClassIntroducer = LeafNode; +using ClassIntroducer = + LeafNode>; // A class signature `class C` -template +template struct ClassSignature { static constexpr auto Kind = KindT.Define( {.category = Category, .bracketed_by = ClassIntroducer::Kind}); @@ -971,13 +1110,17 @@ struct ClassSignature { ClassIntroducerId introducer; llvm::SmallVector modifiers; DeclName name; + TokenKind token; }; // `class C;` -using ClassDecl = ClassSignature; +using ClassDecl = + ClassSignature, + NodeCategory::Decl>; // `class C {` using ClassDefinitionStart = - ClassSignature; + ClassSignature, NodeCategory::None>; // `class C { ... }` struct ClassDefinition { @@ -987,13 +1130,15 @@ struct ClassDefinition { ClassDefinitionStartId signature; llvm::SmallVector members; + Token token; }; // Adapter declaration // ------------------- // `adapt` -using AdaptIntroducer = LeafNode; +using AdaptIntroducer = + LeafNode>; // `adapt SomeType;` struct AdaptDecl { static constexpr auto Kind = NodeKind::AdaptDecl.Define( @@ -1002,14 +1147,16 @@ struct AdaptDecl { AdaptIntroducerId introducer; llvm::SmallVector modifiers; AnyExprId adapted_type; + Token token; }; // Base class declaration // ---------------------- // `base` -using BaseIntroducer = LeafNode; -using BaseColon = LeafNode; +using BaseIntroducer = + LeafNode>; +using BaseColon = LeafNode>; // `extend base: BaseClass;` struct BaseDecl { static constexpr auto Kind = NodeKind::BaseDecl.Define( @@ -1019,16 +1166,19 @@ struct BaseDecl { llvm::SmallVector modifiers; BaseColonId colon; AnyExprId base_class; + Token token; }; // Interface declarations and definitions // -------------------------------------- // `interface` -using InterfaceIntroducer = LeafNode; +using InterfaceIntroducer = + LeafNode>; // `interface I` -template +template struct InterfaceSignature { static constexpr auto Kind = KindT.Define( {.category = Category, .bracketed_by = InterfaceIntroducer::Kind}); @@ -1036,14 +1186,18 @@ struct InterfaceSignature { InterfaceIntroducerId introducer; llvm::SmallVector modifiers; DeclName name; + TokenKind token; }; // `interface I;` using InterfaceDecl = - InterfaceSignature; + InterfaceSignature, + NodeCategory::Decl>; // `interface I {` using InterfaceDefinitionStart = - InterfaceSignature; + InterfaceSignature, + NodeCategory::None>; // `interface I { ... }` struct InterfaceDefinition { @@ -1053,24 +1207,28 @@ struct InterfaceDefinition { InterfaceDefinitionStartId signature; llvm::SmallVector members; + Token token; }; // `impl`...`as` declarations and definitions // ------------------------------------------ // `impl` -using ImplIntroducer = LeafNode; +using ImplIntroducer = + LeafNode>; // `forall [...]` struct ImplForall { static constexpr auto Kind = NodeKind::ImplForall.Define({.child_count = 1}); + Token token; ImplicitParamListId params; }; // `as` with no type before it using DefaultSelfImplAs = - LeafNode; + LeafNode, + NodeCategory::ImplAs>; // ` as` struct TypeImplAs { @@ -1078,10 +1236,12 @@ struct TypeImplAs { {.category = NodeCategory::ImplAs, .child_count = 1}); AnyExprId type_expr; + Token token; }; // `impl T as I` -template +template struct ImplSignature { static constexpr auto Kind = KindT.Define( {.category = Category, .bracketed_by = ImplIntroducer::Kind}); @@ -1091,13 +1251,16 @@ struct ImplSignature { std::optional forall; AnyImplAsId as; AnyExprId interface; + TokenKind token; }; // `impl T as I;` -using ImplDecl = ImplSignature; +using ImplDecl = ImplSignature, + NodeCategory::Decl>; // `impl T as I {` using ImplDefinitionStart = - ImplSignature; + ImplSignature, NodeCategory::None>; // `impl T as I { ... }` struct ImplDefinition { @@ -1107,16 +1270,19 @@ struct ImplDefinition { ImplDefinitionStartId signature; llvm::SmallVector members; + Token token; }; // Named constraint declarations and definitions // --------------------------------------------- // `constraint` -using NamedConstraintIntroducer = LeafNode; +using NamedConstraintIntroducer = LeafNode>; // `constraint NC` -template +template struct NamedConstraintSignature { static constexpr auto Kind = KindT.Define( {.category = Category, .bracketed_by = NamedConstraintIntroducer::Kind}); @@ -1124,14 +1290,17 @@ struct NamedConstraintSignature { NamedConstraintIntroducerId introducer; llvm::SmallVector modifiers; DeclName name; + TokenKind token; }; // `constraint NC;` using NamedConstraintDecl = - NamedConstraintSignature; + NamedConstraintSignature, NodeCategory::Decl>; // `constraint NC {` using NamedConstraintDefinitionStart = NamedConstraintSignature, NodeCategory::None>; // `constraint NC { ... }` @@ -1142,6 +1311,7 @@ struct NamedConstraintDefinition { NamedConstraintDefinitionStartId signature; llvm::SmallVector members; + Token token; }; // --------------------------------------------------------------------------- diff --git a/toolchain/parse/typed_nodes_test.cpp b/toolchain/parse/typed_nodes_test.cpp index 522dc58bb9d8..67e53a51a0de 100644 --- a/toolchain/parse/typed_nodes_test.cpp +++ b/toolchain/parse/typed_nodes_test.cpp @@ -14,6 +14,25 @@ #include "toolchain/parse/parse.h" namespace Carbon::Parse { + +// A test peer (see https://abseil.io/tips/135) to allow these tests to access +// certain implementation details of Tree. +class TypedNodesTestPeer { + public: + template + static auto VerifyExtractAs(const Tree* tree, NodeId node_id, + ErrorBuilder* trace) -> std::optional { + return tree->VerifyExtractAs(node_id, trace); + } + + // Sets the kind of a node. This is intended to allow putting the tree into a + // state where verification can fail, in order to make the failure path of + // `Verify` testable. + static auto SetNodeKind(Tree* tree, NodeId node_id, NodeKind kind) -> void { + tree->SetNodeKindForTesting(node_id, kind); + } +}; + namespace { // Check that each node kind defines a Kind member using the correct @@ -44,6 +63,12 @@ class TypedNodeTest : public ::testing::Test { return tree_storage_.front(); } + auto GetTokenizedBufferAndTree(llvm::StringRef t) + -> std::pair { + auto* tree = &GetTree(t); + return {&token_storage_.front(), tree}; + } + SharedValueStores value_stores_; llvm::vfs::InMemoryFileSystem fs_; std::forward_list source_storage_; @@ -144,7 +169,8 @@ TEST_F(TypedNodeTest, VerifyExtractTraceLibrary) { ASSERT_EQ(file.decls.size(), 1); ErrorBuilder trace; - auto library = tree->VerifyExtractAs(file.decls[0], &trace); + auto library = TypedNodesTestPeer::VerifyExtractAs( + tree, file.decls[0], &trace); EXPECT_TRUE(library.has_value()); Error err = trace; // Use Regex matching to avoid hard-coding the result of `typeinfo(T).name()`. @@ -168,7 +194,8 @@ TEST_F(TypedNodeTest, VerifyExtractTraceVarNoInit) { ASSERT_EQ(file.decls.size(), 1); ErrorBuilder trace; - auto var = tree->VerifyExtractAs(file.decls[0], &trace); + auto var = TypedNodesTestPeer::VerifyExtractAs( + tree, file.decls[0], &trace); ASSERT_TRUE(var.has_value()); Error err = trace; // Use Regex matching to avoid hard-coding the result of `typeinfo(T).name()`. @@ -199,7 +226,8 @@ TEST_F(TypedNodeTest, VerifyExtractTraceExpression) { ASSERT_EQ(file.decls.size(), 1); ErrorBuilder trace1; - auto var = tree->VerifyExtractAs(file.decls[0], &trace1); + auto var = TypedNodesTestPeer::VerifyExtractAs( + tree, file.decls[0], &trace1); ASSERT_TRUE(var.has_value()); Error err1 = trace1; // Use Regex matching to avoid hard-coding the result of `typeinfo(T).name()`. @@ -224,8 +252,8 @@ Aggregate [^:]*: success ASSERT_TRUE(var->initializer.has_value()); ErrorBuilder trace2; - auto value = - tree->VerifyExtractAs(var->initializer->value, &trace2); + auto value = TypedNodesTestPeer::VerifyExtractAs( + tree, var->initializer->value, &trace2); ASSERT_TRUE(value.has_value()); Error err2 = trace2; // Use Regex matching to avoid hard-coding the result of `typeinfo(T).name()`. @@ -245,7 +273,8 @@ TEST_F(TypedNodeTest, VerifyExtractTraceClassDecl) { ASSERT_EQ(file.decls.size(), 1); ErrorBuilder trace; - auto class_decl = tree->VerifyExtractAs(file.decls[0], &trace); + auto class_decl = TypedNodesTestPeer::VerifyExtractAs( + tree, file.decls[0], &trace); EXPECT_TRUE(class_decl.has_value()); Error err = trace; // Use Regex matching to avoid hard-coding the result of `typeinfo(T).name()`. @@ -276,6 +305,88 @@ Aggregate [^:]*: success )Trace")); } +TEST_F(TypedNodeTest, Token) { + auto [tokens, tree] = GetTokenizedBufferAndTree(R"carbon( + var n: i32 = 0; + )carbon"); + auto file = tree->ExtractFile(); + + ASSERT_EQ(file.decls.size(), 1); + + auto n_var = tree->ExtractAs(file.decls[0]); + ASSERT_TRUE(n_var.has_value()); + EXPECT_EQ(tokens->GetKind(n_var->token.index), Lex::TokenKind::Semi); + + auto n_intro = tree->ExtractAs(n_var->introducer); + ASSERT_TRUE(n_intro.has_value()); + EXPECT_EQ(tokens->GetKind(n_intro->token.index), Lex::TokenKind::Var); + + auto n_patt = tree->ExtractAs(n_var->pattern); + ASSERT_TRUE(n_patt.has_value()); + EXPECT_EQ(tokens->GetKind(n_patt->token.index), Lex::TokenKind::Colon); +} + +TEST_F(TypedNodeTest, VerifyInvalid) { + auto* tree = &GetTree(R"carbon( + fn F() -> i32 { return 0; } + )carbon"); + + auto file = tree->ExtractFile(); + ASSERT_EQ(file.decls.size(), 1); + + auto f_fn = tree->ExtractAs(file.decls[0]); + ASSERT_TRUE(f_fn.has_value()); + auto f_sig = tree->ExtractAs(f_fn->signature); + ASSERT_TRUE(f_sig.has_value()); + auto f_intro = tree->ExtractAs(f_sig->introducer); + ASSERT_TRUE(f_intro.has_value()); + + // Change the kind of the introducer and check we get a good trace log. + TypedNodesTestPeer::SetNodeKind(tree, f_sig->introducer, + NodeKind::ClassIntroducer); + + // The introducer should not extract as a FunctionIntroducer any more because + // the kind is wrong. + { + ErrorBuilder trace; + EXPECT_FALSE(TypedNodesTestPeer::VerifyExtractAs( + tree, f_sig->introducer, &trace)); + + Error err = trace; + EXPECT_EQ(err.message(), + "VerifyExtractAs error: wrong kind ClassIntroducer, expected " + "FunctionIntroducer\n"); + } + + // The introducer should also not extract as a ClassIntroducer because the + // token kind is wrong. + { + ErrorBuilder trace; + EXPECT_FALSE(TypedNodesTestPeer::VerifyExtractAs( + tree, f_sig->introducer, &trace)); + + Error err = trace; + EXPECT_THAT(err.message(), + testing::HasSubstr( + "\nToken Class expected for ClassIntroducer, found Fn\n")); + } + + // The signature should not extract as a FunctionDefinitionStart because the + // kind for the introducer is wrong. + { + ErrorBuilder trace; + EXPECT_FALSE(TypedNodesTestPeer::VerifyExtractAs( + tree, f_fn->signature, &trace)); + + Error err = trace; + EXPECT_THAT(err.message(), testing::MatchesRegex( + R"Trace((?s).* +NodeIdForKind error: wrong kind IdentifierName, expected ImplicitParamList +.* +Error: ClassIntroducer node left unconsumed.)Trace")); + } +} + auto CategoryMatches(const NodeKind::Definition& def, NodeKind kind, const char* name) { EXPECT_EQ(def.category(), kind.category()) << name;