diff --git a/common/struct_reflection.h b/common/struct_reflection.h index 6d32b3e938d6..9bb113819817 100644 --- a/common/struct_reflection.h +++ b/common/struct_reflection.h @@ -21,7 +21,7 @@ // - Only simple aggregate structs are supported. Types with base classes, // non-public data members, constructors, or virtual functions are not // supported. -// - Structs with more than 6 fields are not supported. This limit is easy to +// - Structs with more than 8 fields are not supported. This limit is easy to // increase if needed, but removing it entirely is hard. // - Structs containing a reference to the same type are not supported. @@ -76,7 +76,7 @@ constexpr auto CountFields() -> int { return CountFields>(); } else if constexpr (AnyWorkedSoFar) { constexpr int NumFields = sizeof...(Fields) - 1; - static_assert(NumFields <= 6, "Unsupported: too many fields in struct"); + static_assert(NumFields <= 8, "Unsupported: too many fields in struct"); return NumFields; } else if constexpr (sizeof...(Fields) > 32) { // If we go too far without finding a working initializer, something @@ -161,6 +161,31 @@ struct FieldAccessor<6> { } }; +template <> +struct FieldAccessor<7> { + template + static auto Get(T& value) -> auto { + auto& [field0, field1, field2, field3, field4, field5, field6] = value; + return std::tuple(field0, field1, field2, field3, field4, + field5, field6); + } +}; + +template <> +struct FieldAccessor<8> { + template + static auto Get(T& value) -> auto { + auto& [field0, field1, field2, field3, field4, field5, field6, field7] = + value; + return std::tuple( + field0, field1, field2, field3, field4, field5, field6, field7); + } +}; + } // namespace Internal // Get the fields of the struct `T` as a tuple. diff --git a/toolchain/lex/BUILD b/toolchain/lex/BUILD index d956c3ad0e1a..35ef519535b6 100644 --- a/toolchain/lex/BUILD +++ b/toolchain/lex/BUILD @@ -196,6 +196,12 @@ cc_library( ], ) +cc_library( + name = "token_index", + hdrs = ["token_index.h"], + deps = ["//toolchain/base:index_base"], +) + cc_library( name = "tokenized_buffer", srcs = ["tokenized_buffer.cpp"], @@ -205,6 +211,7 @@ cc_library( ":helpers", ":numeric_literal", ":string_literal", + ":token_index", ":token_kind", "//common:check", "//common:ostream", diff --git a/toolchain/lex/token_index.h b/toolchain/lex/token_index.h new file mode 100644 index 000000000000..7588383b598d --- /dev/null +++ b/toolchain/lex/token_index.h @@ -0,0 +1,37 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef CARBON_TOOLCHAIN_LEX_TOKEN_INDEX_H_ +#define CARBON_TOOLCHAIN_LEX_TOKEN_INDEX_H_ + +#include "toolchain/base/index_base.h" + +namespace Carbon::Lex { + +// A lightweight handle to a lexed token in a `TokenizedBuffer`. +// +// `TokenIndex` objects are designed to be passed by value, not reference or +// pointer. They are also designed to be small and efficient to store in data +// structures. +// +// `TokenIndex` objects from the same `TokenizedBuffer` can be compared with +// each other, both for being the same token within the buffer, and to establish +// relative position within the token stream that has been lexed out of the +// buffer. `TokenIndex` objects from different `TokenizedBuffer`s cannot be +// meaningfully compared. +// +// All other APIs to query a `TokenIndex` are on the `TokenizedBuffer`. +struct TokenIndex : public IndexBase { + static const TokenIndex Invalid; + // Comments aren't tokenized, so this is the first token after FileStart. + static const TokenIndex FirstNonCommentToken; + using IndexBase::IndexBase; +}; + +constexpr TokenIndex TokenIndex::Invalid(TokenIndex::InvalidIndex); +constexpr TokenIndex TokenIndex::FirstNonCommentToken(1); + +} // namespace Carbon::Lex + +#endif // CARBON_TOOLCHAIN_LEX_TOKEN_INDEX_H_ diff --git a/toolchain/lex/tokenized_buffer.h b/toolchain/lex/tokenized_buffer.h index ae43a201fbbd..f4ebd3dfe34d 100644 --- a/toolchain/lex/tokenized_buffer.h +++ b/toolchain/lex/tokenized_buffer.h @@ -20,6 +20,7 @@ #include "toolchain/base/index_base.h" #include "toolchain/base/value_store.h" #include "toolchain/diagnostics/diagnostic_emitter.h" +#include "toolchain/lex/token_index.h" #include "toolchain/lex/token_kind.h" #include "toolchain/source/source_buffer.h" @@ -27,29 +28,6 @@ namespace Carbon::Lex { class TokenizedBuffer; -// A lightweight handle to a lexed token in a `TokenizedBuffer`. -// -// `TokenIndex` objects are designed to be passed by value, not reference or -// pointer. They are also designed to be small and efficient to store in data -// structures. -// -// `TokenIndex` objects from the same `TokenizedBuffer` can be compared with -// each other, both for being the same token within the buffer, and to establish -// relative position within the token stream that has been lexed out of the -// buffer. `TokenIndex` objects from different `TokenizedBuffer`s cannot be -// meaningfully compared. -// -// All other APIs to query a `TokenIndex` are on the `TokenizedBuffer`. -struct TokenIndex : public IndexBase { - static const TokenIndex Invalid; - // Comments aren't tokenized, so this is the first token after FileStart. - static const TokenIndex FirstNonCommentToken; - using IndexBase::IndexBase; -}; - -constexpr TokenIndex TokenIndex::Invalid(TokenIndex::InvalidIndex); -constexpr TokenIndex TokenIndex::FirstNonCommentToken(1); - // A lightweight handle to a lexed line in a `TokenizedBuffer`. // // `LineIndex` objects are designed to be passed by value, not reference or diff --git a/toolchain/parse/BUILD b/toolchain/parse/BUILD index 32d49ab8a33f..c3d45e2156d7 100644 --- a/toolchain/parse/BUILD +++ b/toolchain/parse/BUILD @@ -27,6 +27,7 @@ cc_library( "//common:enum_base", "//common:ostream", "//toolchain/base:index_base", + "//toolchain/lex:token_index", "//toolchain/lex:token_kind", "@llvm-project//llvm:Support", ], diff --git a/toolchain/parse/context.cpp b/toolchain/parse/context.cpp index 76f1c70a04d3..237008b6cd97 100644 --- a/toolchain/parse/context.cpp +++ b/toolchain/parse/context.cpp @@ -68,7 +68,6 @@ Context::Context(Tree& tree, Lex::TokenizedBuffer& tokens, auto Context::AddLeafNode(NodeKind kind, Lex::TokenIndex token, bool has_error) -> void { - kind.CheckMatchesTokenKind(tokens_->GetKind(token), has_error); tree_->node_impls_.push_back( Tree::NodeImpl(kind, has_error, token, /*subtree_size=*/1)); if (has_error) { @@ -78,7 +77,6 @@ auto Context::AddLeafNode(NodeKind kind, Lex::TokenIndex token, bool has_error) auto Context::AddNode(NodeKind kind, Lex::TokenIndex token, int subtree_start, bool has_error) -> void { - kind.CheckMatchesTokenKind(tokens_->GetKind(token), has_error); int subtree_size = tree_->size() - subtree_start + 1; tree_->node_impls_.push_back( Tree::NodeImpl(kind, has_error, token, subtree_size)); diff --git a/toolchain/parse/extract.cpp b/toolchain/parse/extract.cpp index ab519dc9e93b..364ac97fa3b4 100644 --- a/toolchain/parse/extract.cpp +++ b/toolchain/parse/extract.cpp @@ -13,6 +13,76 @@ namespace Carbon::Parse { +namespace { +// Implementation of the process of extracting a typed node structure from the +// parse tree. The extraction process uses the class `Extractable`, defined +// below, to extract individual fields of type `T`. +class NodeExtractor { + public: + struct CheckpointState { + Tree::SiblingIterator it; + }; + + NodeExtractor(const Tree* tree, Lex::TokenizedBuffer* tokens, + ErrorBuilder* trace, NodeId node_id, + llvm::iterator_range children) + : tree_(tree), + tokens_(tokens), + trace_(trace), + node_id_(node_id), + it_(children.begin()), + end_(children.end()) {} + + auto at_end() const -> bool { return it_ == end_; } + auto kind() const -> NodeKind { return tree_->node_kind(*it_); } + auto has_token() const -> bool { return node_id_.is_valid(); } + auto token() const -> Lex::TokenIndex { return tree_->node_token(node_id_); } + auto token_kind() const -> Lex::TokenKind { + return tokens_->GetKind(token()); + } + auto trace() const -> ErrorBuilder* { return trace_; } + + // Saves a checkpoint of our current position so we can return later if + // extraction of a child node fails. + auto Checkpoint() const -> CheckpointState { return {.it = it_}; } + auto RestoreCheckpoint(CheckpointState checkpoint) { it_ = checkpoint.it; } + + // Determines whether the current position matches the specified node kind. If + // not, produces a suitable trace message. + auto MatchesNodeIdForKind(NodeKind kind) const -> bool; + + // Determines whether the current position matches the specified node + // category. If not, produces a suitable trace message. + auto MatchesNodeIdInCategory(NodeCategory category) const -> bool; + + // Determines whether the current position matches any of the specified node + // kinds. If not, produces a suitable trace message. + auto MatchesNodeIdOneOf(std::initializer_list kinds) const -> bool; + + // Determines whether the token corresponding to the enclosing node is of the + // specified kind. If not, produces a suitable trace message. + auto MatchesTokenKind(Lex::TokenKind expected_kind) const -> bool; + + // Extracts the next node from the tree. + auto ExtractNode() -> NodeId { return *it_++; } + + // Extracts a tuple-like type `T` by extracting its components and then + // assembling a `T` value. + template + auto ExtractTupleLikeType(std::index_sequence /*indices*/, + std::tuple* /*type*/) -> std::optional; + + private: + const Tree* tree_; + Lex::TokenizedBuffer* tokens_; + ErrorBuilder* trace_; + NodeId node_id_; + Tree::SiblingIterator it_; + Tree::SiblingIterator end_; +}; +} // namespace + +namespace { // A trait type that should be specialized by types that can be extracted // from a parse tree. A specialization should provide the following API: // @@ -23,9 +93,7 @@ namespace Carbon::Parse { // // `it`, and increment `it` past this type. Returns `std::nullopt` if // // the tree is malformed. If `trace != nullptr`, writes what actions // // were taken to `*trace`. -// static auto Extract(Tree* tree, Tree::SiblingIterator& it, -// Tree::SiblingIterator end, -// ErrorBuilder* trace) -> std::optional; +// static auto Extract(NodeExtractor* extractor) -> std::optional; // }; // ``` // @@ -35,44 +103,40 @@ namespace Carbon::Parse { // This class is only in this file. template struct Extractable; +} // namespace // Extract a `NodeId` as a single child. template <> struct Extractable { - static auto Extract(const Tree* tree, Tree::SiblingIterator& it, - Tree::SiblingIterator end, ErrorBuilder* trace) - -> std::optional { - if (it == end) { - if (trace) { + static auto Extract(NodeExtractor& extractor) -> std::optional { + if (extractor.at_end()) { + if (auto* trace = extractor.trace()) { *trace << "NodeId error: no more children\n"; } return std::nullopt; } - if (trace) { - *trace << "NodeId: " << tree->node_kind(*it) << " consumed\n"; + if (auto* trace = extractor.trace()) { + *trace << "NodeId: " << extractor.kind() << " consumed\n"; } - return *it++; + return extractor.ExtractNode(); } }; -static auto NodeIdForKindAccept(const NodeKind& kind, const Tree* tree, - const Tree::SiblingIterator& it, - Tree::SiblingIterator end, ErrorBuilder* trace) - -> bool { - if (it == end || tree->node_kind(*it) != kind) { - if (trace) { - if (it == end) { - *trace << "NodeIdForKind error: no more children, expected " << kind - << "\n"; +auto NodeExtractor::MatchesNodeIdForKind(NodeKind expected_kind) const -> bool { + if (at_end() || kind() != expected_kind) { + if (trace_) { + if (at_end()) { + *trace_ << "NodeIdForKind error: no more children, expected " + << expected_kind << "\n"; } else { - *trace << "NodeIdForKind error: wrong kind " << tree->node_kind(*it) - << ", expected " << kind << "\n"; + *trace_ << "NodeIdForKind error: wrong kind " << kind() << ", expected " + << expected_kind << "\n"; } } return false; } - if (trace) { - *trace << "NodeIdForKind: " << kind << " consumed\n"; + if (trace_) { + *trace_ << "NodeIdForKind: " << expected_kind << " consumed\n"; } return true; } @@ -81,35 +145,32 @@ static auto NodeIdForKindAccept(const NodeKind& kind, const Tree* tree, // as a single required child. template struct Extractable> { - static auto Extract(const Tree* tree, Tree::SiblingIterator& it, - Tree::SiblingIterator end, ErrorBuilder* trace) + static auto Extract(NodeExtractor& extractor) -> std::optional> { - if (NodeIdForKindAccept(Kind, tree, it, end, trace)) { - return NodeIdForKind(*it++); + if (extractor.MatchesNodeIdForKind(Kind)) { + return NodeIdForKind(extractor.ExtractNode()); } else { return std::nullopt; } } }; -static auto NodeIdInCategoryAccept(NodeCategory category, const Tree* tree, - const Tree::SiblingIterator& it, - Tree::SiblingIterator end, - ErrorBuilder* trace) -> bool { - if (it == end || !tree->node_kind(*it).category().HasAnyOf(category)) { - if (trace) { - *trace << "NodeIdInCategory " << category << " error: "; - if (it == end) { - *trace << "no more children\n"; +auto NodeExtractor::MatchesNodeIdInCategory(NodeCategory category) const + -> bool { + if (at_end() || !kind().category().HasAnyOf(category)) { + if (trace_) { + *trace_ << "NodeIdInCategory " << category << " error: "; + if (at_end()) { + *trace_ << "no more children\n"; } else { - *trace << "kind " << tree->node_kind(*it) << " doesn't match\n"; + *trace_ << "kind " << kind() << " doesn't match\n"; } } return false; } - if (trace) { - *trace << "NodeIdInCategory " << category << ": kind " - << tree->node_kind(*it) << " consumed\n"; + if (trace_) { + *trace_ << "NodeIdInCategory " << category << ": kind " << kind() + << " consumed\n"; } return true; } @@ -117,47 +178,45 @@ static auto NodeIdInCategoryAccept(NodeCategory category, const Tree* tree, // Extract a `NodeIdInCategory` as a single child. template struct Extractable> { - static auto Extract(const Tree* tree, Tree::SiblingIterator& it, - Tree::SiblingIterator end, ErrorBuilder* trace) + static auto Extract(NodeExtractor& extractor) -> std::optional> { - if (NodeIdInCategoryAccept(Category, tree, it, end, trace)) { - return NodeIdInCategory(*it++); + if (extractor.MatchesNodeIdInCategory(Category)) { + return NodeIdInCategory(extractor.ExtractNode()); } else { return std::nullopt; } } }; -static auto NodeIdOneOfAccept(std::initializer_list kinds, - const Tree* tree, const Tree::SiblingIterator& it, - Tree::SiblingIterator end, ErrorBuilder* trace) - -> bool { +auto NodeExtractor::MatchesNodeIdOneOf( + std::initializer_list kinds) const -> bool { auto trace_kinds = [&] { llvm::ListSeparator sep(" or "); for (auto kind : kinds) { - *trace << sep << kind; + *trace_ << sep << kind; } }; - auto kind = tree->node_kind(*it); - if (it == end || std::find(kinds.begin(), kinds.end(), kind) == kinds.end()) { - if (trace) { - if (it == end) { - *trace << "NodeIdOneOf error: no more children, expected "; + auto node_kind = kind(); + if (at_end() || + std::find(kinds.begin(), kinds.end(), node_kind) == kinds.end()) { + if (trace_) { + if (at_end()) { + *trace_ << "NodeIdOneOf error: no more children, expected "; trace_kinds(); - *trace << "\n"; + *trace_ << "\n"; } else { - *trace << "NodeIdOneOf error: wrong kind " << tree->node_kind(*it) - << ", expected "; + *trace_ << "NodeIdOneOf error: wrong kind " << node_kind + << ", expected "; trace_kinds(); - *trace << "\n"; + *trace_ << "\n"; } } return false; } - if (trace) { - *trace << "NodeIdOneOf "; + if (trace_) { + *trace_ << "NodeIdOneOf "; trace_kinds(); - *trace << ": " << tree->node_kind(*it) << " consumed\n"; + *trace_ << ": " << node_kind << " consumed\n"; } return true; } @@ -165,11 +224,10 @@ static auto NodeIdOneOfAccept(std::initializer_list kinds, // Extract a `NodeIdOneOf` as a single required child. template struct Extractable> { - static auto Extract(const Tree* tree, Tree::SiblingIterator& it, - Tree::SiblingIterator end, ErrorBuilder* trace) + static auto Extract(NodeExtractor& extractor) -> std::optional> { - if (NodeIdOneOfAccept({T::Kind...}, tree, it, end, trace)) { - return NodeIdOneOf(*it++); + if (extractor.MatchesNodeIdOneOf({T::Kind...})) { + return NodeIdOneOf(extractor.ExtractNode()); } else { return std::nullopt; } @@ -180,12 +238,10 @@ struct Extractable> { // Note: this is only instantiated once, so no need to create a helper function. template struct Extractable> { - static auto Extract(const Tree* tree, Tree::SiblingIterator& it, - Tree::SiblingIterator end, ErrorBuilder* trace) - -> std::optional> { - if (it == end || tree->node_kind(*it) == T::Kind) { - if (trace) { - if (it == end) { + static auto Extract(NodeExtractor& extractor) -> std::optional> { + if (extractor.at_end() || extractor.kind() == T::Kind) { + if (auto* trace = extractor.trace()) { + if (extractor.at_end()) { *trace << "NodeIdNot " << T::Kind << " error: no more children\n"; } else { *trace << "NodeIdNot error: unexpected " << T::Kind << "\n"; @@ -193,35 +249,34 @@ struct Extractable> { } return std::nullopt; } - if (trace) { - *trace << "NodeIdNot " << T::Kind << ": " << tree->node_kind(*it) + if (auto* trace = extractor.trace()) { + *trace << "NodeIdNot " << T::Kind << ": " << extractor.kind() << " consumed\n"; } - return NodeIdNot(*it++); + return NodeIdNot(extractor.ExtractNode()); } }; // Extract an `llvm::SmallVector` by extracting `T`s until we can't. template struct Extractable> { - static auto Extract(const Tree* tree, Tree::SiblingIterator& it, - Tree::SiblingIterator end, ErrorBuilder* trace) + static auto Extract(NodeExtractor& extractor) -> std::optional> { - if (trace) { + if (auto* trace = extractor.trace()) { *trace << "Vector: begin\n"; } llvm::SmallVector result; - while (it != end) { - auto old_it = it; - auto item = Extractable::Extract(tree, it, end, trace); + while (!extractor.at_end()) { + auto checkpoint = extractor.Checkpoint(); + auto item = Extractable::Extract(extractor); if (!item.has_value()) { - it = old_it; + extractor.RestoreCheckpoint(checkpoint); break; } result.push_back(*item); } std::reverse(result.begin(), result.end()); - if (trace) { + if (auto* trace = extractor.trace()) { *trace << "Vector: end\n"; } return result; @@ -232,91 +287,131 @@ struct Extractable> { // a `T`, and extracting nothing if that fails. template struct Extractable> { - static auto Extract(const Tree* tree, Tree::SiblingIterator& it, - Tree::SiblingIterator end, ErrorBuilder* trace) + static auto Extract(NodeExtractor& extractor) -> std::optional> { - if (trace) { + if (auto* trace = extractor.trace()) { *trace << "Optional " << typeid(T).name() << ": begin\n"; } - auto old_it = it; - std::optional value = Extractable::Extract(tree, it, end, trace); + auto checkpoint = extractor.Checkpoint(); + std::optional value = Extractable::Extract(extractor); if (value) { - if (trace) { + if (auto* trace = extractor.trace()) { *trace << "Optional " << typeid(T).name() << ": found\n"; } return value; } - if (trace) { + if (auto* trace = extractor.trace()) { *trace << "Optional " << typeid(T).name() << ": missing\n"; } - it = old_it; + extractor.RestoreCheckpoint(checkpoint); return value; } }; +auto NodeExtractor::MatchesTokenKind(Lex::TokenKind expected_kind) const + -> bool { + if (!node_id_.is_valid()) { + if (trace_) { + *trace_ << "Token " << expected_kind + << " expected but processing root node\n"; + } + return false; + } + if (token_kind() != expected_kind) { + if (trace_) { + *trace_ << "Token " << expected_kind << " expected for " + << tree_->node_kind(node_id_) << ", found " << token_kind() + << "\n"; + } + return false; + } + return true; +} + +// Extract the token corresponding to a node. +template +struct Extractable> { + static auto Extract(NodeExtractor& extractor) -> std::optional> { + if (extractor.MatchesTokenKind(Kind)) { + return Token{.index = extractor.token()}; + } else { + return std::nullopt; + } + } +}; + +// Extract the token corresponding to a node. +template <> +struct Extractable { + static auto Extract(NodeExtractor& extractor) -> std::optional { + if (!extractor.has_token()) { + if (auto* trace = extractor.trace()) { + *trace << "Token expected but processing root node\n"; + } + return std::nullopt; + } + return AnyToken{.index = extractor.token()}; + } +}; + template -static auto ExtractTupleLikeType(const Tree* tree, Tree::SiblingIterator& it, - Tree::SiblingIterator end, ErrorBuilder* trace, - std::index_sequence /*indices*/, - std::tuple* /*type*/) +auto NodeExtractor::ExtractTupleLikeType( + std::index_sequence /*indices*/, std::tuple* /*type*/) -> std::optional { std::tuple...> fields; - if (trace) { - *trace << "Aggregate " << typeid(T).name() << ": begin\n"; + if (trace_) { + *trace_ << "Aggregate " << typeid(T).name() << ": begin\n"; } // Use a fold over the `=` operator to parse fields from right to left. [[maybe_unused]] int unused; bool ok = true; static_cast( - ((ok && (ok = (std::get(fields) = - Extractable::Extract(tree, it, end, trace)) + ((ok && (ok = (std::get(fields) = Extractable::Extract(*this)) .has_value()), unused) = ... = 0)); if (!ok) { - if (trace) { - *trace << "Aggregate " << typeid(T).name() << ": error\n"; + if (trace_) { + *trace_ << "Aggregate " << typeid(T).name() << ": error\n"; } return std::nullopt; } - if (trace) { - *trace << "Aggregate " << typeid(T).name() << ": success\n"; + if (trace_) { + *trace_ << "Aggregate " << typeid(T).name() << ": success\n"; } return T{std::move(std::get(fields).value())...}; } +namespace { // Extract the fields of a simple aggregate type. template struct Extractable { static_assert(std::is_aggregate_v, "Unsupported child type"); - static auto ExtractImpl(const Tree* tree, Tree::SiblingIterator& it, - Tree::SiblingIterator end, ErrorBuilder* trace) - -> std::optional { + static auto ExtractImpl(NodeExtractor& extractor) -> std::optional { // Compute the corresponding tuple type. using TupleType = decltype(StructReflection::AsTuple(std::declval())); - return ExtractTupleLikeType( - tree, it, end, trace, + return extractor.ExtractTupleLikeType( std::make_index_sequence>(), static_cast(nullptr)); } - static auto Extract(const Tree* tree, Tree::SiblingIterator& it, - Tree::SiblingIterator end, ErrorBuilder* trace) - -> std::optional { + static auto Extract(NodeExtractor& extractor) -> std::optional { static_assert(!HasKindMember, "Missing Id suffix"); - return ExtractImpl(tree, it, end, trace); + return ExtractImpl(extractor); } }; +} // namespace template auto Tree::TryExtractNodeFromChildren( - llvm::iterator_range children, + NodeId node_id, llvm::iterator_range children, ErrorBuilder* trace) const -> std::optional { - auto it = children.begin(); - auto result = Extractable::ExtractImpl(this, it, children.end(), trace); - if (it != children.end()) { + NodeExtractor extractor(this, tokens_, trace, node_id, children); + auto result = Extractable::ExtractImpl(extractor); + if (!extractor.at_end()) { if (trace) { - *trace << "Error: " << node_kind(*it) << " node left unconsumed."; + *trace << "Error: " << node_kind(extractor.ExtractNode()) + << " node left unconsumed."; } return std::nullopt; } @@ -324,9 +419,9 @@ auto Tree::TryExtractNodeFromChildren( } // Manually instantiate Tree::TryExtractNodeFromChildren -#define CARBON_PARSE_NODE_KIND(KindName) \ - template auto Tree::TryExtractNodeFromChildren( \ - llvm::iterator_range children, \ +#define CARBON_PARSE_NODE_KIND(KindName) \ + template auto Tree::TryExtractNodeFromChildren( \ + NodeId node_id, llvm::iterator_range children, \ ErrorBuilder * trace) const -> std::optional; // Also instantiate for `File`, even though it isn't a parse node. @@ -334,7 +429,7 @@ CARBON_PARSE_NODE_KIND(File) #include "toolchain/parse/node_kind.def" auto Tree::ExtractFile() const -> File { - return ExtractNodeFromChildren(roots()); + return ExtractNodeFromChildren(NodeId::Invalid, roots()); } } // namespace Carbon::Parse diff --git a/toolchain/parse/node_ids.h b/toolchain/parse/node_ids.h index dc69358ff839..14f915974149 100644 --- a/toolchain/parse/node_ids.h +++ b/toolchain/parse/node_ids.h @@ -6,6 +6,7 @@ #define CARBON_TOOLCHAIN_PARSE_NODE_IDS_H_ #include "toolchain/base/index_base.h" +#include "toolchain/lex/token_index.h" #include "toolchain/parse/node_kind.h" namespace Carbon::Parse { @@ -110,6 +111,32 @@ struct NodeIdNot : public NodeId { : NodeId(NodeId::InvalidIndex) {} }; +// This class holds the token corresponding to a parse node, and defines the +// expected token kind. The specified token index will always have kind K if the +// enclosing node doesn't have errors. Note that we never try to extract a node +// that has errors, and there are no restrictions on the token kind in that +// case. +template +struct Token { + static constexpr const Lex::TokenKind& Kind = K; + Lex::TokenIndex index; +}; + +// This class holds the token corresponding to a parse node in the case where +// the parse node can correspond to any token. This should only be used when the +// node kind is either not used in a finished tree, such as `Placeholder`, or is +// always invalid, such as `InvalidParse`. +struct AnyToken { + Lex::TokenIndex index; +}; + +// This class holds the token corresponding to a parse node in the case of a +// virtual token. The parse node doesn't actually own the token in this case. +template +struct VirtualToken { + Token token; +}; + // Note that the support for extracting these types using the `Tree::Extract*` // functions is defined in `extract.cpp`. diff --git a/toolchain/parse/node_kind.cpp b/toolchain/parse/node_kind.cpp index d49a8f0d50d7..626ebb9fa80d 100644 --- a/toolchain/parse/node_kind.cpp +++ b/toolchain/parse/node_kind.cpp @@ -4,7 +4,6 @@ #include "toolchain/parse/node_kind.h" -#include "common/check.h" #include "llvm/ADT/StringExtras.h" #include "toolchain/parse/typed_nodes.h" @@ -37,30 +36,10 @@ CARBON_DEFINE_ENUM_CLASS_NAMES(NodeKind) = { #include "toolchain/parse/node_kind.def" }; -auto NodeKind::CheckMatchesTokenKind(Lex::TokenKind token_kind, bool has_error) - -> void { - static constexpr Lex::TokenKind TokenIfValid[] = { -#define CARBON_IF_VALID(LexTokenKind) LexTokenKind -#define CARBON_PARSE_NODE_KIND_WITH_TOKEN(Name, LexTokenKind) \ - Lex::TokenKind::LexTokenKind, +// Check that each typed node defines a `token` member. +#define CARBON_PARSE_NODE_KIND(Name) \ + static_assert(requires(Name node) { node.token; }); #include "toolchain/parse/node_kind.def" - }; - static constexpr Lex::TokenKind TokenIfError[] = { -#define CARBON_IF_VALID(LexTokenKind) Error -#define CARBON_PARSE_NODE_KIND_WITH_TOKEN(Name, LexTokenKind) \ - Lex::TokenKind::LexTokenKind, -#include "toolchain/parse/node_kind.def" - }; - - Lex::TokenKind expected_token_kind = - has_error ? TokenIfError[AsInt()] : TokenIfValid[AsInt()]; - // Error indicates that the kind shouldn't be enforced. - CARBON_CHECK(Lex::TokenKind::Error == expected_token_kind || - token_kind == expected_token_kind) - << "Created parse node with NodeKind " << *this << " and has_error " - << has_error << " for lexical token kind " << token_kind - << ", but expected token kind " << expected_token_kind; -} auto NodeKind::has_bracket() const -> bool { return definition().has_bracket(); diff --git a/toolchain/parse/node_kind.def b/toolchain/parse/node_kind.def index aaa8752ac040..dbb8b9e79805 100644 --- a/toolchain/parse/node_kind.def +++ b/toolchain/parse/node_kind.def @@ -9,47 +9,30 @@ // // Supported x-macros are: // - CARBON_PARSE_NODE_KIND(Name) -// Used as a fallback if other macros are missing. -// - CARBON_PARSE_NODE_KIND_WITH_TOKEN(Name, LexTokenKind) -// Defines a parse node kind and specifies its corresponding token kind. -// - CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Name) -// Defines a parse node for a prefix operator, with the Name as token. -// - CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Name) -// Defines a parse node for an infix operator, with the Name as token. -// - CARBON_PARSE_NODE_KIND_POSTFIX_OPERATOR(Name) -// Defines a parse node for a postfix operator, with the Name as token. -// - CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(Name, LexTokenKind) -// Defines a parse node that corresponds to a token that is a single-token -// literal. The token is wrapped for LexTokenKinds. -// - CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER(Name) -// A token-based modifier. The Name is the TokenKind, and will be appended -// with "Modifier" for the parse kind. +// Used as a fallback if other macros are missing. Also, used directly +// when none of the below category-oriented macros apply. +// - CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Name) +// Defines a parse node for a prefix operator, with the Name as token. +// - CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Name) +// Defines a parse node for an infix operator, with the Name as token. +// - CARBON_PARSE_NODE_KIND_POSTFIX_OPERATOR(Name) +// Defines a parse node for a postfix operator, with the Name as token. +// - CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(Name, LexTokenKind) +// Defines a parse node that corresponds to a token that is a single-token +// literal. The token is wrapped for LexTokenKinds. +// - CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER(Name) +// A token-based modifier. The Name is the TokenKind, and will be appended +// with "Modifier" for the parse kind. // // LexTokenKind indicates the token kind required on a valid node. -// It will be either `kind_name` to indicate it's required even if the node is -// an error, or `CARBON_IF_VALID(kind_name)` to indicate that the kind is -// not enforced on error. A token kind of `Error` may be used if the kind should -// never be enforced; this should only be used when the node is either not in -// trees (`Placeholder`) or is always invalid (such as `InvalidParse`). -// -// This tree represents the subset relationship between these macros, where if a -// specific x-macro isn't defined, it'll fall back to the parent macro. // // Parse nodes are clustered based on language feature. See typed_nodes.h for // the expected tree structure under each node kind. -#if !(defined(CARBON_PARSE_NODE_KIND) || \ - defined(CARBON_PARSE_NODE_KIND_WITH_TOKEN)) +#if !defined(CARBON_PARSE_NODE_KIND) #error "Must define CARBON_PARSE_NODE_KIND family x-macros to use this file." #endif -// The WITH_TOKEN macro will use CARBON_PARSE_NODE_KIND by default when -// undefined. -#ifndef CARBON_PARSE_NODE_KIND_WITH_TOKEN -#define CARBON_PARSE_NODE_KIND_WITH_TOKEN(Name, LexTokenKind) \ - CARBON_PARSE_NODE_KIND(Name) -#endif - // This is expected to be used with something like: // // // Use x-macros to handle modifier cases. @@ -57,7 +40,7 @@ // #define CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Name, ...) #ifndef CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR #define CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Name) \ - CARBON_PARSE_NODE_KIND_WITH_TOKEN(PrefixOperator##Name, Name) + CARBON_PARSE_NODE_KIND(PrefixOperator##Name) #endif // This is expected to be used with something like: @@ -67,7 +50,7 @@ // #define CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Name, ...) #ifndef CARBON_PARSE_NODE_KIND_INFIX_OPERATOR #define CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Name) \ - CARBON_PARSE_NODE_KIND_WITH_TOKEN(InfixOperator##Name, Name) + CARBON_PARSE_NODE_KIND(InfixOperator##Name) #endif // This is expected to be used with something like: @@ -77,7 +60,7 @@ // #define CARBON_PARSE_NODE_KIND_POSTFIX_OPERATOR(Name, ...) #ifndef CARBON_PARSE_NODE_KIND_POSTFIX_OPERATOR #define CARBON_PARSE_NODE_KIND_POSTFIX_OPERATOR(Name) \ - CARBON_PARSE_NODE_KIND_WITH_TOKEN(PostfixOperator##Name, Name) + CARBON_PARSE_NODE_KIND(PostfixOperator##Name) #endif // This is expected to be used with something like: @@ -87,7 +70,7 @@ // #define CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(Name, ...) #ifndef CARBON_PARSE_NODE_KIND_TOKEN_LITERAL #define CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(Name, LexTokenKinds) \ - CARBON_PARSE_NODE_KIND_WITH_TOKEN(Name, LexTokenKinds) + CARBON_PARSE_NODE_KIND(Name) #endif // This is expected to be used with something like: @@ -97,405 +80,144 @@ // #define CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER(Name, ...) #ifndef CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER #define CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER(Name) \ - CARBON_PARSE_NODE_KIND_WITH_TOKEN(Name##Modifier, Name) + CARBON_PARSE_NODE_KIND(Name##Modifier) #endif -// The start of the file. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(FileStart, FileStart) +CARBON_PARSE_NODE_KIND(FileStart) +CARBON_PARSE_NODE_KIND(FileEnd) -// The end of the file. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(FileEnd, FileEnd) +CARBON_PARSE_NODE_KIND(InvalidParse) +CARBON_PARSE_NODE_KIND(InvalidParseStart) +CARBON_PARSE_NODE_KIND(InvalidParseSubtree) -// An invalid parse. Used to balance the parse tree. Always has an error. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(InvalidParse, Error) +CARBON_PARSE_NODE_KIND(Placeholder) -// An invalid subtree. Always has an error. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(InvalidParseStart, Error) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(InvalidParseSubtree, Error) +CARBON_PARSE_NODE_KIND(EmptyDecl) -// A placeholder node to be replaced; it will never exist in a valid parse tree. -// Its token kind is not enforced even when valid. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(Placeholder, Error) +CARBON_PARSE_NODE_KIND(IdentifierName) -// An empty declaration, such as `;`. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(EmptyDecl, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(IdentifierNameExpr) -// An identifier name in a non-expression context, such as a declaration. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(IdentifierName, CARBON_IF_VALID(Identifier)) +CARBON_PARSE_NODE_KIND(SelfValueName) +CARBON_PARSE_NODE_KIND(SelfValueNameExpr) +CARBON_PARSE_NODE_KIND(SelfTypeNameExpr) -// An identifier name in an expression context. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(IdentifierNameExpr, Identifier) +CARBON_PARSE_NODE_KIND(BaseName) +CARBON_PARSE_NODE_KIND(PackageExpr) +CARBON_PARSE_NODE_KIND(PackageName) +CARBON_PARSE_NODE_KIND(LibraryName) -// The `self` value and `Self` type identifier keywords. Typically of the form -// `self: Self`. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(SelfValueName, SelfValueIdentifier) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(SelfValueNameExpr, SelfValueIdentifier) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(SelfTypeNameExpr, SelfTypeIdentifier) +CARBON_PARSE_NODE_KIND(PackageIntroducer) +CARBON_PARSE_NODE_KIND(PackageDecl) -// The `base` value keyword, introduced by `base: B`. Typically referenced in -// an expression, as in `x.base` or `{.base = ...}`, but can also be used as a -// declared name, as in `{.base: partial B}`. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(BaseName, Base) +CARBON_PARSE_NODE_KIND(ImportIntroducer) +CARBON_PARSE_NODE_KIND(ImportDecl) -// The `package` keyword in an expression. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(PackageExpr, Package) +CARBON_PARSE_NODE_KIND(DefaultLibrary) +CARBON_PARSE_NODE_KIND(LibraryIntroducer) +CARBON_PARSE_NODE_KIND(LibraryDecl) -// ---------------------------------------------------------------------------- +CARBON_PARSE_NODE_KIND(LibrarySpecifier) -// The comments below follow this pattern: -// -// // Descriptive heading: -// // Child1 -// // Child2 -// // Parent -// -// In this, `Child1`, `Child2`, and `Parent` are all kinds of parse nodes, which -// are then defined using the `CARBON_PARSE_NODE_KIND_*` macros. They are -// written in postorder, with the indentation showing the tree structure. See -// tree.h for more information. -// -// A parse node kind may be preceded by: -// - `_optional_` if this node (or nodes) may be present or omitted in valid -// parses, depending on which tokens are in the source code. -// - `_repeated_` if this node (or nodes) may be repeated or omitted in valid -// parses, depending on which tokens are in the source code. -// - `_external_:` if this node is the child of multiple kinds of nodes and -// is documented separately. -// -// There is generally a close correspondence between handling of tokens and the -// creation of non-external nodes in a given block. +CARBON_PARSE_NODE_KIND(NameQualifier) -// ---------------------------------------------------------------------------- +CARBON_PARSE_NODE_KIND(ExportIntroducer) +CARBON_PARSE_NODE_KIND(ExportDecl) -// The name of a package or library for `package`, `import`, and `library`. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(PackageName, Identifier) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(LibraryName, StringLiteral) +CARBON_PARSE_NODE_KIND(NamespaceStart) +CARBON_PARSE_NODE_KIND(Namespace) -// `package`: -// PackageIntroducer -// _repeated_ _external_: modifier -// _optional_ _external_: PackageName -// _optional_ _external_: LibrarySpecifier -// PackageDecl -CARBON_PARSE_NODE_KIND_WITH_TOKEN(PackageIntroducer, Package) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(PackageDecl, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(CodeBlockStart) +CARBON_PARSE_NODE_KIND(CodeBlock) -// `import`: -// ImportIntroducer -// _repeated_ _external_: modifier -// _optional_ _external_: PackageName -// _optional_ _external_: LibrarySpecifier -// ImportDecl -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ImportIntroducer, Import) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ImportDecl, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(FunctionIntroducer) +CARBON_PARSE_NODE_KIND(ReturnType) +CARBON_PARSE_NODE_KIND(FunctionDefinitionStart) +CARBON_PARSE_NODE_KIND(FunctionDefinition) +CARBON_PARSE_NODE_KIND(FunctionDecl) +CARBON_PARSE_NODE_KIND(BuiltinFunctionDefinitionStart) +CARBON_PARSE_NODE_KIND(BuiltinName) +CARBON_PARSE_NODE_KIND(BuiltinFunctionDefinition) -// `library` as declaration: -// LibraryIntroducer -// _repeated_ _external_: modifier -// DefaultLibrary or _external_: LibraryName -// LibraryDecl -CARBON_PARSE_NODE_KIND_WITH_TOKEN(DefaultLibrary, Default) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(LibraryIntroducer, Library) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(LibraryDecl, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(AliasIntroducer) +CARBON_PARSE_NODE_KIND(AliasInitializer) +CARBON_PARSE_NODE_KIND(Alias) -// `library` in `package` or `import`: -// _external_: LibraryName or DefaultLibrary -// LibrarySpecifier -CARBON_PARSE_NODE_KIND_WITH_TOKEN(LibrarySpecifier, Library) +CARBON_PARSE_NODE_KIND(TuplePatternStart) +CARBON_PARSE_NODE_KIND(PatternListComma) +CARBON_PARSE_NODE_KIND(TuplePattern) -// Declaration names. -// -// _repeated_: NameQualifier -// _external_: IdentifierName -// _optional_ _external_: ImplicitParamList -// _optional_ _external_: TuplePattern -// _declaration name_ -// -// _external_: IdentifierName -// _optional_ _external_: ImplicitParamList -// _optional_ _external_: TuplePattern -// NameQualifier -CARBON_PARSE_NODE_KIND_WITH_TOKEN(NameQualifier, CARBON_IF_VALID(Period)) +CARBON_PARSE_NODE_KIND(ImplicitParamListStart) +CARBON_PARSE_NODE_KIND(ImplicitParamList) -// `export`: -// ExportIntroducer -// _external_: _declaration name_ -// ExportDecl -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ExportIntroducer, Export) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ExportDecl, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(ArrayExprStart) +CARBON_PARSE_NODE_KIND(ArrayExprSemi) +CARBON_PARSE_NODE_KIND(ArrayExpr) -// `namespace`: -// NamespaceStart -// _repeated_ _external_: modifier -// _external_: _declaration name_ -// Namespace -CARBON_PARSE_NODE_KIND_WITH_TOKEN(NamespaceStart, Namespace) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(Namespace, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(BindingPattern) +CARBON_PARSE_NODE_KIND(CompileTimeBindingPattern) +CARBON_PARSE_NODE_KIND(Addr) +CARBON_PARSE_NODE_KIND(Template) -// A code block: -// CodeBlockStart -// _repeated_ _external_: statement -// CodeBlock -CARBON_PARSE_NODE_KIND_WITH_TOKEN(CodeBlockStart, - CARBON_IF_VALID(OpenCurlyBrace)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(CodeBlock, CARBON_IF_VALID(CloseCurlyBrace)) +CARBON_PARSE_NODE_KIND(LetIntroducer) +CARBON_PARSE_NODE_KIND(LetInitializer) +CARBON_PARSE_NODE_KIND(LetDecl) -// `fn` declarations start with a function signature: -// -// FunctionIntroducer -// _repeated_ _external_: modifier -// _external_: _declaration name_ -// ReturnType -// _function signature_ -// -// There are three forms of function declaration: -// -// _function signature_ -// FunctionDecl -// -// _function signature_ -// FunctionDefinitionStart -// _repeated_ _external_: statement -// FunctionDefinition -// -// _function signature_ -// BuiltinFunctionDefinitionStart -// BuiltinName -// BuiltinFunctionDefinition -CARBON_PARSE_NODE_KIND_WITH_TOKEN(FunctionIntroducer, Fn) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ReturnType, MinusGreater) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(FunctionDefinitionStart, OpenCurlyBrace) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(FunctionDefinition, CloseCurlyBrace) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(FunctionDecl, CARBON_IF_VALID(Semi)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(BuiltinFunctionDefinitionStart, Equal) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(BuiltinName, StringLiteral) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(BuiltinFunctionDefinition, - CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(VariableIntroducer) +CARBON_PARSE_NODE_KIND(ReturnedModifier) +CARBON_PARSE_NODE_KIND(VariableInitializer) +CARBON_PARSE_NODE_KIND(VariableDecl) -// `alias`: -// AliasIntroducer -// _repeated_ _external_: modifier -// _external_: _declaration name_ -// AliasInitializer -// _external_: expression -// Alias -CARBON_PARSE_NODE_KIND_WITH_TOKEN(AliasIntroducer, Alias) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(AliasInitializer, Equal) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(Alias, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(ExprStatement) -// A tuple pattern: -// TuplePatternStart -// _external_: [Generic]BindingPattern -// PatternListComma -// _repeated_ -// TuplePattern -// -// Patterns and PatternListComma may repeat with PatternListComma as a -// separator. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(TuplePatternStart, OpenParen) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(PatternListComma, Comma) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(TuplePattern, CloseParen) +CARBON_PARSE_NODE_KIND(BreakStatementStart) +CARBON_PARSE_NODE_KIND(BreakStatement) -// An implicit parameter list: -// ImplicitParamListStart -// _external_: [Generic]BindingPattern -// PatternListComma -// _repeated_ -// ImplicitParamList -// -// Patterns and PatternListComma may repeat with PatternListComma as a -// separator. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ImplicitParamListStart, OpenSquareBracket) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ImplicitParamList, CloseSquareBracket) +CARBON_PARSE_NODE_KIND(ContinueStatementStart) +CARBON_PARSE_NODE_KIND(ContinueStatement) -// An array type, such as `[i32; 3]` or `[i32;]`: -// ArrayExprStart -// _external_: type expression -// ArrayExprSemi -// _optional_ _external_: expression -// ArrayExpr -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ArrayExprStart, OpenSquareBracket) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ArrayExprSemi, CARBON_IF_VALID(Semi)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ArrayExpr, CloseSquareBracket) +CARBON_PARSE_NODE_KIND(ReturnStatementStart) +CARBON_PARSE_NODE_KIND(ReturnVarModifier) +CARBON_PARSE_NODE_KIND(ReturnStatement) -// A binding pattern, such as `name: Type`: -// IdentifierName or SelfValueName -// _external_: type expression -// [Generic]BindingPattern -// _optional_ Addr -// _optional_ Template -CARBON_PARSE_NODE_KIND_WITH_TOKEN(BindingPattern, CARBON_IF_VALID(Colon)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(CompileTimeBindingPattern, ColonExclaim) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(Addr, Addr) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(Template, Template) +CARBON_PARSE_NODE_KIND(ForHeaderStart) +CARBON_PARSE_NODE_KIND(ForIn) +CARBON_PARSE_NODE_KIND(ForHeader) +CARBON_PARSE_NODE_KIND(ForStatement) -// `let` declarations, including associated constant declarations: -// LetIntroducer -// _repeated_ _external_: modifier -// _external_: BindingPattern or TuplePattern -// LetInitializer -// _external_: expression -// _optional_ -// LetDecl -// -// Modifier keywords only appear for `let` declarations, not `let` statements. -// -// The LetInitializer and following expression are paired: either both will be -// present, or neither will. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(LetIntroducer, Let) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(LetInitializer, Equal) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(LetDecl, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(IfConditionStart) +CARBON_PARSE_NODE_KIND(IfCondition) +CARBON_PARSE_NODE_KIND(IfStatementElse) +CARBON_PARSE_NODE_KIND(IfStatement) -// `var` and `returned var`: -// VariableIntroducer -// _repeated_ _external_: modifier -// _optional_ ReturnedModifier -// _external_: BindingPattern or TuplePattern -// VariableInitializer -// _external_: expression -// _optional_ -// VariableDecl -// -// Access and declaration modifier keywords only appear for `var` declarations, -// whereas the returned modifier only appears on `var` statements. -// -// The VariableInitializer and following expression are paired: either both will -// be present, or neither will. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(VariableIntroducer, CARBON_IF_VALID(Var)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ReturnedModifier, Returned) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(VariableInitializer, Equal) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(VariableDecl, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(WhileConditionStart) +CARBON_PARSE_NODE_KIND(WhileCondition) +CARBON_PARSE_NODE_KIND(WhileStatement) -// An expression statement: -// _external_: expression -// ExprStatement -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ExprStatement, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(IndexExprStart) +CARBON_PARSE_NODE_KIND(IndexExpr) -// `break`: -// BreakStatementStart -// BreakStatement -CARBON_PARSE_NODE_KIND_WITH_TOKEN(BreakStatementStart, Break) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(BreakStatement, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(ParenExprStart) +CARBON_PARSE_NODE_KIND(ParenExpr) -// `continue`: -// ContinueStatementStart -// ContinueStatement -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ContinueStatementStart, Continue) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ContinueStatement, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(TupleLiteralStart) +CARBON_PARSE_NODE_KIND(TupleLiteralComma) +CARBON_PARSE_NODE_KIND(TupleLiteral) -// `return`: -// ReturnStatementStart -// _optional_ ReturnVarModifier or _external_: expression -// ReturnStatement -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ReturnStatementStart, Return) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ReturnVarModifier, Var) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ReturnStatement, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(CallExprStart) +CARBON_PARSE_NODE_KIND(CallExprComma) +CARBON_PARSE_NODE_KIND(CallExpr) -// `for`: -// ForHeaderStart -// VariableIntroducer -// _external_: BindingPattern -// ForIn -// _external_: expression -// ForHeader -// _external_: CodeBlock -// ForStatement -// -// Versus a normal `var`, ForIn replaces VariableDecl. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ForHeaderStart, CARBON_IF_VALID(OpenParen)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ForIn, CARBON_IF_VALID(In)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ForHeader, CARBON_IF_VALID(CloseParen)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ForStatement, For) +CARBON_PARSE_NODE_KIND(MemberAccessExpr) -// `if` statement + `else`: -// IfConditionStart -// _external_: expression -// IfCondition -// _external_: CodeBlock -// IfStatementElse -// _external_: CodeBlock or IfStatement -// IfStatement -// -// IfStatementElse and the following node are optional based on `else` presence. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(IfConditionStart, CARBON_IF_VALID(OpenParen)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(IfCondition, CARBON_IF_VALID(CloseParen)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(IfStatementElse, Else) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(IfStatement, If) +CARBON_PARSE_NODE_KIND(PointerMemberAccessExpr) -// `while`: -// WhileConditionStart -// _external_: expression -// WhileCondition -// _external_: CodeBlock -// WhileStatement -CARBON_PARSE_NODE_KIND_WITH_TOKEN(WhileConditionStart, - CARBON_IF_VALID(OpenParen)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(WhileCondition, CARBON_IF_VALID(CloseParen)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(WhileStatement, While) - -// Index expressions, such as `a[1]`: -// _external_: expression -// IndexExprStart -// _external_: expression -// IndexExpr -CARBON_PARSE_NODE_KIND_WITH_TOKEN(IndexExprStart, OpenSquareBracket) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(IndexExpr, CloseSquareBracket) - -// Parenthesized single expressions, such as `(2)`: -// ParenExprStart -// _external_: expression -// ParenExpr -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ParenExprStart, OpenParen) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ParenExpr, CloseParen) - -// Tuples, such as `(1, 2)`: -// TupleLiteralStart -// _external_: expression -// TupleLiteralComma -// _repeated_ -// TupleLiteral -// -// Expressions and TupleLiteralComma may repeat with TupleLiteralComma as a -// separator. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(TupleLiteralStart, OpenParen) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(TupleLiteralComma, Comma) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(TupleLiteral, CloseParen) - -// Call expressions, such as `a()`: -// _external_: expression -// CallExprStart -// _external_: expression -// CallExprComma -// _repeated_ -// CallExpr -// -// Exprs and CallExprComma may repeat with CallExprComma as a separator. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(CallExprStart, OpenParen) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(CallExprComma, Comma) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(CallExpr, CloseParen) - -// A member access expression, such as `a.b` or -// `GetObject().(Interface.member)`: -// _external_: lhs expression -// _external_: rhs expression -// MemberAccessExpr -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MemberAccessExpr, Period) - -// A pointer member access expression, such as `a->b` or -// `GetObject()->(Interface.member)`: -// _external_: lhs expression -// _external_: rhs expression -// PointerMemberAccessExpr -CARBON_PARSE_NODE_KIND_WITH_TOKEN(PointerMemberAccessExpr, MinusGreater) - -// A value literal. CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(BoolLiteralFalse, False) CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(BoolLiteralTrue, True) CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(IntLiteral, IntLiteral) CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(RealLiteral, RealLiteral) CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(StringLiteral, StringLiteral) -// A type literal. CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(BoolTypeLiteral, Bool) CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(IntTypeLiteral, IntTypeLiteral) CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(UnsignedIntTypeLiteral, @@ -505,9 +227,6 @@ CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(StringTypeLiteral, StringTypeLiteral) CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(TypeTypeLiteral, Type) CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(AutoTypeLiteral, Auto) -// A prefix operator, such as `not`: -// _external_: expression -// PrefixOperator CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Amp) CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Caret) CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Const) @@ -517,10 +236,6 @@ CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(MinusMinus) CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(PlusPlus) CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Star) -// An infix operator, such as `+`: -// _external_: lhs expression -// _external_: rhs expression -// InfixOperator CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Amp) CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(AmpEqual) CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(As) @@ -551,68 +266,26 @@ CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(SlashEqual) CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Star) CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(StarEqual) -// A postfix operator, currently only `*`: -// _external_: expression -// PostfixOperator CARBON_PARSE_NODE_KIND_POSTFIX_OPERATOR(Star) -// A short-circuiting infix operator, such as `and`: -// _external_: expression -// ShortCircuitOperand(And|Or) -// _external_: expression -// ShortCircuitOperand(And|Or) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ShortCircuitOperandAnd, And) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ShortCircuitOperandOr, Or) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ShortCircuitOperatorAnd, And) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ShortCircuitOperatorOr, Or) +CARBON_PARSE_NODE_KIND(ShortCircuitOperandAnd) +CARBON_PARSE_NODE_KIND(ShortCircuitOperandOr) +CARBON_PARSE_NODE_KIND(ShortCircuitOperatorAnd) +CARBON_PARSE_NODE_KIND(ShortCircuitOperatorOr) -// `if` expression + `then` + `else`: -// _external_: expression -// IfExprIf -// _external_: expression -// IfExprThen -// _external_: expression -// IfExprElse -CARBON_PARSE_NODE_KIND_WITH_TOKEN(IfExprIf, If) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(IfExprThen, Then) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(IfExprElse, CARBON_IF_VALID(Else)) +CARBON_PARSE_NODE_KIND(IfExprIf) +CARBON_PARSE_NODE_KIND(IfExprThen) +CARBON_PARSE_NODE_KIND(IfExprElse) -// Struct literals, such as `{.a = 0}`: -// StructLiteralStart -// _external_: IdentifierName or BaseName -// StructFieldDesignator -// _external_: expression -// StructField -// StructComma -// _repeated_ -// StructLiteral -// -// Struct type literals, such as `{.a: i32}`: -// StructTypeLiteralStart -// _external_: IdentifierName or BaseName -// StructFieldDesignator -// _external_: type expression -// StructTypeField -// StructComma -// _repeated_ -// StructTypeLiteral -// -// Elements (StructField and StructTypeField, respectively) and StructComma -// may repeat with StructComma as a separator. -// -// When a valid StructTypeField or StructField cannot be formed, elements -// may be replaced by InvalidParse, which may have a preceding sibling -// StructFieldDesignator if one was successfully parsed. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(StructLiteralStart, OpenCurlyBrace) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(StructTypeLiteralStart, OpenCurlyBrace) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(StructFieldDesignator, Period) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(StructField, Equal) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(StructTypeField, Colon) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(StructComma, Comma) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(StructLiteral, CloseCurlyBrace) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(StructTypeLiteral, CloseCurlyBrace) +CARBON_PARSE_NODE_KIND(StructLiteralStart) +CARBON_PARSE_NODE_KIND(StructTypeLiteralStart) +CARBON_PARSE_NODE_KIND(StructFieldDesignator) +CARBON_PARSE_NODE_KIND(StructField) +CARBON_PARSE_NODE_KIND(StructTypeField) +CARBON_PARSE_NODE_KIND(StructComma) +CARBON_PARSE_NODE_KIND(StructLiteral) +CARBON_PARSE_NODE_KIND(StructTypeLiteral) -// Various modifiers. These are all a single token. CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER(Abstract) CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER(Base) CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER(Default) @@ -625,186 +298,59 @@ CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER(Private) CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER(Protected) CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER(Virtual) -// `class`: -// ClassIntroducer -// _repeated_ _external_: modifier -// _external_: _declaration name_ -// _optional_ _external_: ImplicitParamList -// _optional_ _external_: TuplePattern -// ClassDefinitionStart -// _external_: declarations -// ClassDefinition -// -// The above is the structure for a definition; for a declaration, -// ClassDefinitionStart and later nodes are removed and replaced by -// ClassDecl. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ClassIntroducer, Class) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ClassDefinitionStart, OpenCurlyBrace) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ClassDefinition, CloseCurlyBrace) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ClassDecl, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(ClassIntroducer) +CARBON_PARSE_NODE_KIND(ClassDefinitionStart) +CARBON_PARSE_NODE_KIND(ClassDefinition) +CARBON_PARSE_NODE_KIND(ClassDecl) -// `adapt`: -// AdaptIntroducer -// _repeated_ _external_: modifier -// _external_: expression -// AdaptDecl -CARBON_PARSE_NODE_KIND_WITH_TOKEN(AdaptIntroducer, Adapt) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(AdaptDecl, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(AdaptIntroducer) +CARBON_PARSE_NODE_KIND(AdaptDecl) -// `base`: -// BaseIntroducer -// _repeated_ _external_: modifier -// BaseColon -// _external_: expression -// BaseDecl -CARBON_PARSE_NODE_KIND_WITH_TOKEN(BaseIntroducer, Base) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(BaseColon, Colon) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(BaseDecl, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(BaseIntroducer) +CARBON_PARSE_NODE_KIND(BaseColon) +CARBON_PARSE_NODE_KIND(BaseDecl) -// `interface`: -// InterfaceIntroducer -// _repeated_ _external_: modifier -// _external_: _declaration name_ -// _optional_ _external_: ImplicitParamList -// _optional_ _external_: TuplePattern -// InterfaceDefinitionStart -// _external_: declarations -// InterfaceDefinition -// -// The above is the structure for a definition; for a declaration, -// InterfaceDefinitionStart and later nodes are removed and replaced by -// InterfaceDecl. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(InterfaceIntroducer, Interface) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(InterfaceDefinitionStart, OpenCurlyBrace) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(InterfaceDefinition, CloseCurlyBrace) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(InterfaceDecl, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(InterfaceIntroducer) +CARBON_PARSE_NODE_KIND(InterfaceDefinitionStart) +CARBON_PARSE_NODE_KIND(InterfaceDefinition) +CARBON_PARSE_NODE_KIND(InterfaceDecl) -// `impl ... as`: -// ImplIntroducer -// _repeated_ _external_: modifier -// _optional_ _external_: ImplForall -// _optional_ _external_: expression -// _external_: DefaultSelfImplAs or TypeImplAs -// _external_: expression -// ImplDefinitionStart -// _external_: declarations -// ImplDefinition -// -// The above is the structure for a definition; for a declaration, -// ImplDefinitionStart and later nodes are removed and replaced by -// ImplDecl. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ImplIntroducer, Impl) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ImplDefinitionStart, OpenCurlyBrace) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ImplDefinition, CloseCurlyBrace) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ImplDecl, CARBON_IF_VALID(Semi)) +CARBON_PARSE_NODE_KIND(ImplIntroducer) +CARBON_PARSE_NODE_KIND(ImplDefinitionStart) +CARBON_PARSE_NODE_KIND(ImplDefinition) +CARBON_PARSE_NODE_KIND(ImplDecl) +CARBON_PARSE_NODE_KIND(ImplForall) +CARBON_PARSE_NODE_KIND(TypeImplAs) +CARBON_PARSE_NODE_KIND(DefaultSelfImplAs) -// `forall ...`: -// _external_: ImplicitParamList -// ImplForall -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ImplForall, Forall) +CARBON_PARSE_NODE_KIND(NamedConstraintIntroducer) +CARBON_PARSE_NODE_KIND(NamedConstraintDefinitionStart) +CARBON_PARSE_NODE_KIND(NamedConstraintDefinition) +CARBON_PARSE_NODE_KIND(NamedConstraintDecl) -// `... as`: -// _external_: expression -// TypeImplAs -CARBON_PARSE_NODE_KIND_WITH_TOKEN(TypeImplAs, As) -// `as` without a type before it -CARBON_PARSE_NODE_KIND_WITH_TOKEN(DefaultSelfImplAs, As) +CARBON_PARSE_NODE_KIND(ChoiceIntroducer) +CARBON_PARSE_NODE_KIND(ChoiceDefinitionStart) +CARBON_PARSE_NODE_KIND(ChoiceDefinition) +CARBON_PARSE_NODE_KIND(ChoiceAlternativeListComma) -// `constraint`: -// NamedConstraintIntroducer -// _repeated_ _external_: modifier -// _external_: _declaration name_ -// _optional_ _external_: ImplicitParamList -// _optional_ _external_: TuplePattern -// NamedConstraintDefinitionStart -// _external_: declarations -// NamedConstraintDefinition -// -// The above is the structure for a definition; for a declaration, -// NamedConstraintDefinitionStart and later nodes are removed and replaced by -// NamedConstraintDecl. -CARBON_PARSE_NODE_KIND_WITH_TOKEN(NamedConstraintIntroducer, Constraint) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(NamedConstraintDefinitionStart, - OpenCurlyBrace) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(NamedConstraintDefinition, CloseCurlyBrace) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(NamedConstraintDecl, CARBON_IF_VALID(Semi)) - -// `choice`: -// ChoiceIntroducer -// _external_: _declaration name_ -// ChoiceDefinitionStart -// _optional_ _external_: ChoiceAlternativeList -// ChoiceDefinition -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ChoiceIntroducer, Choice) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ChoiceDefinitionStart, - CARBON_IF_VALID(OpenCurlyBrace)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ChoiceDefinition, - CARBON_IF_VALID(CloseCurlyBrace)) - -// Choice alternative list: -// _external_: IdentifierName -// _optional_ _external_ : TuplePattern -// _optional_: ChoiceAlternativeListComma -// _repeated_ -// ChoiceAlternativeList -CARBON_PARSE_NODE_KIND_WITH_TOKEN(ChoiceAlternativeListComma, Comma) - -// `match`: -// MatchIntroducer -// MatchConditionStart -// _external_: expression -// MatchCondition -// MatchStatementStart -// _repeated_ _external_: MatchCase -// _optional_ _external_: MatchStatementDefault -// MatchStatement -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchIntroducer, Match) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchConditionStart, - CARBON_IF_VALID(OpenParen)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchCondition, CARBON_IF_VALID(CloseParen)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchStatementStart, - CARBON_IF_VALID(OpenCurlyBrace)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchStatement, - CARBON_IF_VALID(CloseCurlyBrace)) - -// `case`: -// MatchCaseIntroducer -// _external_: Pattern -// MatchCaseGuardIntroducer -// MatchCaseGuardStart -// _external_: expression -// MatchCaseGuard -// MatchCaseEqualGreater -// MatchCaseStart -// _repeated_ _external_: statement -// MatchCase -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchCaseIntroducer, Case) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchCaseGuardIntroducer, If) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchCaseGuardStart, - CARBON_IF_VALID(OpenParen)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchCaseGuard, CARBON_IF_VALID(CloseParen)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchCaseEqualGreater, - CARBON_IF_VALID(EqualGreater)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchCaseStart, - CARBON_IF_VALID(OpenCurlyBrace)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchCase, CARBON_IF_VALID(CloseCurlyBrace)) - -// `default`: -// MatchDefaultIntroducer -// MatchDefaultEqualGreater -// MatchDefaultStart -// _repeated_ _external_: statement -// MatchDefault -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchDefaultIntroducer, Default) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchDefaultEqualGreater, - CARBON_IF_VALID(EqualGreater)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchDefaultStart, - CARBON_IF_VALID(OpenCurlyBrace)) -CARBON_PARSE_NODE_KIND_WITH_TOKEN(MatchDefault, - CARBON_IF_VALID(CloseCurlyBrace)) +CARBON_PARSE_NODE_KIND(MatchIntroducer) +CARBON_PARSE_NODE_KIND(MatchConditionStart) +CARBON_PARSE_NODE_KIND(MatchCondition) +CARBON_PARSE_NODE_KIND(MatchStatementStart) +CARBON_PARSE_NODE_KIND(MatchStatement) +CARBON_PARSE_NODE_KIND(MatchCaseIntroducer) +CARBON_PARSE_NODE_KIND(MatchCaseGuardIntroducer) +CARBON_PARSE_NODE_KIND(MatchCaseGuardStart) +CARBON_PARSE_NODE_KIND(MatchCaseGuard) +CARBON_PARSE_NODE_KIND(MatchCaseEqualGreater) +CARBON_PARSE_NODE_KIND(MatchCaseStart) +CARBON_PARSE_NODE_KIND(MatchCase) +CARBON_PARSE_NODE_KIND(MatchDefaultIntroducer) +CARBON_PARSE_NODE_KIND(MatchDefaultEqualGreater) +CARBON_PARSE_NODE_KIND(MatchDefaultStart) +CARBON_PARSE_NODE_KIND(MatchDefault) #undef CARBON_PARSE_NODE_KIND -#undef CARBON_PARSE_NODE_KIND_WITH_TOKEN #undef CARBON_PARSE_NODE_KIND_INFIX_OPERATOR #undef CARBON_PARSE_NODE_KIND_POSTFIX_OPERATOR #undef CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR diff --git a/toolchain/parse/tree.cpp b/toolchain/parse/tree.cpp index 2d15f9bc0613..f610f87cc6e7 100644 --- a/toolchain/parse/tree.cpp +++ b/toolchain/parse/tree.cpp @@ -170,12 +170,12 @@ auto Tree::Print(llvm::raw_ostream& output, bool preorder) const -> void { output << " ]\n"; } -static auto TestExtract(const Tree* tree, NodeId node_id, NodeKind kind, - ErrorBuilder* trace) -> bool { +auto Tree::VerifyExtract(NodeId node_id, NodeKind kind, + ErrorBuilder* trace) const -> bool { switch (kind) { #define CARBON_PARSE_NODE_KIND(Name) \ case NodeKind::Name: \ - return tree->VerifyExtractAs(node_id, trace).has_value(); + return VerifyExtractAs(node_id, trace).has_value(); #include "toolchain/parse/node_kind.def" } } @@ -200,12 +200,12 @@ auto Tree::Verify() const -> ErrorOr { // Without this code, a 10 mloc test case of lex & parse takes // 4.129 s ± 0.041 s. With this additional verification, it takes // 5.768 s ± 0.036 s. - if (!n_impl.has_error && !TestExtract(this, n, n_impl.kind, nullptr)) { + if (!n_impl.has_error && !VerifyExtract(n, n_impl.kind, nullptr)) { ErrorBuilder trace; trace << llvm::formatv( "NodeId #{0} couldn't be extracted as a {1}. Trace:\n", n, n_impl.kind); - TestExtract(this, n, n_impl.kind, &trace); + VerifyExtract(n, n_impl.kind, &trace); return trace; } @@ -273,10 +273,10 @@ auto Tree::Verify() const -> ErrorOr { } // Validate the roots, ensures Tree::ExtractFile() doesn't CHECK-fail. - if (!TryExtractNodeFromChildren(roots(), nullptr)) { + if (!TryExtractNodeFromChildren(NodeId::Invalid, roots(), nullptr)) { ErrorBuilder trace; trace << "Roots of tree couldn't be extracted as a `File`. Trace:\n"; - TryExtractNodeFromChildren(roots(), &trace); + TryExtractNodeFromChildren(NodeId::Invalid, roots(), &trace); return trace; } diff --git a/toolchain/parse/tree.h b/toolchain/parse/tree.h index e80cf7117745..b7017e12307e 100644 --- a/toolchain/parse/tree.h +++ b/toolchain/parse/tree.h @@ -255,7 +255,8 @@ class Tree : public Printable { // Extract a `File` object representing the parse tree for the whole file. // #include "toolchain/parse/typed_nodes.h" to get the definition of `File` - // and the types representing its children nodes. + // and the types representing its children nodes. This is implemented in + // extract.cpp. auto ExtractFile() const -> File; // Converts this node_id to a typed node of a specified type, if it is a valid @@ -276,14 +277,9 @@ class Tree : public Printable { // debugger. auto Verify() const -> ErrorOr; - // Like ExtractAs(), but malformed tree errors are not fatal. Should only be - // used by `Verify()`. - template - auto VerifyExtractAs(NodeId node_id, ErrorBuilder* trace) const - -> std::optional; - private: friend class Context; + friend class TypedNodesTestPeer; template struct ConvertTo; @@ -342,6 +338,24 @@ class Tree : public Printable { static_assert(sizeof(NodeImpl) == 12, "Unexpected size of node implementation!"); + // Like ExtractAs(), but malformed tree errors are not fatal. Should only be + // used by `Verify()` or by tests. + template + auto VerifyExtractAs(NodeId node_id, ErrorBuilder* trace) const + -> std::optional; + + // Wrapper around `VerifyExtractAs` to dispatch based on a runtime node kind. + // Returns true if extraction was successful. + auto VerifyExtract(NodeId node_id, NodeKind kind, ErrorBuilder* trace) const + -> bool; + + // Sets the kind of a node. This is intended to allow putting the tree into a + // state where verification can fail, in order to make the failure path of + // `Verify` testable. + auto SetNodeKindForTesting(NodeId node_id, NodeKind kind) -> void { + node_impls_[node_id.index].kind = kind; + } + // Prints a single node for Print(). Returns true when preorder and there are // children. auto PrintNode(llvm::raw_ostream& output, NodeId n, int depth, @@ -349,16 +363,17 @@ class Tree : public Printable { // Extract a node of type `T` from a sibling range. This is expected to // consume the complete sibling range. Malformed tree errors are written - // to `*trace`, if `trace != nullptr`. + // to `*trace`, if `trace != nullptr`. This is implemented in extract.cpp. template auto TryExtractNodeFromChildren( - llvm::iterator_range children, + NodeId node_id, llvm::iterator_range children, ErrorBuilder* trace) const -> std::optional; // Extract a node of type `T` from a sibling range. This is expected to // consume the complete sibling range. Malformed tree errors are fatal. template auto ExtractNodeFromChildren( + NodeId node_id, llvm::iterator_range children) const -> T; // Depth-first postorder sequence of node implementation data. @@ -476,12 +491,13 @@ class Tree::SiblingIterator template auto Tree::ExtractNodeFromChildren( - llvm::iterator_range children) const -> T { - auto result = TryExtractNodeFromChildren(children, nullptr); + NodeId node_id, llvm::iterator_range children) const + -> T { + auto result = TryExtractNodeFromChildren(node_id, children, nullptr); if (!result.has_value()) { // On error try again, this time capturing a trace. ErrorBuilder trace; - TryExtractNodeFromChildren(children, &trace); + TryExtractNodeFromChildren(node_id, children, &trace); CARBON_FATAL() << "Malformed parse node:\n" << static_cast(trace).message(); } @@ -495,7 +511,7 @@ auto Tree::ExtractAs(NodeId node_id) const -> std::optional { return std::nullopt; } - return ExtractNodeFromChildren(children(node_id)); + return ExtractNodeFromChildren(node_id, children(node_id)); } template @@ -503,10 +519,14 @@ auto Tree::VerifyExtractAs(NodeId node_id, ErrorBuilder* trace) const -> std::optional { static_assert(HasKindMember, "Not a parse node type"); if (!IsValid(node_id)) { + if (trace) { + *trace << "VerifyExtractAs error: wrong kind " << node_kind(node_id) + << ", expected " << T::Kind << "\n"; + } return std::nullopt; } - return TryExtractNodeFromChildren(children(node_id), trace); + return TryExtractNodeFromChildren(node_id, children(node_id), trace); } template @@ -517,7 +537,7 @@ auto Tree::Extract(IdT id) const } using T = typename NodeForId::TypedNode; - return ExtractNodeFromChildren(children(id)); + return ExtractNodeFromChildren(id, children(id)); } template diff --git a/toolchain/parse/typed_nodes.h b/toolchain/parse/typed_nodes.h index 319678cd0ee8..66cc5e6a2dfd 100644 --- a/toolchain/parse/typed_nodes.h +++ b/toolchain/parse/typed_nodes.h @@ -7,6 +7,7 @@ #include +#include "toolchain/lex/token_kind.h" #include "toolchain/parse/node_ids.h" #include "toolchain/parse/node_kind.h" @@ -27,11 +28,13 @@ template using CommaSeparatedList = llvm::SmallVector>; // This class provides a shorthand for defining parse node kinds for leaf nodes. -template struct LeafNode { static constexpr auto Kind = KindT.Define({.category = Category, .child_count = 0}); + + TokenKind token; }; // ---------------------------------------------------------------------------- @@ -77,11 +80,11 @@ struct LeafNode { // An invalid parse. Used to balance the parse tree. This type is here only to // ensure we have a type for each parse node kind. This node kind always has an // error, so can never be extracted. -using InvalidParse = - LeafNode; +using InvalidParse = LeafNode; // An invalid subtree. Always has an error so can never be extracted. -using InvalidParseStart = LeafNode; +using InvalidParseStart = LeafNode; struct InvalidParseSubtree { static constexpr auto Kind = NodeKind::InvalidParseSubtree.Define( {.category = NodeCategory::Decl, @@ -89,48 +92,56 @@ struct InvalidParseSubtree { InvalidParseStartId start; llvm::SmallVector> extra; + AnyToken token; }; // A placeholder node to be replaced; it will never exist in a valid parse tree. // Its token kind is not enforced even when valid. -using Placeholder = LeafNode; +using Placeholder = LeafNode; // File nodes // ---------- // The start of the file. -using FileStart = LeafNode; +using FileStart = + LeafNode>; // The end of the file. -using FileEnd = LeafNode; +using FileEnd = LeafNode>; // General-purpose nodes // --------------------- // An empty declaration, such as `;`. -using EmptyDecl = - LeafNode; +using EmptyDecl = LeafNode, + NodeCategory::Decl | NodeCategory::Statement>; // A name in a non-expression context, such as a declaration. using IdentifierName = - LeafNode; + LeafNode, + NodeCategory::MemberName>; // A name in an expression context. using IdentifierNameExpr = - LeafNode; + LeafNode, + NodeCategory::Expr>; // The `self` value and `Self` type identifier keywords. Typically of the form // `self: Self`. -using SelfValueName = LeafNode; +using SelfValueName = LeafNode>; using SelfValueNameExpr = - LeafNode; + LeafNode, NodeCategory::Expr>; using SelfTypeNameExpr = - LeafNode; + LeafNode, NodeCategory::Expr>; // The `base` value keyword, introduced by `base: B`. Typically referenced in // an expression, as in `x.base` or `{.base = ...}`, but can also be used as a // declared name, as in `{.base: partial B}`. -using BaseName = LeafNode; +using BaseName = LeafNode, + NodeCategory::MemberName>; // An unqualified name and optionally a following sequence of parameters. // For example, `A`, `A(n: i32)`, or `A[T:! type](n: T)`. @@ -146,6 +157,7 @@ struct NameQualifier { NodeKind::NameQualifier.Define({.bracketed_by = IdentifierName::Kind}); NameAndParams name_and_params; + Token token; }; // A complete name in a declaration: `A.C(T:! type).F(n: i32)`. @@ -159,20 +171,27 @@ struct DeclName { // -------------------------------- // The `package` keyword in an expression. -using PackageExpr = LeafNode; +using PackageExpr = + LeafNode, + NodeCategory::Expr>; // The name of a package or library for `package`, `import`, and `library`. -using PackageName = LeafNode; -using LibraryName = LeafNode; -using DefaultLibrary = LeafNode; +using PackageName = + LeafNode>; +using LibraryName = + LeafNode>; +using DefaultLibrary = + LeafNode>; -using PackageIntroducer = LeafNode; +using PackageIntroducer = + LeafNode>; // `library` in `package` or `import`. struct LibrarySpecifier { static constexpr auto Kind = NodeKind::LibrarySpecifier.Define({.child_count = 1}); + Token token; NodeIdOneOf name; }; @@ -187,10 +206,12 @@ struct PackageDecl { llvm::SmallVector modifiers; std::optional name; std::optional library; + Token token; }; // `import TheirPackage library "TheirLibrary";` -using ImportIntroducer = LeafNode; +using ImportIntroducer = + LeafNode>; struct ImportDecl { static constexpr auto Kind = NodeKind::ImportDecl.Define( {.category = NodeCategory::Decl, .bracketed_by = ImportIntroducer::Kind}); @@ -199,10 +220,12 @@ struct ImportDecl { llvm::SmallVector modifiers; std::optional name; std::optional library; + Token token; }; // `library` as declaration. -using LibraryIntroducer = LeafNode; +using LibraryIntroducer = + LeafNode>; struct LibraryDecl { static constexpr auto Kind = NodeKind::LibraryDecl.Define({.category = NodeCategory::Decl, @@ -211,10 +234,12 @@ struct LibraryDecl { LibraryIntroducerId introducer; llvm::SmallVector modifiers; NodeIdOneOf library_name; + Token token; }; // `export` as a declaration. -using ExportIntroducer = LeafNode; +using ExportIntroducer = + LeafNode>; struct ExportDecl { static constexpr auto Kind = NodeKind::ExportDecl.Define( {.category = NodeCategory::Decl, .bracketed_by = ExportIntroducer::Kind}); @@ -222,12 +247,14 @@ struct ExportDecl { ExportIntroducerId introducer; llvm::SmallVector modifiers; DeclName name; + Token token; }; // Namespace nodes // --------------- -using NamespaceStart = LeafNode; +using NamespaceStart = + LeafNode>; // A namespace: `namespace N;`. struct Namespace { @@ -237,6 +264,7 @@ struct Namespace { NamespaceStartId introducer; llvm::SmallVector modifiers; DeclName name; + Token token; }; // Pattern nodes @@ -248,6 +276,7 @@ struct BindingPattern { {.category = NodeCategory::Pattern, .child_count = 2}); NodeIdOneOf name; + Token token; AnyExprId type; }; @@ -257,6 +286,7 @@ struct CompileTimeBindingPattern { {.category = NodeCategory::Pattern, .child_count = 2}); NodeIdOneOf name; + Token token; AnyExprId type; }; @@ -265,6 +295,7 @@ struct Addr { static constexpr auto Kind = NodeKind::Addr.Define( {.category = NodeCategory::Pattern, .child_count = 1}); + Token token; AnyPatternId inner; }; @@ -273,13 +304,16 @@ struct Template { static constexpr auto Kind = NodeKind::Template.Define( {.category = NodeCategory::Pattern, .child_count = 1}); + Token token; // This is a CompileTimeBindingPatternId in any valid program. // TODO: Should the parser enforce that? AnyPatternId inner; }; -using TuplePatternStart = LeafNode; -using PatternListComma = LeafNode; +using TuplePatternStart = + LeafNode>; +using PatternListComma = + LeafNode>; // A parameter list or tuple pattern: `(a: i32, b: i32)`. struct TuplePattern { @@ -289,9 +323,12 @@ struct TuplePattern { TuplePatternStartId left_paren; CommaSeparatedList params; + Token token; }; -using ImplicitParamListStart = LeafNode; +using ImplicitParamListStart = + LeafNode>; // An implicit parameter list: `[T:! type, self: Self]`. struct ImplicitParamList { @@ -300,22 +337,26 @@ struct ImplicitParamList { ImplicitParamListStartId left_square; CommaSeparatedList params; + Token token; }; // Function nodes // -------------- -using FunctionIntroducer = LeafNode; +using FunctionIntroducer = + LeafNode>; // A return type: `-> i32`. struct ReturnType { static constexpr auto Kind = NodeKind::ReturnType.Define({.child_count = 1}); + Token token; AnyExprId type; }; // A function signature: `fn F() -> i32`. -template +template struct FunctionSignature { static constexpr auto Kind = KindT.Define( {.category = Category, .bracketed_by = FunctionIntroducer::Kind}); @@ -324,12 +365,16 @@ struct FunctionSignature { llvm::SmallVector modifiers; DeclName name; std::optional return_type; + TokenKind token; }; using FunctionDecl = - FunctionSignature; + FunctionSignature, + NodeCategory::Decl>; using FunctionDefinitionStart = - FunctionSignature; + FunctionSignature, + NodeCategory::None>; // A function definition: `fn F() -> i32 { ... }`. struct FunctionDefinition { @@ -339,12 +384,14 @@ struct FunctionDefinition { FunctionDefinitionStartId signature; llvm::SmallVector body; + Token token; }; using BuiltinFunctionDefinitionStart = FunctionSignature; -using BuiltinName = LeafNode; + Token, NodeCategory::None>; +using BuiltinName = + LeafNode>; // A builtin function definition: `fn F() -> i32 = "builtin name";` struct BuiltinFunctionDefinition { @@ -354,13 +401,16 @@ struct BuiltinFunctionDefinition { BuiltinFunctionDefinitionStartId signature; BuiltinNameId builtin_name; + Token token; }; // `alias` nodes // ------------- -using AliasIntroducer = LeafNode; -using AliasInitializer = LeafNode; +using AliasIntroducer = + LeafNode>; +using AliasInitializer = + LeafNode>; // An `alias` declaration: `alias a = b;`. struct Alias { @@ -373,13 +423,16 @@ struct Alias { DeclName name; AliasInitializerId equals; AnyExprId initializer; + Token token; }; // `let` nodes // ----------- -using LetIntroducer = LeafNode; -using LetInitializer = LeafNode; +using LetIntroducer = + LeafNode>; +using LetInitializer = + LeafNode>; // A `let` declaration: `let a: i32 = 5;`. struct LetDecl { @@ -396,14 +449,18 @@ struct LetDecl { AnyExprId initializer; }; std::optional initializer; + Token token; }; // `var` nodes // ----------- -using VariableIntroducer = LeafNode; -using ReturnedModifier = LeafNode; -using VariableInitializer = LeafNode; +using VariableIntroducer = + LeafNode>; +using ReturnedModifier = + LeafNode>; +using VariableInitializer = + LeafNode>; // A `var` declaration: `var a: i32;` or `var a: i32 = 5;`. struct VariableDecl { @@ -421,12 +478,14 @@ struct VariableDecl { AnyExprId value; }; std::optional initializer; + Token token; }; // Statement nodes // --------------- -using CodeBlockStart = LeafNode; +using CodeBlockStart = + LeafNode>; // A code block: `{ statement; statement; ... }`. struct CodeBlock { @@ -435,6 +494,7 @@ struct CodeBlock { CodeBlockStartId left_brace; llvm::SmallVector statements; + Token token; }; // An expression statement: `F(x);`. @@ -443,9 +503,11 @@ struct ExprStatement { {.category = NodeCategory::Statement, .child_count = 1}); AnyExprId expr; + Token token; }; -using BreakStatementStart = LeafNode; +using BreakStatementStart = + LeafNode>; // A break statement: `break;`. struct BreakStatement { @@ -455,9 +517,11 @@ struct BreakStatement { .child_count = 1}); BreakStatementStartId introducer; + Token token; }; -using ContinueStatementStart = LeafNode; +using ContinueStatementStart = + LeafNode>; // A continue statement: `continue;`. struct ContinueStatement { @@ -467,10 +531,13 @@ struct ContinueStatement { .child_count = 1}); ContinueStatementStartId introducer; + Token token; }; -using ReturnStatementStart = LeafNode; -using ReturnVarModifier = LeafNode; +using ReturnStatementStart = + LeafNode>; +using ReturnVarModifier = + LeafNode>; // A return statement: `return;` or `return expr;` or `return var;`. struct ReturnStatement { @@ -481,9 +548,11 @@ struct ReturnStatement { ReturnStatementStartId introducer; std::optional expr; std::optional var; + Token token; }; -using ForHeaderStart = LeafNode; +using ForHeaderStart = + LeafNode>; // The `var ... in` portion of a `for` statement. struct ForIn { @@ -491,6 +560,7 @@ struct ForIn { {.bracketed_by = VariableIntroducer::Kind, .child_count = 2}); VariableIntroducerId introducer; + Token token; AnyPatternId pattern; }; @@ -502,6 +572,7 @@ struct ForHeader { ForHeaderStartId introducer; ForInId var; AnyExprId range; + Token token; }; // A complete `for (...) { ... }` statement. @@ -511,11 +582,13 @@ struct ForStatement { .bracketed_by = ForHeader::Kind, .child_count = 2}); + Token token; ForHeaderId header; CodeBlockId body; }; -using IfConditionStart = LeafNode; +using IfConditionStart = + LeafNode>; // The condition portion of an `if` statement: `(expr)`. struct IfCondition { @@ -524,15 +597,18 @@ struct IfCondition { IfConditionStartId left_paren; AnyExprId condition; + Token token; }; -using IfStatementElse = LeafNode; +using IfStatementElse = + LeafNode>; // An `if` statement: `if (expr) { ... } else { ... }`. struct IfStatement { static constexpr auto Kind = NodeKind::IfStatement.Define( {.category = NodeCategory::Statement, .bracketed_by = IfCondition::Kind}); + Token token; IfConditionId head; CodeBlockId then; @@ -543,7 +619,8 @@ struct IfStatement { std::optional else_clause; }; -using WhileConditionStart = LeafNode; +using WhileConditionStart = + LeafNode>; // The condition portion of a `while` statement: `(expr)`. struct WhileCondition { @@ -552,6 +629,7 @@ struct WhileCondition { WhileConditionStartId left_paren; AnyExprId condition; + Token token; }; // A `while` statement: `while (expr) { ... }`. @@ -561,11 +639,13 @@ struct WhileStatement { .bracketed_by = WhileCondition::Kind, .child_count = 2}); + Token token; WhileConditionId head; CodeBlockId body; }; -using MatchConditionStart = LeafNode; +using MatchConditionStart = + LeafNode>; struct MatchCondition { static constexpr auto Kind = NodeKind::MatchCondition.Define( @@ -573,20 +653,26 @@ struct MatchCondition { MatchConditionStartId left_paren; AnyExprId condition; + Token token; }; -using MatchIntroducer = LeafNode; +using MatchIntroducer = + LeafNode>; struct MatchStatementStart { static constexpr auto Kind = NodeKind::MatchStatementStart.Define( {.bracketed_by = MatchIntroducer::Kind, .child_count = 2}); MatchIntroducerId introducer; - MatchConditionId left_brace; + MatchConditionId condition; + Token token; }; -using MatchCaseIntroducer = LeafNode; -using MatchCaseGuardIntroducer = LeafNode; -using MatchCaseGuardStart = LeafNode; +using MatchCaseIntroducer = + LeafNode>; +using MatchCaseGuardIntroducer = + LeafNode>; +using MatchCaseGuardStart = + LeafNode>; struct MatchCaseGuard { static constexpr auto Kind = NodeKind::MatchCaseGuard.Define( @@ -595,9 +681,11 @@ struct MatchCaseGuard { MatchCaseGuardIntroducerId introducer; MatchCaseGuardStartId left_paren; AnyExprId condition; + Token token; }; -using MatchCaseEqualGreater = LeafNode; +using MatchCaseEqualGreater = LeafNode>; struct MatchCaseStart { static constexpr auto Kind = NodeKind::MatchCaseStart.Define( @@ -607,6 +695,7 @@ struct MatchCaseStart { AnyPatternId pattern; std::optional guard; MatchCaseEqualGreaterId equal_greater_token; + Token token; }; struct MatchCase { @@ -615,10 +704,13 @@ struct MatchCase { MatchCaseStartId head; llvm::SmallVector statements; + Token token; }; -using MatchDefaultIntroducer = LeafNode; -using MatchDefaultEqualGreater = LeafNode; +using MatchDefaultIntroducer = + LeafNode>; +using MatchDefaultEqualGreater = LeafNode>; struct MatchDefaultStart { static constexpr auto Kind = NodeKind::MatchDefaultStart.Define( @@ -626,6 +718,7 @@ struct MatchDefaultStart { MatchDefaultIntroducerId introducer; MatchDefaultEqualGreaterId equal_greater_token; + Token token; }; struct MatchDefault { @@ -634,6 +727,7 @@ struct MatchDefault { MatchDefaultStartId introducer; llvm::SmallVector statements; + Token token; }; // A `match` statement: `match (expr) { case (...) => {...} default => {...}}`. @@ -646,12 +740,14 @@ struct MatchStatement { llvm::SmallVector cases; std::optional default_case; + Token token; }; // Expression nodes // ---------------- -using ArrayExprStart = LeafNode; +using ArrayExprStart = LeafNode>; // The start of an array type, `[i32;`. // @@ -662,6 +758,7 @@ struct ArrayExprSemi { ArrayExprStartId left_square; AnyExprId type; + Token token; }; // An array type, such as `[i32; 3]` or `[i32;]`. @@ -671,6 +768,7 @@ struct ArrayExpr { ArrayExprSemiId start; std::optional bound; + Token token; }; // The opening portion of an indexing expression: `a[`. @@ -681,6 +779,7 @@ struct IndexExprStart { NodeKind::IndexExprStart.Define({.child_count = 1}); AnyExprId sequence; + Token token; }; // An indexing expression, such as `a[1]`. @@ -692,9 +791,11 @@ struct IndexExpr { IndexExprStartId start; AnyExprId index; + Token token; }; -using ParenExprStart = LeafNode; +using ParenExprStart = + LeafNode>; // A parenthesized expression: `(a)`. struct ParenExpr { @@ -705,10 +806,13 @@ struct ParenExpr { ParenExprStartId start; AnyExprId expr; + Token token; }; -using TupleLiteralStart = LeafNode; -using TupleLiteralComma = LeafNode; +using TupleLiteralStart = + LeafNode>; +using TupleLiteralComma = + LeafNode>; // A tuple literal: `()`, `(a, b, c)`, or `(a,)`. struct TupleLiteral { @@ -718,6 +822,7 @@ struct TupleLiteral { TupleLiteralStartId start; CommaSeparatedList elements; + Token token; }; // The opening portion of a call expression: `F(`. @@ -728,9 +833,11 @@ struct CallExprStart { NodeKind::CallExprStart.Define({.child_count = 1}); AnyExprId callee; + Token token; }; -using CallExprComma = LeafNode; +using CallExprComma = + LeafNode>; // A call expression: `F(a, b, c)`. struct CallExpr { @@ -739,6 +846,7 @@ struct CallExpr { CallExprStartId start; CommaSeparatedList arguments; + Token token; }; // A member access expression: `a.b` or `a.(b)`. @@ -747,6 +855,7 @@ struct MemberAccessExpr { {.category = NodeCategory::Expr, .child_count = 2}); AnyExprId lhs; + Token token; AnyMemberNameOrMemberExprId rhs; }; @@ -756,52 +865,61 @@ struct PointerMemberAccessExpr { {.category = NodeCategory::Expr, .child_count = 2}); AnyExprId lhs; + Token token; AnyMemberNameOrMemberExprId rhs; }; // A prefix operator expression. -template +template struct PrefixOperator { static constexpr auto Kind = KindT.Define({.category = NodeCategory::Expr, .child_count = 1}); + TokenKind token; AnyExprId operand; }; // An infix operator expression. -template +template struct InfixOperator { static constexpr auto Kind = KindT.Define({.category = NodeCategory::Expr, .child_count = 2}); AnyExprId lhs; + TokenKind token; AnyExprId rhs; }; // A postfix operator expression. -template +template struct PostfixOperator { static constexpr auto Kind = KindT.Define({.category = NodeCategory::Expr, .child_count = 1}); AnyExprId operand; + TokenKind token; }; // Literals, operators, and modifiers #define CARBON_PARSE_NODE_KIND(...) -#define CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(Name, ...) \ - using Name = LeafNode; -#define CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER(Name, ...) \ - using Name##Modifier = \ - LeafNode; -#define CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Name, ...) \ - using PrefixOperator##Name = PrefixOperator; -#define CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Name, ...) \ - using InfixOperator##Name = InfixOperator; -#define CARBON_PARSE_NODE_KIND_POSTFIX_OPERATOR(Name, ...) \ - using PostfixOperator##Name = \ - PostfixOperator; +#define CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(Name, LexTokenKind) \ + using Name = LeafNode, \ + NodeCategory::Expr>; +#define CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER(Name) \ + using Name##Modifier = \ + LeafNode, \ + NodeCategory::Modifier>; +#define CARBON_PARSE_NODE_KIND_PREFIX_OPERATOR(Name) \ + using PrefixOperator##Name = PrefixOperator>; +#define CARBON_PARSE_NODE_KIND_INFIX_OPERATOR(Name) \ + using InfixOperator##Name = InfixOperator>; +#define CARBON_PARSE_NODE_KIND_POSTFIX_OPERATOR(Name) \ + using PostfixOperator##Name = \ + PostfixOperator>; #include "toolchain/parse/node_kind.def" // The first operand of a short-circuiting infix operator: `a and` or `a or`. @@ -814,6 +932,7 @@ struct ShortCircuitOperandAnd { NodeKind::ShortCircuitOperandAnd.Define({.child_count = 1}); AnyExprId operand; + VirtualToken token; }; struct ShortCircuitOperandOr { @@ -821,6 +940,7 @@ struct ShortCircuitOperandOr { NodeKind::ShortCircuitOperandOr.Define({.child_count = 1}); AnyExprId operand; + VirtualToken token; }; struct ShortCircuitOperatorAnd { @@ -830,6 +950,7 @@ struct ShortCircuitOperatorAnd { .child_count = 2}); ShortCircuitOperandAndId lhs; + Token token; AnyExprId rhs; }; @@ -840,6 +961,7 @@ struct ShortCircuitOperatorOr { .child_count = 2}); ShortCircuitOperandOrId lhs; + Token token; AnyExprId rhs; }; @@ -847,6 +969,7 @@ struct ShortCircuitOperatorOr { struct IfExprIf { static constexpr auto Kind = NodeKind::IfExprIf.Define({.child_count = 1}); + Token token; AnyExprId condition; }; @@ -854,6 +977,7 @@ struct IfExprIf { struct IfExprThen { static constexpr auto Kind = NodeKind::IfExprThen.Define({.child_count = 1}); + Token token; AnyExprId result; }; @@ -866,13 +990,15 @@ struct IfExprElse { IfExprIfId start; IfExprThenId then; + Token token; AnyExprId else_result; }; // Choice nodes // ------------ -using ChoiceIntroducer = LeafNode; +using ChoiceIntroducer = + LeafNode>; struct ChoiceSignature { static constexpr auto Kind = NodeKind::ChoiceDefinitionStart.Define( @@ -881,12 +1007,14 @@ struct ChoiceSignature { ChoiceIntroducerId introducer; llvm::SmallVector modifiers; DeclName name; + Token token; }; using ChoiceDefinitionStart = ChoiceSignature; using ChoiceAlternativeListComma = - LeafNode; + LeafNode>; struct ChoiceDefinition { static constexpr auto Kind = NodeKind::ChoiceDefinition.Define( @@ -899,22 +1027,27 @@ struct ChoiceDefinition { std::optional parameters; }; CommaSeparatedList alternatives; + Token token; }; // Struct type and value literals // ---------------------------------------- // `{` -using StructLiteralStart = LeafNode; -using StructTypeLiteralStart = LeafNode; +using StructLiteralStart = LeafNode>; +using StructTypeLiteralStart = LeafNode>; // `,` -using StructComma = LeafNode; +using StructComma = + LeafNode>; // `.a` struct StructFieldDesignator { static constexpr auto Kind = NodeKind::StructFieldDesignator.Define({.child_count = 1}); + Token token; NodeIdOneOf name; }; @@ -924,6 +1057,7 @@ struct StructField { {.bracketed_by = StructFieldDesignator::Kind, .child_count = 2}); StructFieldDesignatorId designator; + Token token; AnyExprId expr; }; @@ -933,6 +1067,7 @@ struct StructTypeField { {.bracketed_by = StructFieldDesignator::Kind, .child_count = 2}); StructFieldDesignatorId designator; + Token token; AnyExprId type_expr; }; @@ -944,6 +1079,7 @@ struct StructLiteral { StructLiteralStartId start; CommaSeparatedList fields; + Token token; }; // Struct type literals, such as `{.a: i32}`. @@ -954,16 +1090,19 @@ struct StructTypeLiteral { StructTypeLiteralStartId start; CommaSeparatedList fields; + Token token; }; // `class` declarations and definitions // ------------------------------------ // `class` -using ClassIntroducer = LeafNode; +using ClassIntroducer = + LeafNode>; // A class signature `class C` -template +template struct ClassSignature { static constexpr auto Kind = KindT.Define( {.category = Category, .bracketed_by = ClassIntroducer::Kind}); @@ -971,13 +1110,17 @@ struct ClassSignature { ClassIntroducerId introducer; llvm::SmallVector modifiers; DeclName name; + TokenKind token; }; // `class C;` -using ClassDecl = ClassSignature; +using ClassDecl = + ClassSignature, + NodeCategory::Decl>; // `class C {` using ClassDefinitionStart = - ClassSignature; + ClassSignature, NodeCategory::None>; // `class C { ... }` struct ClassDefinition { @@ -987,13 +1130,15 @@ struct ClassDefinition { ClassDefinitionStartId signature; llvm::SmallVector members; + Token token; }; // Adapter declaration // ------------------- // `adapt` -using AdaptIntroducer = LeafNode; +using AdaptIntroducer = + LeafNode>; // `adapt SomeType;` struct AdaptDecl { static constexpr auto Kind = NodeKind::AdaptDecl.Define( @@ -1002,14 +1147,16 @@ struct AdaptDecl { AdaptIntroducerId introducer; llvm::SmallVector modifiers; AnyExprId adapted_type; + Token token; }; // Base class declaration // ---------------------- // `base` -using BaseIntroducer = LeafNode; -using BaseColon = LeafNode; +using BaseIntroducer = + LeafNode>; +using BaseColon = LeafNode>; // `extend base: BaseClass;` struct BaseDecl { static constexpr auto Kind = NodeKind::BaseDecl.Define( @@ -1019,16 +1166,19 @@ struct BaseDecl { llvm::SmallVector modifiers; BaseColonId colon; AnyExprId base_class; + Token token; }; // Interface declarations and definitions // -------------------------------------- // `interface` -using InterfaceIntroducer = LeafNode; +using InterfaceIntroducer = + LeafNode>; // `interface I` -template +template struct InterfaceSignature { static constexpr auto Kind = KindT.Define( {.category = Category, .bracketed_by = InterfaceIntroducer::Kind}); @@ -1036,14 +1186,18 @@ struct InterfaceSignature { InterfaceIntroducerId introducer; llvm::SmallVector modifiers; DeclName name; + TokenKind token; }; // `interface I;` using InterfaceDecl = - InterfaceSignature; + InterfaceSignature, + NodeCategory::Decl>; // `interface I {` using InterfaceDefinitionStart = - InterfaceSignature; + InterfaceSignature, + NodeCategory::None>; // `interface I { ... }` struct InterfaceDefinition { @@ -1053,24 +1207,28 @@ struct InterfaceDefinition { InterfaceDefinitionStartId signature; llvm::SmallVector members; + Token token; }; // `impl`...`as` declarations and definitions // ------------------------------------------ // `impl` -using ImplIntroducer = LeafNode; +using ImplIntroducer = + LeafNode>; // `forall [...]` struct ImplForall { static constexpr auto Kind = NodeKind::ImplForall.Define({.child_count = 1}); + Token token; ImplicitParamListId params; }; // `as` with no type before it using DefaultSelfImplAs = - LeafNode; + LeafNode, + NodeCategory::ImplAs>; // ` as` struct TypeImplAs { @@ -1078,10 +1236,12 @@ struct TypeImplAs { {.category = NodeCategory::ImplAs, .child_count = 1}); AnyExprId type_expr; + Token token; }; // `impl T as I` -template +template struct ImplSignature { static constexpr auto Kind = KindT.Define( {.category = Category, .bracketed_by = ImplIntroducer::Kind}); @@ -1091,13 +1251,16 @@ struct ImplSignature { std::optional forall; AnyImplAsId as; AnyExprId interface; + TokenKind token; }; // `impl T as I;` -using ImplDecl = ImplSignature; +using ImplDecl = ImplSignature, + NodeCategory::Decl>; // `impl T as I {` using ImplDefinitionStart = - ImplSignature; + ImplSignature, NodeCategory::None>; // `impl T as I { ... }` struct ImplDefinition { @@ -1107,16 +1270,19 @@ struct ImplDefinition { ImplDefinitionStartId signature; llvm::SmallVector members; + Token token; }; // Named constraint declarations and definitions // --------------------------------------------- // `constraint` -using NamedConstraintIntroducer = LeafNode; +using NamedConstraintIntroducer = LeafNode>; // `constraint NC` -template +template struct NamedConstraintSignature { static constexpr auto Kind = KindT.Define( {.category = Category, .bracketed_by = NamedConstraintIntroducer::Kind}); @@ -1124,14 +1290,17 @@ struct NamedConstraintSignature { NamedConstraintIntroducerId introducer; llvm::SmallVector modifiers; DeclName name; + TokenKind token; }; // `constraint NC;` using NamedConstraintDecl = - NamedConstraintSignature; + NamedConstraintSignature, NodeCategory::Decl>; // `constraint NC {` using NamedConstraintDefinitionStart = NamedConstraintSignature, NodeCategory::None>; // `constraint NC { ... }` @@ -1142,6 +1311,7 @@ struct NamedConstraintDefinition { NamedConstraintDefinitionStartId signature; llvm::SmallVector members; + Token token; }; // --------------------------------------------------------------------------- diff --git a/toolchain/parse/typed_nodes_test.cpp b/toolchain/parse/typed_nodes_test.cpp index 522dc58bb9d8..67e53a51a0de 100644 --- a/toolchain/parse/typed_nodes_test.cpp +++ b/toolchain/parse/typed_nodes_test.cpp @@ -14,6 +14,25 @@ #include "toolchain/parse/parse.h" namespace Carbon::Parse { + +// A test peer (see https://abseil.io/tips/135) to allow these tests to access +// certain implementation details of Tree. +class TypedNodesTestPeer { + public: + template + static auto VerifyExtractAs(const Tree* tree, NodeId node_id, + ErrorBuilder* trace) -> std::optional { + return tree->VerifyExtractAs(node_id, trace); + } + + // Sets the kind of a node. This is intended to allow putting the tree into a + // state where verification can fail, in order to make the failure path of + // `Verify` testable. + static auto SetNodeKind(Tree* tree, NodeId node_id, NodeKind kind) -> void { + tree->SetNodeKindForTesting(node_id, kind); + } +}; + namespace { // Check that each node kind defines a Kind member using the correct @@ -44,6 +63,12 @@ class TypedNodeTest : public ::testing::Test { return tree_storage_.front(); } + auto GetTokenizedBufferAndTree(llvm::StringRef t) + -> std::pair { + auto* tree = &GetTree(t); + return {&token_storage_.front(), tree}; + } + SharedValueStores value_stores_; llvm::vfs::InMemoryFileSystem fs_; std::forward_list source_storage_; @@ -144,7 +169,8 @@ TEST_F(TypedNodeTest, VerifyExtractTraceLibrary) { ASSERT_EQ(file.decls.size(), 1); ErrorBuilder trace; - auto library = tree->VerifyExtractAs(file.decls[0], &trace); + auto library = TypedNodesTestPeer::VerifyExtractAs( + tree, file.decls[0], &trace); EXPECT_TRUE(library.has_value()); Error err = trace; // Use Regex matching to avoid hard-coding the result of `typeinfo(T).name()`. @@ -168,7 +194,8 @@ TEST_F(TypedNodeTest, VerifyExtractTraceVarNoInit) { ASSERT_EQ(file.decls.size(), 1); ErrorBuilder trace; - auto var = tree->VerifyExtractAs(file.decls[0], &trace); + auto var = TypedNodesTestPeer::VerifyExtractAs( + tree, file.decls[0], &trace); ASSERT_TRUE(var.has_value()); Error err = trace; // Use Regex matching to avoid hard-coding the result of `typeinfo(T).name()`. @@ -199,7 +226,8 @@ TEST_F(TypedNodeTest, VerifyExtractTraceExpression) { ASSERT_EQ(file.decls.size(), 1); ErrorBuilder trace1; - auto var = tree->VerifyExtractAs(file.decls[0], &trace1); + auto var = TypedNodesTestPeer::VerifyExtractAs( + tree, file.decls[0], &trace1); ASSERT_TRUE(var.has_value()); Error err1 = trace1; // Use Regex matching to avoid hard-coding the result of `typeinfo(T).name()`. @@ -224,8 +252,8 @@ Aggregate [^:]*: success ASSERT_TRUE(var->initializer.has_value()); ErrorBuilder trace2; - auto value = - tree->VerifyExtractAs(var->initializer->value, &trace2); + auto value = TypedNodesTestPeer::VerifyExtractAs( + tree, var->initializer->value, &trace2); ASSERT_TRUE(value.has_value()); Error err2 = trace2; // Use Regex matching to avoid hard-coding the result of `typeinfo(T).name()`. @@ -245,7 +273,8 @@ TEST_F(TypedNodeTest, VerifyExtractTraceClassDecl) { ASSERT_EQ(file.decls.size(), 1); ErrorBuilder trace; - auto class_decl = tree->VerifyExtractAs(file.decls[0], &trace); + auto class_decl = TypedNodesTestPeer::VerifyExtractAs( + tree, file.decls[0], &trace); EXPECT_TRUE(class_decl.has_value()); Error err = trace; // Use Regex matching to avoid hard-coding the result of `typeinfo(T).name()`. @@ -276,6 +305,88 @@ Aggregate [^:]*: success )Trace")); } +TEST_F(TypedNodeTest, Token) { + auto [tokens, tree] = GetTokenizedBufferAndTree(R"carbon( + var n: i32 = 0; + )carbon"); + auto file = tree->ExtractFile(); + + ASSERT_EQ(file.decls.size(), 1); + + auto n_var = tree->ExtractAs(file.decls[0]); + ASSERT_TRUE(n_var.has_value()); + EXPECT_EQ(tokens->GetKind(n_var->token.index), Lex::TokenKind::Semi); + + auto n_intro = tree->ExtractAs(n_var->introducer); + ASSERT_TRUE(n_intro.has_value()); + EXPECT_EQ(tokens->GetKind(n_intro->token.index), Lex::TokenKind::Var); + + auto n_patt = tree->ExtractAs(n_var->pattern); + ASSERT_TRUE(n_patt.has_value()); + EXPECT_EQ(tokens->GetKind(n_patt->token.index), Lex::TokenKind::Colon); +} + +TEST_F(TypedNodeTest, VerifyInvalid) { + auto* tree = &GetTree(R"carbon( + fn F() -> i32 { return 0; } + )carbon"); + + auto file = tree->ExtractFile(); + ASSERT_EQ(file.decls.size(), 1); + + auto f_fn = tree->ExtractAs(file.decls[0]); + ASSERT_TRUE(f_fn.has_value()); + auto f_sig = tree->ExtractAs(f_fn->signature); + ASSERT_TRUE(f_sig.has_value()); + auto f_intro = tree->ExtractAs(f_sig->introducer); + ASSERT_TRUE(f_intro.has_value()); + + // Change the kind of the introducer and check we get a good trace log. + TypedNodesTestPeer::SetNodeKind(tree, f_sig->introducer, + NodeKind::ClassIntroducer); + + // The introducer should not extract as a FunctionIntroducer any more because + // the kind is wrong. + { + ErrorBuilder trace; + EXPECT_FALSE(TypedNodesTestPeer::VerifyExtractAs( + tree, f_sig->introducer, &trace)); + + Error err = trace; + EXPECT_EQ(err.message(), + "VerifyExtractAs error: wrong kind ClassIntroducer, expected " + "FunctionIntroducer\n"); + } + + // The introducer should also not extract as a ClassIntroducer because the + // token kind is wrong. + { + ErrorBuilder trace; + EXPECT_FALSE(TypedNodesTestPeer::VerifyExtractAs( + tree, f_sig->introducer, &trace)); + + Error err = trace; + EXPECT_THAT(err.message(), + testing::HasSubstr( + "\nToken Class expected for ClassIntroducer, found Fn\n")); + } + + // The signature should not extract as a FunctionDefinitionStart because the + // kind for the introducer is wrong. + { + ErrorBuilder trace; + EXPECT_FALSE(TypedNodesTestPeer::VerifyExtractAs( + tree, f_fn->signature, &trace)); + + Error err = trace; + EXPECT_THAT(err.message(), testing::MatchesRegex( + R"Trace((?s).* +NodeIdForKind error: wrong kind IdentifierName, expected ImplicitParamList +.* +Error: ClassIntroducer node left unconsumed.)Trace")); + } +} + auto CategoryMatches(const NodeKind::Definition& def, NodeKind kind, const char* name) { EXPECT_EQ(def.category(), kind.category()) << name;