diff --git a/language_server/BUILD b/language_server/BUILD index d010d89cc0ff..9a2ffb1e8020 100644 --- a/language_server/BUILD +++ b/language_server/BUILD @@ -25,6 +25,7 @@ cc_binary( "//toolchain/diagnostics:null_diagnostics", "//toolchain/lex", "//toolchain/lex:tokenized_buffer", + "//toolchain/parse", "//toolchain/parse:node_kind", "//toolchain/parse:tree", "//toolchain/source:source_buffer", diff --git a/language_server/language_server.cpp b/language_server/language_server.cpp index 7058106cddbb..227ec507089a 100644 --- a/language_server/language_server.cpp +++ b/language_server/language_server.cpp @@ -9,7 +9,7 @@ #include "toolchain/diagnostics/null_diagnostics.h" #include "toolchain/lex/lex.h" #include "toolchain/parse/node_kind.h" -#include "toolchain/parse/tree.h" +#include "toolchain/parse/parse.h" #include "toolchain/source/source_buffer.h" namespace Carbon::LS { @@ -101,7 +101,7 @@ void LanguageServer::OnDocumentSymbol( auto buf = SourceBuffer::CreateFromFile(vfs, file, NullDiagnosticConsumer()); auto lexed = Lex::Lex(value_stores, *buf, NullDiagnosticConsumer()); - auto parsed = Parse::Tree::Parse(lexed, NullDiagnosticConsumer(), nullptr); + auto parsed = Parse::Parse(lexed, NullDiagnosticConsumer(), nullptr); std::vector result; for (const auto& node : parsed.postorder()) { clang::clangd::SymbolKind symbol_kind; diff --git a/toolchain/driver/BUILD b/toolchain/driver/BUILD index 98a5468d73a9..e603f954cfe9 100644 --- a/toolchain/driver/BUILD +++ b/toolchain/driver/BUILD @@ -28,7 +28,7 @@ cc_library( "//toolchain/diagnostics:sorting_diagnostic_consumer", "//toolchain/lex", "//toolchain/lower", - "//toolchain/parse:tree", + "//toolchain/parse", "//toolchain/sem_ir:file", "//toolchain/sem_ir:formatter", "//toolchain/source:source_buffer", diff --git a/toolchain/driver/driver.cpp b/toolchain/driver/driver.cpp index 607be8726226..786e3f6cc371 100644 --- a/toolchain/driver/driver.cpp +++ b/toolchain/driver/driver.cpp @@ -23,7 +23,7 @@ #include "toolchain/diagnostics/sorting_diagnostic_consumer.h" #include "toolchain/lex/lex.h" #include "toolchain/lower/lower.h" -#include "toolchain/parse/tree.h" +#include "toolchain/parse/parse.h" #include "toolchain/sem_ir/formatter.h" #include "toolchain/source/source_buffer.h" @@ -438,8 +438,8 @@ class Driver::CompilationUnit { auto RunParse() -> bool { CARBON_CHECK(tokens_); - LogCall("Parse::Tree::Parse", [&] { - parse_tree_ = Parse::Tree::Parse(*tokens_, *consumer_, vlog_stream_); + LogCall("Parse::Parse", [&] { + parse_tree_ = Parse::Parse(*tokens_, *consumer_, vlog_stream_); }); if (options_.dump_parse_tree) { consumer_->Flush(); diff --git a/toolchain/parse/BUILD b/toolchain/parse/BUILD index 55f146795919..09f0fd58a064 100644 --- a/toolchain/parse/BUILD +++ b/toolchain/parse/BUILD @@ -37,6 +37,7 @@ cc_test( srcs = ["typed_nodes_test.cpp"], deps = [ ":node_kind", + ":parse", ":tree", "//testing/base:gtest_main", "//toolchain/diagnostics:diagnostic_emitter", @@ -47,6 +48,35 @@ cc_test( ], ) +cc_library( + name = "parse", + srcs = [ + "context.cpp", + "context.h", + "parse.cpp", + ] + + # Glob handler files to avoid missing any. + glob([ + "handle_*.cpp", + ]), + hdrs = ["parse.h"], + deps = [ + ":node_kind", + ":precedence", + ":state", + ":tree", + "//common:check", + "//common:ostream", + "//common:vlog", + "//toolchain/base:pretty_stack_trace_function", + "//toolchain/base:value_store", + "//toolchain/diagnostics:diagnostic_emitter", + "//toolchain/lex:token_kind", + "//toolchain/lex:tokenized_buffer", + "@llvm-project//llvm:Support", + ], +) + cc_library( name = "state", srcs = ["state.cpp"], @@ -58,29 +88,18 @@ cc_library( cc_library( name = "tree", srcs = [ - "context.cpp", - "context.h", "extract.cpp", "tree.cpp", - ] + - # Glob handler files to avoid missing any. - glob([ - "handle_*.cpp", - ]), + ], hdrs = ["tree.h"], deps = [ ":node_kind", - ":precedence", - ":state", "//common:check", "//common:error", "//common:ostream", "//common:struct_reflection", - "//common:vlog", "//toolchain/base:pretty_stack_trace_function", - "//toolchain/base:value_store", "//toolchain/diagnostics:diagnostic_emitter", - "//toolchain/lex:token_kind", "//toolchain/lex:tokenized_buffer", "@llvm-project//llvm:Support", ], @@ -92,6 +111,7 @@ cc_test( srcs = ["tree_test.cpp"], deps = [ ":node_kind", + ":parse", ":tree", "//common:ostream", "//testing/base:gtest_main", @@ -113,7 +133,7 @@ cc_fuzz_test( srcs = ["parse_fuzzer.cpp"], corpus = glob(["fuzzer_corpus/*"]), deps = [ - ":tree", + ":parse", "//common:check", "//toolchain/base:value_store", "//toolchain/diagnostics:diagnostic_emitter", diff --git a/toolchain/parse/parse.cpp b/toolchain/parse/parse.cpp new file mode 100644 index 000000000000..081eadd76fe8 --- /dev/null +++ b/toolchain/parse/parse.cpp @@ -0,0 +1,61 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "common/check.h" +#include "toolchain/base/pretty_stack_trace_function.h" +#include "toolchain/parse/context.h" +#include "toolchain/parse/node_kind.h" +#include "toolchain/parse/typed_nodes.h" + +namespace Carbon::Parse { + +auto HandleInvalid(Context& context) -> void { + CARBON_FATAL() << "The Invalid state shouldn't be on the stack: " + << context.PopState(); +} + +auto Parse(Lex::TokenizedBuffer& tokens, DiagnosticConsumer& consumer, + llvm::raw_ostream* vlog_stream) -> Tree { + Lex::TokenLocationTranslator translator(&tokens); + Lex::TokenDiagnosticEmitter emitter(translator, consumer); + + // Delegate to the parser. + Tree tree(tokens); + Context context(tree, tokens, emitter, vlog_stream); + PrettyStackTraceFunction context_dumper( + [&](llvm::raw_ostream& output) { context.PrintForStackDump(output); }); + + context.AddLeafNode(NodeKind::FileStart, + context.ConsumeChecked(Lex::TokenKind::FileStart)); + + context.PushState(State::DeclScopeLoop); + + while (!context.state_stack().empty()) { + // clang warns on unhandled enum values; clang-tidy is incorrect here. + // NOLINTNEXTLINE(bugprone-switch-missing-default-case) + switch (context.state_stack().back().state) { +#define CARBON_PARSE_STATE(Name) \ + case State::Name: \ + Handle##Name(context); \ + break; +#include "toolchain/parse/state.def" + } + } + + context.AddLeafNode(NodeKind::FileEnd, *context.position()); + + if (auto verify = tree.Verify(); !verify.ok()) { + // TODO: This is temporarily printing to stderr directly during development. + // If we can, restrict this to a subtree with the error and add it to the + // stack trace (such as with PrettyStackTraceFunction). Otherwise, switch + // back to vlog_stream prior to broader distribution so that end users are + // hopefully comfortable copy-pasting stderr when there are bugs in tree + // construction. + tree.Print(llvm::errs()); + CARBON_FATAL() << "Invalid tree returned by Parse(): " << verify.error(); + } + return tree; +} + +} // namespace Carbon::Parse diff --git a/toolchain/parse/parse.h b/toolchain/parse/parse.h new file mode 100644 index 000000000000..f86fb11336c4 --- /dev/null +++ b/toolchain/parse/parse.h @@ -0,0 +1,23 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef CARBON_TOOLCHAIN_PARSE_PARSE_H_ +#define CARBON_TOOLCHAIN_PARSE_PARSE_H_ + +#include "common/ostream.h" +#include "toolchain/diagnostics/diagnostic_emitter.h" +#include "toolchain/lex/tokenized_buffer.h" +#include "toolchain/parse/tree.h" + +namespace Carbon::Parse { + +// Parses the token buffer into a `Tree`. +// +// This is the factory function which is used to build parse trees. +auto Parse(Lex::TokenizedBuffer& tokens, DiagnosticConsumer& consumer, + llvm::raw_ostream* vlog_stream) -> Tree; + +} // namespace Carbon::Parse + +#endif // CARBON_TOOLCHAIN_PARSE_PARSE_H_ diff --git a/toolchain/parse/parse_fuzzer.cpp b/toolchain/parse/parse_fuzzer.cpp index c87f3469ede2..7c36ab5deba7 100644 --- a/toolchain/parse/parse_fuzzer.cpp +++ b/toolchain/parse/parse_fuzzer.cpp @@ -9,7 +9,7 @@ #include "toolchain/base/value_store.h" #include "toolchain/diagnostics/null_diagnostics.h" #include "toolchain/lex/lex.h" -#include "toolchain/parse/tree.h" +#include "toolchain/parse/parse.h" namespace Carbon::Testing { @@ -40,7 +40,7 @@ extern "C" int LLVMFuzzerTestOneInput(const unsigned char* data, // Now parse it into a tree. Note that parsing will (when asserts are enabled) // walk the entire tree to verify it so we don't have to do that here. - Parse::Tree::Parse(tokens, NullDiagnosticConsumer(), /*vlog_stream=*/nullptr); + Parse::Parse(tokens, NullDiagnosticConsumer(), /*vlog_stream=*/nullptr); return 0; } diff --git a/toolchain/parse/tree.cpp b/toolchain/parse/tree.cpp index 2c3bb77f0eb6..83b5b40a2148 100644 --- a/toolchain/parse/tree.cpp +++ b/toolchain/parse/tree.cpp @@ -10,60 +10,11 @@ #include "llvm/ADT/SmallVector.h" #include "toolchain/base/pretty_stack_trace_function.h" #include "toolchain/lex/tokenized_buffer.h" -#include "toolchain/parse/context.h" #include "toolchain/parse/node_kind.h" #include "toolchain/parse/typed_nodes.h" namespace Carbon::Parse { -auto HandleInvalid(Context& context) -> void { - CARBON_FATAL() << "The Invalid state shouldn't be on the stack: " - << context.PopState(); -} - -auto Tree::Parse(Lex::TokenizedBuffer& tokens, DiagnosticConsumer& consumer, - llvm::raw_ostream* vlog_stream) -> Tree { - Lex::TokenLocationTranslator translator(&tokens); - Lex::TokenDiagnosticEmitter emitter(translator, consumer); - - // Delegate to the parser. - Tree tree(tokens); - Context context(tree, tokens, emitter, vlog_stream); - PrettyStackTraceFunction context_dumper( - [&](llvm::raw_ostream& output) { context.PrintForStackDump(output); }); - - context.AddLeafNode(NodeKind::FileStart, - context.ConsumeChecked(Lex::TokenKind::FileStart)); - - context.PushState(State::DeclScopeLoop); - - while (!context.state_stack().empty()) { - // clang warns on unhandled enum values; clang-tidy is incorrect here. - // NOLINTNEXTLINE(bugprone-switch-missing-default-case) - switch (context.state_stack().back().state) { -#define CARBON_PARSE_STATE(Name) \ - case State::Name: \ - Handle##Name(context); \ - break; -#include "toolchain/parse/state.def" - } - } - - context.AddLeafNode(NodeKind::FileEnd, *context.position()); - - if (auto verify = tree.Verify(); !verify.ok()) { - // TODO: This is temporarily printing to stderr directly during development. - // If we can, restrict this to a subtree with the error and add it to the - // stack trace (such as with PrettyStackTraceFunction). Otherwise, switch - // back to vlog_stream prior to broader distribution so that end users are - // hopefully comfortable copy-pasting stderr when there are bugs in tree - // construction. - tree.Print(llvm::errs()); - CARBON_FATAL() << "Invalid tree returned by Parse(): " << verify.error(); - } - return tree; -} - auto Tree::postorder() const -> llvm::iterator_range { return {PostorderIterator(NodeId(0)), PostorderIterator(NodeId(node_impls_.size()))}; diff --git a/toolchain/parse/tree.h b/toolchain/parse/tree.h index 811da401e8e0..a611de1209de 100644 --- a/toolchain/parse/tree.h +++ b/toolchain/parse/tree.h @@ -69,11 +69,12 @@ class Tree : public Printable { ApiOrImpl api_or_impl; }; - // Parses the token buffer into a `Tree`. - // - // This is the factory function which is used to build parse trees. - static auto Parse(Lex::TokenizedBuffer& tokens, DiagnosticConsumer& consumer, - llvm::raw_ostream* vlog_stream) -> Tree; + // Wires up the reference to the tokenized buffer. The `Parse` function should + // be used to actually parse the tokens into a tree. + explicit Tree(Lex::TokenizedBuffer& tokens_arg) : tokens_(&tokens_arg) { + // If the tree is valid, there will be one node per token, so reserve once. + node_impls_.reserve(tokens_->expected_parse_tree_size()); + } // Tests whether there are any errors in the parse tree. auto has_errors() const -> bool { return has_errors_; } @@ -279,13 +280,6 @@ class Tree : public Printable { static_assert(sizeof(NodeImpl) == 12, "Unexpected size of node implementation!"); - // Wires up the reference to the tokenized buffer. The `Parse` function should - // be used to actually parse the tokens into a tree. - explicit Tree(Lex::TokenizedBuffer& tokens_arg) : tokens_(&tokens_arg) { - // If the tree is valid, there will be one node per token, so reserve once. - node_impls_.reserve(tokens_->expected_parse_tree_size()); - } - // Prints a single node for Print(). Returns true when preorder and there are // children. auto PrintNode(llvm::raw_ostream& output, NodeId n, int depth, diff --git a/toolchain/parse/tree_test.cpp b/toolchain/parse/tree_test.cpp index a07866c02a13..dcbe0ad13bf6 100644 --- a/toolchain/parse/tree_test.cpp +++ b/toolchain/parse/tree_test.cpp @@ -15,6 +15,7 @@ #include "toolchain/diagnostics/mocks.h" #include "toolchain/lex/lex.h" #include "toolchain/lex/tokenized_buffer.h" +#include "toolchain/parse/parse.h" #include "toolchain/testing/yaml_test_helpers.h" namespace Carbon::Parse { @@ -51,13 +52,13 @@ class TreeTest : public ::testing::Test { TEST_F(TreeTest, IsValid) { Lex::TokenizedBuffer& tokens = GetTokenizedBuffer(""); - Tree tree = Tree::Parse(tokens, consumer_, /*vlog_stream=*/nullptr); + Tree tree = Parse(tokens, consumer_, /*vlog_stream=*/nullptr); EXPECT_TRUE((*tree.postorder().begin()).is_valid()); } TEST_F(TreeTest, PrintPostorderAsYAML) { Lex::TokenizedBuffer& tokens = GetTokenizedBuffer("fn F();"); - Tree tree = Tree::Parse(tokens, consumer_, /*vlog_stream=*/nullptr); + Tree tree = Parse(tokens, consumer_, /*vlog_stream=*/nullptr); EXPECT_FALSE(tree.has_errors()); TestRawOstream print_stream; tree.Print(print_stream); @@ -85,7 +86,7 @@ TEST_F(TreeTest, PrintPostorderAsYAML) { TEST_F(TreeTest, PrintPreorderAsYAML) { Lex::TokenizedBuffer& tokens = GetTokenizedBuffer("fn F();"); - Tree tree = Tree::Parse(tokens, consumer_, /*vlog_stream=*/nullptr); + Tree tree = Parse(tokens, consumer_, /*vlog_stream=*/nullptr); EXPECT_FALSE(tree.has_errors()); TestRawOstream print_stream; tree.Print(print_stream, /*preorder=*/true); @@ -131,7 +132,7 @@ TEST_F(TreeTest, HighRecursion) { Lex::TokenizedBuffer& tokens = GetTokenizedBuffer(code); ASSERT_FALSE(tokens.has_errors()); Testing::MockDiagnosticConsumer consumer; - Tree tree = Tree::Parse(tokens, consumer, /*vlog_stream=*/nullptr); + Tree tree = Parse(tokens, consumer, /*vlog_stream=*/nullptr); EXPECT_FALSE(tree.has_errors()); } diff --git a/toolchain/parse/typed_nodes_test.cpp b/toolchain/parse/typed_nodes_test.cpp index 39c5f84ec88f..ffa54132ba4c 100644 --- a/toolchain/parse/typed_nodes_test.cpp +++ b/toolchain/parse/typed_nodes_test.cpp @@ -12,7 +12,7 @@ #include "toolchain/diagnostics/mocks.h" #include "toolchain/lex/lex.h" #include "toolchain/lex/tokenized_buffer.h" -#include "toolchain/parse/tree.h" +#include "toolchain/parse/parse.h" namespace Carbon::Parse { namespace { @@ -40,8 +40,8 @@ class TypedNodeTest : public ::testing::Test { } auto GetTree(llvm::StringRef t) -> Tree& { - tree_storage_.push_front(Tree::Parse(GetTokenizedBuffer(t), consumer_, - /*vlog_stream=*/nullptr)); + tree_storage_.push_front(Parse(GetTokenizedBuffer(t), consumer_, + /*vlog_stream=*/nullptr)); return tree_storage_.front(); }