From 16bbdbbdb84d0682891fa72286c85bb721ad8773 Mon Sep 17 00:00:00 2001 From: Jon Ross-Perkins Date: Wed, 30 Nov 2022 17:28:49 -0800 Subject: [PATCH] Add vlog output to the parser. (#2435) Might eventually want to change this further, but I'm just adding the quick framework for it. --- toolchain/driver/driver.cpp | 2 +- toolchain/parser/BUILD | 1 + toolchain/parser/parse_tree.cpp | 6 ++--- toolchain/parser/parse_tree.h | 4 ++-- toolchain/parser/parse_tree_fuzzer.cpp | 2 +- toolchain/parser/parse_tree_test.cpp | 17 ++++++++------ toolchain/parser/parser.cpp | 7 +++++- toolchain/parser/parser.h | 31 +++++++++++++++++++++----- toolchain/parser/testdata/BUILD | 12 ++++++++++ 9 files changed, 61 insertions(+), 21 deletions(-) diff --git a/toolchain/driver/driver.cpp b/toolchain/driver/driver.cpp index dcfd9efe9d8a..f151ee79f03b 100644 --- a/toolchain/driver/driver.cpp +++ b/toolchain/driver/driver.cpp @@ -166,7 +166,7 @@ auto Driver::RunDumpSubcommand(DiagnosticConsumer& consumer, return !tokenized_source.has_errors(); } - auto parse_tree = ParseTree::Parse(tokenized_source, consumer); + auto parse_tree = ParseTree::Parse(tokenized_source, consumer, vlog_stream_); if (dump_mode == DumpMode::ParseTree) { consumer.Flush(); parse_tree.Print(output_stream_, parse_tree_preorder); diff --git a/toolchain/parser/BUILD b/toolchain/parser/BUILD index 457f3c171301..53bf7d02d06e 100644 --- a/toolchain/parser/BUILD +++ b/toolchain/parser/BUILD @@ -68,6 +68,7 @@ cc_library( "//common:check", "//common:error", "//common:ostream", + "//common:vlog", "//toolchain/diagnostics:diagnostic_emitter", "//toolchain/lexer:token_kind", "//toolchain/lexer:tokenized_buffer", diff --git a/toolchain/parser/parse_tree.cpp b/toolchain/parser/parse_tree.cpp index bf1c95809f9a..d3a2ef29d6bb 100644 --- a/toolchain/parser/parse_tree.cpp +++ b/toolchain/parser/parse_tree.cpp @@ -17,14 +17,14 @@ namespace Carbon { -auto ParseTree::Parse(TokenizedBuffer& tokens, DiagnosticConsumer& consumer) - -> ParseTree { +auto ParseTree::Parse(TokenizedBuffer& tokens, DiagnosticConsumer& consumer, + llvm::raw_ostream* vlog_stream) -> ParseTree { TokenizedBuffer::TokenLocationTranslator translator( &tokens, /*last_line_lexed_to_column=*/nullptr); TokenDiagnosticEmitter emitter(translator, consumer); // Delegate to the parser. - auto tree = Parser::Parse(tokens, emitter); + auto tree = Parser::Parse(tokens, emitter, vlog_stream); auto verify_error = tree.Verify(); CARBON_CHECK(!verify_error) << tree << *verify_error; return tree; diff --git a/toolchain/parser/parse_tree.h b/toolchain/parser/parse_tree.h index 840e262dd1cc..42a8181bcaae 100644 --- a/toolchain/parser/parse_tree.h +++ b/toolchain/parser/parse_tree.h @@ -49,8 +49,8 @@ class ParseTree { // Parses the token buffer into a `ParseTree`. // // This is the factory function which is used to build parse trees. - static auto Parse(TokenizedBuffer& tokens, DiagnosticConsumer& consumer) - -> ParseTree; + static auto Parse(TokenizedBuffer& tokens, DiagnosticConsumer& consumer, + llvm::raw_ostream* vlog_stream) -> ParseTree; // Tests whether there are any errors in the parse tree. [[nodiscard]] auto has_errors() const -> bool { return has_errors_; } diff --git a/toolchain/parser/parse_tree_fuzzer.cpp b/toolchain/parser/parse_tree_fuzzer.cpp index 4f12dac3258c..fd6ebdbb36d5 100644 --- a/toolchain/parser/parse_tree_fuzzer.cpp +++ b/toolchain/parser/parse_tree_fuzzer.cpp @@ -35,7 +35,7 @@ extern "C" int LLVMFuzzerTestOneInput(const unsigned char* data, // Now parse it into a tree. Note that parsing will (when asserts are enabled) // walk the entire tree to verify it so we don't have to do that here. - ParseTree::Parse(tokens, NullDiagnosticConsumer()); + ParseTree::Parse(tokens, NullDiagnosticConsumer(), /*vlog_stream=*/nullptr); return 0; } diff --git a/toolchain/parser/parse_tree_test.cpp b/toolchain/parser/parse_tree_test.cpp index 7adfb05529e1..5a82d5830e45 100644 --- a/toolchain/parser/parse_tree_test.cpp +++ b/toolchain/parser/parse_tree_test.cpp @@ -48,7 +48,7 @@ TEST_F(ParseTreeTest, DefaultInvalid) { TEST_F(ParseTreeTest, IsValid) { TokenizedBuffer tokens = GetTokenizedBuffer(""); - ParseTree tree = ParseTree::Parse(tokens, consumer); + ParseTree tree = ParseTree::Parse(tokens, consumer, /*vlog_stream=*/nullptr); EXPECT_TRUE((*tree.postorder().begin()).is_valid()); } @@ -91,7 +91,8 @@ TEST_F(ParseTreeTest, OperatorWhitespaceErrors) { for (auto [input, kind] : testcases) { TokenizedBuffer tokens = GetTokenizedBuffer(input); ErrorTrackingDiagnosticConsumer error_tracker(consumer); - ParseTree tree = ParseTree::Parse(tokens, error_tracker); + ParseTree tree = + ParseTree::Parse(tokens, error_tracker, /*vlog_stream=*/nullptr); EXPECT_THAT(tree.has_errors(), Eq(kind == Failed)) << input; EXPECT_THAT(error_tracker.seen_error(), Eq(kind != Valid)) << input; } @@ -141,14 +142,15 @@ TEST_F(ParseTreeTest, StructErrors) { TokenizedBuffer tokens = GetTokenizedBuffer(testcase.input); Testing::MockDiagnosticConsumer consumer; EXPECT_CALL(consumer, HandleDiagnostic(testcase.diag_matcher)); - ParseTree tree = ParseTree::Parse(tokens, consumer); + ParseTree tree = + ParseTree::Parse(tokens, consumer, /*vlog_stream=*/nullptr); EXPECT_TRUE(tree.has_errors()); } } TEST_F(ParseTreeTest, PrintPostorderAsYAML) { TokenizedBuffer tokens = GetTokenizedBuffer("fn F();"); - ParseTree tree = ParseTree::Parse(tokens, consumer); + ParseTree tree = ParseTree::Parse(tokens, consumer, /*vlog_stream=*/nullptr); EXPECT_FALSE(tree.has_errors()); std::string print_output; llvm::raw_string_ostream print_stream(print_output); @@ -172,7 +174,7 @@ TEST_F(ParseTreeTest, PrintPostorderAsYAML) { TEST_F(ParseTreeTest, PrintPreorderAsYAML) { TokenizedBuffer tokens = GetTokenizedBuffer("fn F();"); - ParseTree tree = ParseTree::Parse(tokens, consumer); + ParseTree tree = ParseTree::Parse(tokens, consumer, /*vlog_stream=*/nullptr); EXPECT_FALSE(tree.has_errors()); std::string print_output; llvm::raw_string_ostream print_stream(print_output); @@ -217,7 +219,7 @@ TEST_F(ParseTreeTest, HighRecursion) { TokenizedBuffer tokens = GetTokenizedBuffer(code); ASSERT_FALSE(tokens.has_errors()); Testing::MockDiagnosticConsumer consumer; - ParseTree tree = ParseTree::Parse(tokens, consumer); + ParseTree tree = ParseTree::Parse(tokens, consumer, /*vlog_stream=*/nullptr); EXPECT_FALSE(tree.has_errors()); } @@ -248,7 +250,8 @@ TEST_F(ParseTreeTest, PackageErrors) { TokenizedBuffer tokens = GetTokenizedBuffer(testcase.input); Testing::MockDiagnosticConsumer consumer; EXPECT_CALL(consumer, HandleDiagnostic(testcase.diag_matcher)); - ParseTree tree = ParseTree::Parse(tokens, consumer); + ParseTree tree = + ParseTree::Parse(tokens, consumer, /*vlog_stream=*/nullptr); EXPECT_TRUE(tree.has_errors()); } } diff --git a/toolchain/parser/parser.cpp b/toolchain/parser/parser.cpp index 9d3c79488a27..c77e2a64e6c5 100644 --- a/toolchain/parser/parser.cpp +++ b/toolchain/parser/parser.cpp @@ -82,10 +82,11 @@ class Parser::PrettyStackTraceParseState : public llvm::PrettyStackTraceEntry { }; Parser::Parser(ParseTree& tree, TokenizedBuffer& tokens, - TokenDiagnosticEmitter& emitter) + TokenDiagnosticEmitter& emitter, llvm::raw_ostream* vlog_stream) : tree_(&tree), tokens_(&tokens), emitter_(&emitter), + vlog_stream_(vlog_stream), position_(tokens_->tokens().begin()), end_(tokens_->tokens().end()) { CARBON_CHECK(position_ != end_) << "Empty TokenizedBuffer"; @@ -432,6 +433,8 @@ auto Parser::Parse() -> void { // Traces state_stack_. This runs even in opt because it's low overhead. PrettyStackTraceParseState pretty_stack(this); + CARBON_VLOG() << "*** Parser::Parse Begin ***\n"; + PushState(ParserState::DeclarationLoop()); while (!state_stack_.empty()) { switch (state_stack_.back().state) { @@ -444,6 +447,8 @@ auto Parser::Parse() -> void { } AddLeafNode(ParseNodeKind::FileEnd(), *position_); + + CARBON_VLOG() << "*** Parser::Parse End ***\n"; } auto Parser::HandleBraceExpressionState() -> void { diff --git a/toolchain/parser/parser.h b/toolchain/parser/parser.h index 8d2fed1bc8da..0880e558d9ba 100644 --- a/toolchain/parser/parser.h +++ b/toolchain/parser/parser.h @@ -6,6 +6,7 @@ #define CARBON_TOOLCHAIN_PARSER_PARSER_H_ #include "common/check.h" +#include "common/vlog.h" #include "llvm/ADT/Optional.h" #include "toolchain/lexer/token_kind.h" #include "toolchain/lexer/tokenized_buffer.h" @@ -23,10 +24,10 @@ class Parser { // Parses the tokens into a parse tree, emitting any errors encountered. // // This is the entry point to the parser implementation. - static auto Parse(TokenizedBuffer& tokens, TokenDiagnosticEmitter& emitter) - -> ParseTree { + static auto Parse(TokenizedBuffer& tokens, TokenDiagnosticEmitter& emitter, + llvm::raw_ostream* vlog_stream) -> ParseTree { ParseTree tree(tokens); - Parser parser(tree, tokens, emitter); + Parser parser(tree, tokens, emitter, vlog_stream); parser.Parse(); return tree; } @@ -58,6 +59,12 @@ class Parser { token(token), subtree_start(subtree_start) {} + // Prints state information for verbose output. + auto Print(llvm::raw_ostream& output) const -> void { + output << state << " @" << token << " subtree_start=" << subtree_start + << " has_error=" << has_error; + }; + // The state. ParserState state; // Set to true to indicate that an error was found, and that contextual @@ -92,7 +99,7 @@ class Parser { "StateStackEntry has unexpected size!"); Parser(ParseTree& tree, TokenizedBuffer& tokens, - TokenDiagnosticEmitter& emitter); + TokenDiagnosticEmitter& emitter, llvm::raw_ostream* vlog_stream); auto Parse() -> void; @@ -196,10 +203,18 @@ class Parser { } // Pops the state and keeps the value for inspection. - auto PopState() -> StateStackEntry { return state_stack_.pop_back_val(); } + auto PopState() -> StateStackEntry { + auto back = state_stack_.pop_back_val(); + CARBON_VLOG() << "Pop " << state_stack_.size() << ": " << back << "\n"; + return back; + } // Pops the state and discards it. - auto PopAndDiscardState() -> void { state_stack_.pop_back(); } + auto PopAndDiscardState() -> void { + CARBON_VLOG() << "PopAndDiscard " << state_stack_.size() - 1 << ": " + << state_stack_.back() << "\n"; + state_stack_.pop_back(); + } // Pushes a new state with the current position for context. auto PushState(ParserState state) -> void { @@ -225,6 +240,7 @@ class Parser { // Pushes a constructed state onto the stack. auto PushState(StateStackEntry state) -> void { + CARBON_VLOG() << "Push " << state_stack_.size() << ": " << state << "\n"; state_stack_.push_back(state); CARBON_CHECK(state_stack_.size() < (1 << 20)) << "Excessive stack size: likely infinite loop"; @@ -291,6 +307,9 @@ class Parser { TokenizedBuffer* tokens_; TokenDiagnosticEmitter* emitter_; + // Whether to print verbose output. + llvm::raw_ostream* vlog_stream_; + // The current position within the token buffer. TokenizedBuffer::TokenIterator position_; // The EndOfFile token. diff --git a/toolchain/parser/testdata/BUILD b/toolchain/parser/testdata/BUILD index 2524446953e5..39a9a50c86e0 100644 --- a/toolchain/parser/testdata/BUILD +++ b/toolchain/parser/testdata/BUILD @@ -25,3 +25,15 @@ glob_sh_run( data = ["//toolchain/driver:carbon"], file_exts = ["carbon"], ) + +glob_sh_run( + args = [ + "$(location //toolchain/driver:carbon)", + "-v", + "dump", + "parse-tree", + ], + data = ["//toolchain/driver:carbon"], + file_exts = ["carbon"], + run_ext = "verbose", +)