Add vlog output to the parser. (#2435)

Might eventually want to change this further, but I'm just adding the quick framework for it.

toolchain/driver/driver.cpp

@@ -166,7 +166,7 @@ auto Driver::RunDumpSubcommand(DiagnosticConsumer& consumer,
     return !tokenized_source.has_errors();
   }
 
-  auto parse_tree = ParseTree::Parse(tokenized_source, consumer);
+  auto parse_tree = ParseTree::Parse(tokenized_source, consumer, vlog_stream_);
   if (dump_mode == DumpMode::ParseTree) {
     consumer.Flush();
     parse_tree.Print(output_stream_, parse_tree_preorder);

toolchain/parser/BUILD

@@ -68,6 +68,7 @@ cc_library(
         "//common:check",
         "//common:error",
         "//common:ostream",
+        "//common:vlog",
         "//toolchain/diagnostics:diagnostic_emitter",
         "//toolchain/lexer:token_kind",
         "//toolchain/lexer:tokenized_buffer",

toolchain/parser/parse_tree.cpp

@@ -17,14 +17,14 @@
 
 namespace Carbon {
 
-auto ParseTree::Parse(TokenizedBuffer& tokens, DiagnosticConsumer& consumer)
-    -> ParseTree {
+auto ParseTree::Parse(TokenizedBuffer& tokens, DiagnosticConsumer& consumer,
+                      llvm::raw_ostream* vlog_stream) -> ParseTree {
   TokenizedBuffer::TokenLocationTranslator translator(
       &tokens, /*last_line_lexed_to_column=*/nullptr);
   TokenDiagnosticEmitter emitter(translator, consumer);
 
   // Delegate to the parser.
-  auto tree = Parser::Parse(tokens, emitter);
+  auto tree = Parser::Parse(tokens, emitter, vlog_stream);
   auto verify_error = tree.Verify();
   CARBON_CHECK(!verify_error) << tree << *verify_error;
   return tree;

toolchain/parser/parse_tree.h

@@ -49,8 +49,8 @@ class ParseTree {
   // Parses the token buffer into a `ParseTree`.
   //
   // This is the factory function which is used to build parse trees.
-  static auto Parse(TokenizedBuffer& tokens, DiagnosticConsumer& consumer)
-      -> ParseTree;
+  static auto Parse(TokenizedBuffer& tokens, DiagnosticConsumer& consumer,
+                    llvm::raw_ostream* vlog_stream) -> ParseTree;
 
   // Tests whether there are any errors in the parse tree.
   [[nodiscard]] auto has_errors() const -> bool { return has_errors_; }

toolchain/parser/parse_tree_fuzzer.cpp

@@ -35,7 +35,7 @@ extern "C" int LLVMFuzzerTestOneInput(const unsigned char* data,
 
   // Now parse it into a tree. Note that parsing will (when asserts are enabled)
   // walk the entire tree to verify it so we don't have to do that here.
-  ParseTree::Parse(tokens, NullDiagnosticConsumer());
+  ParseTree::Parse(tokens, NullDiagnosticConsumer(), /*vlog_stream=*/nullptr);
   return 0;
 }
 

toolchain/parser/parse_tree_test.cpp

@@ -48,7 +48,7 @@ TEST_F(ParseTreeTest, DefaultInvalid) {
 
 TEST_F(ParseTreeTest, IsValid) {
   TokenizedBuffer tokens = GetTokenizedBuffer("");
-  ParseTree tree = ParseTree::Parse(tokens, consumer);
+  ParseTree tree = ParseTree::Parse(tokens, consumer, /*vlog_stream=*/nullptr);
   EXPECT_TRUE((*tree.postorder().begin()).is_valid());
 }
 
@@ -91,7 +91,8 @@ TEST_F(ParseTreeTest, OperatorWhitespaceErrors) {
   for (auto [input, kind] : testcases) {
     TokenizedBuffer tokens = GetTokenizedBuffer(input);
     ErrorTrackingDiagnosticConsumer error_tracker(consumer);
-    ParseTree tree = ParseTree::Parse(tokens, error_tracker);
+    ParseTree tree =
+        ParseTree::Parse(tokens, error_tracker, /*vlog_stream=*/nullptr);
     EXPECT_THAT(tree.has_errors(), Eq(kind == Failed)) << input;
     EXPECT_THAT(error_tracker.seen_error(), Eq(kind != Valid)) << input;
   }
@@ -141,14 +142,15 @@ TEST_F(ParseTreeTest, StructErrors) {
     TokenizedBuffer tokens = GetTokenizedBuffer(testcase.input);
     Testing::MockDiagnosticConsumer consumer;
     EXPECT_CALL(consumer, HandleDiagnostic(testcase.diag_matcher));
-    ParseTree tree = ParseTree::Parse(tokens, consumer);
+    ParseTree tree =
+        ParseTree::Parse(tokens, consumer, /*vlog_stream=*/nullptr);
     EXPECT_TRUE(tree.has_errors());
   }
 }
 
 TEST_F(ParseTreeTest, PrintPostorderAsYAML) {
   TokenizedBuffer tokens = GetTokenizedBuffer("fn F();");
-  ParseTree tree = ParseTree::Parse(tokens, consumer);
+  ParseTree tree = ParseTree::Parse(tokens, consumer, /*vlog_stream=*/nullptr);
   EXPECT_FALSE(tree.has_errors());
   std::string print_output;
   llvm::raw_string_ostream print_stream(print_output);
@@ -172,7 +174,7 @@ TEST_F(ParseTreeTest, PrintPostorderAsYAML) {
 
 TEST_F(ParseTreeTest, PrintPreorderAsYAML) {
   TokenizedBuffer tokens = GetTokenizedBuffer("fn F();");
-  ParseTree tree = ParseTree::Parse(tokens, consumer);
+  ParseTree tree = ParseTree::Parse(tokens, consumer, /*vlog_stream=*/nullptr);
   EXPECT_FALSE(tree.has_errors());
   std::string print_output;
   llvm::raw_string_ostream print_stream(print_output);
@@ -217,7 +219,7 @@ TEST_F(ParseTreeTest, HighRecursion) {
   TokenizedBuffer tokens = GetTokenizedBuffer(code);
   ASSERT_FALSE(tokens.has_errors());
   Testing::MockDiagnosticConsumer consumer;
-  ParseTree tree = ParseTree::Parse(tokens, consumer);
+  ParseTree tree = ParseTree::Parse(tokens, consumer, /*vlog_stream=*/nullptr);
   EXPECT_FALSE(tree.has_errors());
 }
 
@@ -248,7 +250,8 @@ TEST_F(ParseTreeTest, PackageErrors) {
     TokenizedBuffer tokens = GetTokenizedBuffer(testcase.input);
     Testing::MockDiagnosticConsumer consumer;
     EXPECT_CALL(consumer, HandleDiagnostic(testcase.diag_matcher));
-    ParseTree tree = ParseTree::Parse(tokens, consumer);
+    ParseTree tree =
+        ParseTree::Parse(tokens, consumer, /*vlog_stream=*/nullptr);
     EXPECT_TRUE(tree.has_errors());
   }
 }

toolchain/parser/parser.cpp

@@ -82,10 +82,11 @@ class Parser::PrettyStackTraceParseState : public llvm::PrettyStackTraceEntry {
 };
 
 Parser::Parser(ParseTree& tree, TokenizedBuffer& tokens,
-               TokenDiagnosticEmitter& emitter)
+               TokenDiagnosticEmitter& emitter, llvm::raw_ostream* vlog_stream)
     : tree_(&tree),
       tokens_(&tokens),
       emitter_(&emitter),
+      vlog_stream_(vlog_stream),
       position_(tokens_->tokens().begin()),
       end_(tokens_->tokens().end()) {
   CARBON_CHECK(position_ != end_) << "Empty TokenizedBuffer";
@@ -432,6 +433,8 @@ auto Parser::Parse() -> void {
   // Traces state_stack_. This runs even in opt because it's low overhead.
   PrettyStackTraceParseState pretty_stack(this);
 
+  CARBON_VLOG() << "*** Parser::Parse Begin ***\n";
+
   PushState(ParserState::DeclarationLoop());
   while (!state_stack_.empty()) {
     switch (state_stack_.back().state) {
@@ -444,6 +447,8 @@ auto Parser::Parse() -> void {
   }
 
   AddLeafNode(ParseNodeKind::FileEnd(), *position_);
+
+  CARBON_VLOG() << "*** Parser::Parse End ***\n";
 }
 
 auto Parser::HandleBraceExpressionState() -> void {

toolchain/parser/parser.h

@@ -6,6 +6,7 @@
 #define CARBON_TOOLCHAIN_PARSER_PARSER_H_
 
 #include "common/check.h"
+#include "common/vlog.h"
 #include "llvm/ADT/Optional.h"
 #include "toolchain/lexer/token_kind.h"
 #include "toolchain/lexer/tokenized_buffer.h"
@@ -23,10 +24,10 @@ class Parser {
   // Parses the tokens into a parse tree, emitting any errors encountered.
   //
   // This is the entry point to the parser implementation.
-  static auto Parse(TokenizedBuffer& tokens, TokenDiagnosticEmitter& emitter)
-      -> ParseTree {
+  static auto Parse(TokenizedBuffer& tokens, TokenDiagnosticEmitter& emitter,
+                    llvm::raw_ostream* vlog_stream) -> ParseTree {
     ParseTree tree(tokens);
-    Parser parser(tree, tokens, emitter);
+    Parser parser(tree, tokens, emitter, vlog_stream);
     parser.Parse();
     return tree;
   }
@@ -58,6 +59,12 @@ class Parser {
           token(token),
           subtree_start(subtree_start) {}
 
+    // Prints state information for verbose output.
+    auto Print(llvm::raw_ostream& output) const -> void {
+      output << state << " @" << token << " subtree_start=" << subtree_start
+             << " has_error=" << has_error;
+    };
+
     // The state.
     ParserState state;
     // Set to true to indicate that an error was found, and that contextual
@@ -92,7 +99,7 @@ class Parser {
                 "StateStackEntry has unexpected size!");
 
   Parser(ParseTree& tree, TokenizedBuffer& tokens,
-         TokenDiagnosticEmitter& emitter);
+         TokenDiagnosticEmitter& emitter, llvm::raw_ostream* vlog_stream);
 
   auto Parse() -> void;
 
@@ -196,10 +203,18 @@ class Parser {
   }
 
   // Pops the state and keeps the value for inspection.
-  auto PopState() -> StateStackEntry { return state_stack_.pop_back_val(); }
+  auto PopState() -> StateStackEntry {
+    auto back = state_stack_.pop_back_val();
+    CARBON_VLOG() << "Pop " << state_stack_.size() << ": " << back << "\n";
+    return back;
+  }
 
   // Pops the state and discards it.
-  auto PopAndDiscardState() -> void { state_stack_.pop_back(); }
+  auto PopAndDiscardState() -> void {
+    CARBON_VLOG() << "PopAndDiscard " << state_stack_.size() - 1 << ": "
+                  << state_stack_.back() << "\n";
+    state_stack_.pop_back();
+  }
 
   // Pushes a new state with the current position for context.
   auto PushState(ParserState state) -> void {
@@ -225,6 +240,7 @@ class Parser {
 
   // Pushes a constructed state onto the stack.
   auto PushState(StateStackEntry state) -> void {
+    CARBON_VLOG() << "Push " << state_stack_.size() << ": " << state << "\n";
     state_stack_.push_back(state);
     CARBON_CHECK(state_stack_.size() < (1 << 20))
         << "Excessive stack size: likely infinite loop";
@@ -291,6 +307,9 @@ class Parser {
   TokenizedBuffer* tokens_;
   TokenDiagnosticEmitter* emitter_;
 
+  // Whether to print verbose output.
+  llvm::raw_ostream* vlog_stream_;
+
   // The current position within the token buffer.
   TokenizedBuffer::TokenIterator position_;
   // The EndOfFile token.

toolchain/parser/testdata/BUILD

@@ -25,3 +25,15 @@ glob_sh_run(
     data = ["//toolchain/driver:carbon"],
     file_exts = ["carbon"],
 )
+
+glob_sh_run(
+    args = [
+        "$(location //toolchain/driver:carbon)",
+        "-v",
+        "dump",
+        "parse-tree",
+    ],
+    data = ["//toolchain/driver:carbon"],
+    file_exts = ["carbon"],
+    run_ext = "verbose",
+)