Check explorer's full trace output (#2934)

Created `explorer/trace_testdata/full_trace.carbon` to test the whole trace and some changes in `explorer/file_test.cpp` to treat `/trace_testdata/` tests differently.

explorer/BUILD

@@ -46,7 +46,10 @@ file_test(
     # Bazel limits sharding to 50. We have lots of tests, so this should
     # maximize parallelism.
     shard_count = 50,
-    tests = glob(["testdata/**/*.carbon"]),
+    tests = glob([
+        "testdata/**/*.carbon",
+        "trace_testdata/**/*.carbon",
+    ]),
     deps = [
         "//explorer/parse_and_execute",
         "//testing/file_test:file_test_base",

explorer/autoupdate_trace_testdata.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+
+"""Updates the CHECK: lines in tests with an AUTOUPDATE line."""
+
+__copyright__ = """
+Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+Exceptions. See /LICENSE for license information.
+SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+"""
+
+import subprocess
+import sys
+from pathlib import Path
+
+
+def main() -> None:
+    # Subprocess to the main script in order to avoid Python import behaviors.
+    this_py = Path(__file__).resolve()
+    autoupdate_py = this_py.parent.parent.joinpath(
+        "testing", "scripts", "autoupdate_testdata_base.py"
+    )
+
+    args = [
+        str(autoupdate_py),
+        # Flags to configure for explorer testing.
+        "--tool=explorer",
+        "--testdata=explorer/trace_testdata",
+        "--autoupdate_arg=--trace_file=-",
+        "--autoupdate_arg=-trace_all",
+    ] + sys.argv[1:]
+    exit(subprocess.call(args))
+
+
+if __name__ == "__main__":
+    main()

explorer/file_test.cpp

@@ -16,9 +16,14 @@ class ParseAndExecuteTestFile : public FileTestBase {
       : FileTestBase(path), trace_(trace) {}
 
   auto SetUp() -> void override {
+    std::string path_str = path().string();
+    llvm::StringRef path_ref = path_str;
+
+    if (path_ref.find("trace_testdata") != llvm::StringRef::npos) {
+      is_trace_test = true;
+    }
+
     if (trace_) {
-      std::string path_str = path().string();
-      llvm::StringRef path_ref = path_str;
       if (path_ref.find("/limits/") != llvm::StringRef::npos) {
         GTEST_SKIP()
             << "`limits` tests check for various limit conditions (such as an "
@@ -30,6 +35,10 @@ class ParseAndExecuteTestFile : public FileTestBase {
                      "testdata/linked_list/typed_linked_list.carbon")) {
         GTEST_SKIP() << "Expensive test to trace";
       }
+    } else {
+      if (is_trace_test) {
+        GTEST_SKIP() << "`trace` tests only check for trace output.";
+      }
     }
   }
 
@@ -46,7 +55,7 @@ class ParseAndExecuteTestFile : public FileTestBase {
     TraceStream trace_stream;
     TestRawOstream trace_stream_ostream;
     if (trace_) {
-      trace_stream.set_stream(&trace_stream_ostream);
+      trace_stream.set_stream(is_trace_test ? &stdout : &trace_stream_ostream);
       trace_stream.set_allowed_phases({ProgramPhase::All});
     }
 
@@ -68,7 +77,9 @@ class ParseAndExecuteTestFile : public FileTestBase {
       stderr << result.error() << "\n";
     }
 
-    if (trace_) {
+    // Skip trace test check as they use stdout stream instead of
+    // trace_stream_ostream
+    if (trace_ && !is_trace_test) {
       EXPECT_FALSE(trace_stream_ostream.TakeStr().empty())
           << "Tracing should always do something";
     }
@@ -78,6 +89,7 @@ class ParseAndExecuteTestFile : public FileTestBase {
 
  private:
   bool trace_;
+  bool is_trace_test = false;
 };
 
 }  // namespace

testing/scripts/autoupdate_testdata_base.py

@@ -198,9 +198,13 @@ class CheckLine(Line):
         super().__init__()
         self.filename = Path(test).name
         self.indent = ""
-        self.out_line = out_line.rstrip()
+        self.out_line = out_line
         self.line_number_delta_prefix = line_number_delta_prefix
         self.line_number_pattern = line_number_pattern
+        self.time_elapsed_pattern = re.compile(
+            r"Time elapsed in (\S+): (\d+)ms"
+        )
+        self.trailing_whitespace_pattern = re.compile(r"(\s+$)")
 
         # If any match is specific to this file, use the first matched line for
         # the location of the CHECK comment.
@@ -216,26 +220,36 @@ class CheckLine(Line):
         assert self.out_line
         result = self.out_line
         while True:
-            match = self.line_number_pattern.search(result)
-            if not match:
-                break
-            if self._matches_filename(match):
-                line_number = int(match.group("line")) - 1
-                delta = line_number_remap[line_number] - output_line_number
-                # We use `:+d` here to produce `LINE-n` or `LINE+n` as
-                # appropriate.
-                result = self.line_number_pattern.sub(
-                    rf"\g<prefix>{self.line_number_delta_prefix}"
-                    rf"[[@LINE{delta:+d}]]\g<suffix>",
-                    result,
-                    count=1,
+            line_match = self.line_number_pattern.search(result)
+            time_match = self.time_elapsed_pattern.search(result)
+            trailing_match = self.trailing_whitespace_pattern.search(result)
+            if line_match:
+                if self._matches_filename(line_match):
+                    line_number = int(line_match.group("line")) - 1
+                    delta = line_number_remap[line_number] - output_line_number
+                    # We use `:+d` here to produce `LINE-n` or `LINE+n` as
+                    # appropriate.
+                    result = self.line_number_pattern.sub(
+                        rf"\g<prefix>{self.line_number_delta_prefix}"
+                        rf"[[@LINE{delta:+d}]]\g<suffix>",
+                        result,
+                        count=1,
+                    )
+                else:
+                    result = self.line_number_pattern.sub(
+                        r"\g<prefix>{{.*}}\g<suffix>",
+                        result,
+                        count=1,
+                    )
+            elif time_match:
+                result = self.time_elapsed_pattern.sub(
+                    r"Time elapsed in \1: {{[0-9]+}}ms", result, count=1
                 )
+            elif trailing_match:
+                result = self.trailing_whitespace_pattern.sub(r"{{\1}}", result)
             else:
-                result = self.line_number_pattern.sub(
-                    r"\g<prefix>{{.*}}\g<suffix>",
-                    result,
-                    count=1,
-                )
+                break
+
         return f"{self.indent}// CHECK:{result}\n"
 
     def _matches_filename(self, match: Match) -> bool: