From cc6f21e402a3700f945945fd441d0630d34b1083 Mon Sep 17 00:00:00 2001 From: Jon Ross-Perkins Date: Mon, 12 Feb 2024 08:25:28 -0800 Subject: [PATCH] Add --dump_output to file_test to help debugging. (#3698) When we always had a single file in the test file, we could run the driver over it to see output. This adds the ability to do something similar for multi-file tests. So for example in test output, there's now: ``` [ RUN ] ToolchainFileTest.toolchain/check/testdata/class/fail_todo_import.carbon To test this file alone, run: bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/check/testdata/class/fail_todo_import.carbon To view output, run: bazel run //toolchain/testing:file_test --test_arg=--file_tests=toolchain/check/testdata/class/fail_todo_import.carbon --test_arg=--dump_output [ OK ] ToolchainFileTest.toolchain/check/testdata/class/fail_todo_import.carbon (64 ms) ``` And here's what a run looks like: ``` $ bazel run //toolchain/testing:file_test --test_arg=--file_tests=toolchain/check/testdata/class/fail_todo_import.carbon --test_arg=--dump_output (elided bazel output) Executing tests from //toolchain/testing:file_test ----------------------------------------------------------------------------- =============================================================================== = toolchain/check/testdata/class/fail_todo_import.carbon =============================================================================== = stderr =============================================================================== b.carbon: ERROR: Semantics TODO: `TryResolveImportRefUnused on ClassDecl`. b.carbon: ERROR: Semantics TODO: `TryResolveImportRefUnused on ClassDecl`. b.carbon: ERROR: Semantics TODO: `TryResolveInst on ClassType`. b.carbon: ERROR: Semantics TODO: `TryResolveInst on ClassType`. b.carbon:18:29: ERROR: Name `d_ref` not found. var d: (ForwardDeclared,) = d_ref; ^~~~~ =============================================================================== = stdout =============================================================================== --- a.carbon (eliding semir output) } =============================================================================== = Exit with success: false =============================================================================== Done! ``` This also switches from PrettyStackTraceFormat to PrettyStackTraceString. This is partly so that we can share the produced command (avoiding two different format strings corresponding to the same value), but also it's in my mind to step away from https://github.com/llvm/llvm-project/pull/77351. --- testing/file_test/file_test_base.cpp | 60 +++++++++++++++++++++++----- testing/file_test/file_test_base.h | 3 ++ 2 files changed, 54 insertions(+), 9 deletions(-) diff --git a/testing/file_test/file_test_base.cpp b/testing/file_test/file_test_base.cpp index 3946f4d0afec..f9f724a133f0 100644 --- a/testing/file_test/file_test_base.cpp +++ b/testing/file_test/file_test_base.cpp @@ -35,6 +35,9 @@ ABSL_FLAG(bool, autoupdate, false, ABSL_FLAG(unsigned int, threads, 0, "Number of threads to use when autoupdating tests, or 0 to " "automatically determine a thread count."); +ABSL_FLAG(bool, dump_output, false, + "Instead of verifying files match test output, directly dump output " + "to stderr."); namespace Carbon::Testing { @@ -66,20 +69,25 @@ static auto SplitOutput(llvm::StringRef output) // Runs a test and compares output. This keeps output split by line so that // issues are a little easier to identify by the different line. auto FileTestBase::TestBody() -> void { - std::optional stack_trace_entry; + std::string test_command; + std::optional stack_trace_entry; // If we're being run from bazel, provide some assistance for understanding // and reproducing failures. const char* target = getenv("TEST_TARGET"); if (target) { + constexpr const char* CommandFormat = + "bazel {0} {1} --test_arg=--file_tests={2}"; + test_command = llvm::formatv(CommandFormat, "test", target, test_name_); + // Add a crash trace entry with the run command. + stack_trace_entry.emplace(test_command.c_str()); + // This advice overrides the --file_tests flag provided by the file_test // rule. - llvm::errs() << "\nTo test this file alone, run:\n bazel test " << target - << " --test_arg=--file_tests=" << test_name_ << "\n\n"; - - // Add a crash trace entry with a command that runs this test in isolation. - stack_trace_entry.emplace("bazel test %s --test_arg=--file_tests=%s", - target, test_name_); + llvm::errs() << "\nTo test this file alone, run:\n " << test_command + << "\n\nTo view output, run:\n " + << llvm::formatv(CommandFormat, "run", target, test_name_) + << " --test_arg=--dump_output\n\n"; } TestContext context; @@ -159,8 +167,9 @@ auto FileTestBase::RunAutoupdater(const TestContext& context, bool dry_run) auto FileTestBase::Autoupdate() -> ErrorOr { // Add a crash trace entry mentioning which file we're updating. - llvm::PrettyStackTraceFormat stack_trace_entry("performing autoupdate for %s", - test_name_); + std::string stack_trace_string = + llvm::formatv("performing autoupdate for {0}", test_name_); + llvm::PrettyStackTraceString stack_trace_entry(stack_trace_string.c_str()); TestContext context; auto run_result = ProcessTestFileAndRun(context); @@ -171,6 +180,25 @@ auto FileTestBase::Autoupdate() -> ErrorOr { return RunAutoupdater(context, /*dry_run=*/false); } +auto FileTestBase::DumpOutput() -> ErrorOr { + TestContext context; + std::string banner(79, '='); + banner.append("\n"); + llvm::errs() << banner << "= " << test_name_ << "\n"; + + auto run_result = ProcessTestFileAndRun(context); + if (!run_result.ok()) { + return ErrorBuilder() << "Error updating " << test_name_ << ": " + << run_result.error(); + } + llvm::errs() << banner << "= stderr\n" + << banner << context.stderr << banner << "= stdout\n" + << banner << context.stdout << banner << "= Exit with success: " + << (context.exit_with_success ? "true" : "false") << "\n" + << banner; + return Success(); +} + auto FileTestBase::GetLineNumberReplacements( llvm::ArrayRef filenames) -> llvm::SmallVector { @@ -741,6 +769,10 @@ static auto Main(int argc, char** argv) -> int { << error.message() << "\n"; return EXIT_FAILURE; } + if (absl::GetFlag(FLAGS_autoupdate) && absl::GetFlag(FLAGS_dump_output)) { + llvm::errs() << "--autoupdate and --dump_output are mutually exclusive.\n"; + return EXIT_FAILURE; + } llvm::SmallVector tests = GetTests(); auto test_factory = GetFileTestFactory(); @@ -765,6 +797,16 @@ static auto Main(int argc, char** argv) -> int { pool.wait(); llvm::errs() << "\nDone!\n"; return EXIT_SUCCESS; + } else if (absl::GetFlag(FLAGS_dump_output)) { + for (const auto& test_name : tests) { + std::unique_ptr test(test_factory.factory_fn(test_name)); + auto result = test->DumpOutput(); + if (!result.ok()) { + llvm::errs() << "\n" << result.error().message() << "\n"; + } + } + llvm::errs() << "\nDone!\n"; + return EXIT_SUCCESS; } else { for (llvm::StringRef test_name : tests) { testing::RegisterTest(test_factory.name, test_name.data(), nullptr, diff --git a/testing/file_test/file_test_base.h b/testing/file_test/file_test_base.h index 2e12883eec36..6437529ab08e 100644 --- a/testing/file_test/file_test_base.h +++ b/testing/file_test/file_test_base.h @@ -87,6 +87,9 @@ class FileTestBase : public testing::Test { // Runs the test and autoupdates checks. Returns true if updated. auto Autoupdate() -> ErrorOr; + // Runs the test and dumps output. + auto DumpOutput() -> ErrorOr; + // Returns the name of the test (relative to the repo root). auto test_name() const -> llvm::StringRef { return test_name_; }