diff --git a/testing/file_test/file_test_base.cpp b/testing/file_test/file_test_base.cpp index 3946f4d0afec..f9f724a133f0 100644 --- a/testing/file_test/file_test_base.cpp +++ b/testing/file_test/file_test_base.cpp @@ -35,6 +35,9 @@ ABSL_FLAG(bool, autoupdate, false, ABSL_FLAG(unsigned int, threads, 0, "Number of threads to use when autoupdating tests, or 0 to " "automatically determine a thread count."); +ABSL_FLAG(bool, dump_output, false, + "Instead of verifying files match test output, directly dump output " + "to stderr."); namespace Carbon::Testing { @@ -66,20 +69,25 @@ static auto SplitOutput(llvm::StringRef output) // Runs a test and compares output. This keeps output split by line so that // issues are a little easier to identify by the different line. auto FileTestBase::TestBody() -> void { - std::optional stack_trace_entry; + std::string test_command; + std::optional stack_trace_entry; // If we're being run from bazel, provide some assistance for understanding // and reproducing failures. const char* target = getenv("TEST_TARGET"); if (target) { + constexpr const char* CommandFormat = + "bazel {0} {1} --test_arg=--file_tests={2}"; + test_command = llvm::formatv(CommandFormat, "test", target, test_name_); + // Add a crash trace entry with the run command. + stack_trace_entry.emplace(test_command.c_str()); + // This advice overrides the --file_tests flag provided by the file_test // rule. - llvm::errs() << "\nTo test this file alone, run:\n bazel test " << target - << " --test_arg=--file_tests=" << test_name_ << "\n\n"; - - // Add a crash trace entry with a command that runs this test in isolation. - stack_trace_entry.emplace("bazel test %s --test_arg=--file_tests=%s", - target, test_name_); + llvm::errs() << "\nTo test this file alone, run:\n " << test_command + << "\n\nTo view output, run:\n " + << llvm::formatv(CommandFormat, "run", target, test_name_) + << " --test_arg=--dump_output\n\n"; } TestContext context; @@ -159,8 +167,9 @@ auto FileTestBase::RunAutoupdater(const TestContext& context, bool dry_run) auto FileTestBase::Autoupdate() -> ErrorOr { // Add a crash trace entry mentioning which file we're updating. - llvm::PrettyStackTraceFormat stack_trace_entry("performing autoupdate for %s", - test_name_); + std::string stack_trace_string = + llvm::formatv("performing autoupdate for {0}", test_name_); + llvm::PrettyStackTraceString stack_trace_entry(stack_trace_string.c_str()); TestContext context; auto run_result = ProcessTestFileAndRun(context); @@ -171,6 +180,25 @@ auto FileTestBase::Autoupdate() -> ErrorOr { return RunAutoupdater(context, /*dry_run=*/false); } +auto FileTestBase::DumpOutput() -> ErrorOr { + TestContext context; + std::string banner(79, '='); + banner.append("\n"); + llvm::errs() << banner << "= " << test_name_ << "\n"; + + auto run_result = ProcessTestFileAndRun(context); + if (!run_result.ok()) { + return ErrorBuilder() << "Error updating " << test_name_ << ": " + << run_result.error(); + } + llvm::errs() << banner << "= stderr\n" + << banner << context.stderr << banner << "= stdout\n" + << banner << context.stdout << banner << "= Exit with success: " + << (context.exit_with_success ? "true" : "false") << "\n" + << banner; + return Success(); +} + auto FileTestBase::GetLineNumberReplacements( llvm::ArrayRef filenames) -> llvm::SmallVector { @@ -741,6 +769,10 @@ static auto Main(int argc, char** argv) -> int { << error.message() << "\n"; return EXIT_FAILURE; } + if (absl::GetFlag(FLAGS_autoupdate) && absl::GetFlag(FLAGS_dump_output)) { + llvm::errs() << "--autoupdate and --dump_output are mutually exclusive.\n"; + return EXIT_FAILURE; + } llvm::SmallVector tests = GetTests(); auto test_factory = GetFileTestFactory(); @@ -765,6 +797,16 @@ static auto Main(int argc, char** argv) -> int { pool.wait(); llvm::errs() << "\nDone!\n"; return EXIT_SUCCESS; + } else if (absl::GetFlag(FLAGS_dump_output)) { + for (const auto& test_name : tests) { + std::unique_ptr test(test_factory.factory_fn(test_name)); + auto result = test->DumpOutput(); + if (!result.ok()) { + llvm::errs() << "\n" << result.error().message() << "\n"; + } + } + llvm::errs() << "\nDone!\n"; + return EXIT_SUCCESS; } else { for (llvm::StringRef test_name : tests) { testing::RegisterTest(test_factory.name, test_name.data(), nullptr, diff --git a/testing/file_test/file_test_base.h b/testing/file_test/file_test_base.h index 2e12883eec36..6437529ab08e 100644 --- a/testing/file_test/file_test_base.h +++ b/testing/file_test/file_test_base.h @@ -87,6 +87,9 @@ class FileTestBase : public testing::Test { // Runs the test and autoupdates checks. Returns true if updated. auto Autoupdate() -> ErrorOr; + // Runs the test and dumps output. + auto DumpOutput() -> ErrorOr; + // Returns the name of the test (relative to the repo root). auto test_name() const -> llvm::StringRef { return test_name_; }