mirror of
https://github.com/carbon-language/carbon-lang.git
synced 2026-09-24 21:30:12 +01:00
Add --dump_output to file_test to help debugging. (#3698)
When we always had a single file in the test file, we could run the
driver over it to see output. This adds the ability to do something
similar for multi-file tests. So for example in test output, there's
now:
```
[ RUN ] ToolchainFileTest.toolchain/check/testdata/class/fail_todo_import.carbon
To test this file alone, run:
bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/check/testdata/class/fail_todo_import.carbon
To view output, run:
bazel run //toolchain/testing:file_test --test_arg=--file_tests=toolchain/check/testdata/class/fail_todo_import.carbon --test_arg=--dump_output
[ OK ] ToolchainFileTest.toolchain/check/testdata/class/fail_todo_import.carbon (64 ms)
```
And here's what a run looks like:
```
$ bazel run //toolchain/testing:file_test --test_arg=--file_tests=toolchain/check/testdata/class/fail_todo_import.carbon --test_arg=--dump_output
(elided bazel output)
Executing tests from //toolchain/testing:file_test
-----------------------------------------------------------------------------
===============================================================================
= toolchain/check/testdata/class/fail_todo_import.carbon
===============================================================================
= stderr
===============================================================================
b.carbon: ERROR: Semantics TODO: `TryResolveImportRefUnused on ClassDecl`.
b.carbon: ERROR: Semantics TODO: `TryResolveImportRefUnused on ClassDecl`.
b.carbon: ERROR: Semantics TODO: `TryResolveInst on ClassType`.
b.carbon: ERROR: Semantics TODO: `TryResolveInst on ClassType`.
b.carbon:18:29: ERROR: Name `d_ref` not found.
var d: (ForwardDeclared,) = d_ref;
^~~~~
===============================================================================
= stdout
===============================================================================
--- a.carbon
(eliding semir output)
}
===============================================================================
= Exit with success: false
===============================================================================
Done!
```
This also switches from PrettyStackTraceFormat to
PrettyStackTraceString. This is partly so that we can share the produced
command (avoiding two different format strings corresponding to the same
value), but also it's in my mind to step away from
https://github.com/llvm/llvm-project/pull/77351.
This commit is contained in:
@@ -35,6 +35,9 @@ ABSL_FLAG(bool, autoupdate, false,
|
||||
ABSL_FLAG(unsigned int, threads, 0,
|
||||
"Number of threads to use when autoupdating tests, or 0 to "
|
||||
"automatically determine a thread count.");
|
||||
ABSL_FLAG(bool, dump_output, false,
|
||||
"Instead of verifying files match test output, directly dump output "
|
||||
"to stderr.");
|
||||
|
||||
namespace Carbon::Testing {
|
||||
|
||||
@@ -66,20 +69,25 @@ static auto SplitOutput(llvm::StringRef output)
|
||||
// Runs a test and compares output. This keeps output split by line so that
|
||||
// issues are a little easier to identify by the different line.
|
||||
auto FileTestBase::TestBody() -> void {
|
||||
std::optional<llvm::PrettyStackTraceFormat> stack_trace_entry;
|
||||
std::string test_command;
|
||||
std::optional<llvm::PrettyStackTraceString> stack_trace_entry;
|
||||
|
||||
// If we're being run from bazel, provide some assistance for understanding
|
||||
// and reproducing failures.
|
||||
const char* target = getenv("TEST_TARGET");
|
||||
if (target) {
|
||||
constexpr const char* CommandFormat =
|
||||
"bazel {0} {1} --test_arg=--file_tests={2}";
|
||||
test_command = llvm::formatv(CommandFormat, "test", target, test_name_);
|
||||
// Add a crash trace entry with the run command.
|
||||
stack_trace_entry.emplace(test_command.c_str());
|
||||
|
||||
// This advice overrides the --file_tests flag provided by the file_test
|
||||
// rule.
|
||||
llvm::errs() << "\nTo test this file alone, run:\n bazel test " << target
|
||||
<< " --test_arg=--file_tests=" << test_name_ << "\n\n";
|
||||
|
||||
// Add a crash trace entry with a command that runs this test in isolation.
|
||||
stack_trace_entry.emplace("bazel test %s --test_arg=--file_tests=%s",
|
||||
target, test_name_);
|
||||
llvm::errs() << "\nTo test this file alone, run:\n " << test_command
|
||||
<< "\n\nTo view output, run:\n "
|
||||
<< llvm::formatv(CommandFormat, "run", target, test_name_)
|
||||
<< " --test_arg=--dump_output\n\n";
|
||||
}
|
||||
|
||||
TestContext context;
|
||||
@@ -159,8 +167,9 @@ auto FileTestBase::RunAutoupdater(const TestContext& context, bool dry_run)
|
||||
|
||||
auto FileTestBase::Autoupdate() -> ErrorOr<bool> {
|
||||
// Add a crash trace entry mentioning which file we're updating.
|
||||
llvm::PrettyStackTraceFormat stack_trace_entry("performing autoupdate for %s",
|
||||
test_name_);
|
||||
std::string stack_trace_string =
|
||||
llvm::formatv("performing autoupdate for {0}", test_name_);
|
||||
llvm::PrettyStackTraceString stack_trace_entry(stack_trace_string.c_str());
|
||||
|
||||
TestContext context;
|
||||
auto run_result = ProcessTestFileAndRun(context);
|
||||
@@ -171,6 +180,25 @@ auto FileTestBase::Autoupdate() -> ErrorOr<bool> {
|
||||
return RunAutoupdater(context, /*dry_run=*/false);
|
||||
}
|
||||
|
||||
auto FileTestBase::DumpOutput() -> ErrorOr<Success> {
|
||||
TestContext context;
|
||||
std::string banner(79, '=');
|
||||
banner.append("\n");
|
||||
llvm::errs() << banner << "= " << test_name_ << "\n";
|
||||
|
||||
auto run_result = ProcessTestFileAndRun(context);
|
||||
if (!run_result.ok()) {
|
||||
return ErrorBuilder() << "Error updating " << test_name_ << ": "
|
||||
<< run_result.error();
|
||||
}
|
||||
llvm::errs() << banner << "= stderr\n"
|
||||
<< banner << context.stderr << banner << "= stdout\n"
|
||||
<< banner << context.stdout << banner << "= Exit with success: "
|
||||
<< (context.exit_with_success ? "true" : "false") << "\n"
|
||||
<< banner;
|
||||
return Success();
|
||||
}
|
||||
|
||||
auto FileTestBase::GetLineNumberReplacements(
|
||||
llvm::ArrayRef<llvm::StringRef> filenames)
|
||||
-> llvm::SmallVector<LineNumberReplacement> {
|
||||
@@ -741,6 +769,10 @@ static auto Main(int argc, char** argv) -> int {
|
||||
<< error.message() << "\n";
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
if (absl::GetFlag(FLAGS_autoupdate) && absl::GetFlag(FLAGS_dump_output)) {
|
||||
llvm::errs() << "--autoupdate and --dump_output are mutually exclusive.\n";
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
llvm::SmallVector<std::string> tests = GetTests();
|
||||
auto test_factory = GetFileTestFactory();
|
||||
@@ -765,6 +797,16 @@ static auto Main(int argc, char** argv) -> int {
|
||||
pool.wait();
|
||||
llvm::errs() << "\nDone!\n";
|
||||
return EXIT_SUCCESS;
|
||||
} else if (absl::GetFlag(FLAGS_dump_output)) {
|
||||
for (const auto& test_name : tests) {
|
||||
std::unique_ptr<FileTestBase> test(test_factory.factory_fn(test_name));
|
||||
auto result = test->DumpOutput();
|
||||
if (!result.ok()) {
|
||||
llvm::errs() << "\n" << result.error().message() << "\n";
|
||||
}
|
||||
}
|
||||
llvm::errs() << "\nDone!\n";
|
||||
return EXIT_SUCCESS;
|
||||
} else {
|
||||
for (llvm::StringRef test_name : tests) {
|
||||
testing::RegisterTest(test_factory.name, test_name.data(), nullptr,
|
||||
|
||||
Reference in New Issue
Block a user