diff --git a/explorer/file_test.cpp b/explorer/file_test.cpp index f9886335b6bd..3e9785f974a3 100644 --- a/explorer/file_test.cpp +++ b/explorer/file_test.cpp @@ -19,17 +19,16 @@ class ExplorerFileTest : public FileTestBase { public: using FileTestBase::FileTestBase; - auto RunWithFiles(const llvm::SmallVector& test_args, - const llvm::SmallVector& test_files, - llvm::raw_pwrite_stream& stdout, - llvm::raw_pwrite_stream& stderr) -> bool override { + auto Run(const llvm::SmallVector& test_args, + const llvm::SmallVector& test_files, + llvm::raw_pwrite_stream& stdout, llvm::raw_pwrite_stream& stderr) + -> ErrorOr override { // Create the files in-memory. llvm::vfs::InMemoryFileSystem fs(new llvm::vfs::InMemoryFileSystem()); for (const auto& test_file : test_files) { if (!fs.addFile(test_file.filename, /*ModificationTime=*/0, llvm::MemoryBuffer::getMemBuffer(test_file.content))) { - ADD_FAILURE() << "File is repeated: " << test_file.filename; - return false; + return ErrorBuilder() << "File is repeated: " << test_file.filename; } } @@ -37,8 +36,7 @@ class ExplorerFileTest : public FileTestBase { llvm::ErrorOr> prelude = llvm::MemoryBuffer::getFile("explorer/data/prelude.carbon"); if (prelude.getError()) { - ADD_FAILURE() << prelude.getError().message(); - return false; + return ErrorBuilder() << prelude.getError().message(); } // TODO: This path is long with a prefix / because of the path expectations // in tests. Change those to allow a shorter path (e.g., `prelude.carbon`) @@ -46,15 +44,13 @@ class ExplorerFileTest : public FileTestBase { static constexpr llvm::StringLiteral PreludePath = "/explorer/data/prelude.carbon"; if (!fs.addFile(PreludePath, /*ModificationTime=*/0, std::move(*prelude))) { - ADD_FAILURE() << "Duplicate prelude.carbon"; - return false; + return ErrorBuilder() << "Duplicate prelude.carbon"; } llvm::SmallVector args = {"explorer"}; for (auto arg : test_args) { args.push_back(arg.data()); } - TestRawOstream trace_stream; // Trace output is only checked for a few tests. bool check_trace_output = @@ -62,16 +58,19 @@ class ExplorerFileTest : public FileTestBase { int exit_code = ExplorerMain( args.size(), args.data(), /*install_path=*/"", PreludePath, stdout, - stderr, check_trace_output ? stdout : trace_stream, fs); + stderr, check_trace_output ? stdout : trace_stream_, fs); + return exit_code == EXIT_SUCCESS; + } + + auto ValidateRun(const llvm::SmallVector& /*test_files*/) + -> void override { // Skip trace test check as they use stdout stream instead of // trace_stream_ostream if (absl::GetFlag(FLAGS_trace)) { - EXPECT_FALSE(trace_stream.TakeStr().empty()) + EXPECT_FALSE(trace_stream_.TakeStr().empty()) << "Tracing should always do something"; } - - return exit_code == EXIT_SUCCESS; } auto GetDefaultArgs() -> llvm::SmallVector override { @@ -83,6 +82,8 @@ class ExplorerFileTest : public FileTestBase { args.push_back("%s"); return args; } + + TestRawOstream trace_stream_; }; } // namespace diff --git a/scripts/fix_cc_deps.py b/scripts/fix_cc_deps.py index 63bb1c3ffa44..97ab56d8410e 100755 --- a/scripts/fix_cc_deps.py +++ b/scripts/fix_cc_deps.py @@ -58,6 +58,10 @@ EXTERNAL_REPOS: Dict[str, ExternalRepo] = { "@com_google_absl": ExternalRepo( lambda x: re.sub(":", "/", x), "...", None ), + # :re2/re2.h -> re2/re2.h + "@com_googlesource_code_re2": ExternalRepo( + lambda x: re.sub(":", "", x), ":re2", None + ), } # TODO: proto rules are aspect-based and their generated files don't show up in diff --git a/testing/file_test/BUILD b/testing/file_test/BUILD index ad6d7f17c795..066a693e012a 100644 --- a/testing/file_test/BUILD +++ b/testing/file_test/BUILD @@ -6,14 +6,33 @@ load("rules.bzl", "file_test") package(default_visibility = ["//visibility:public"]) +cc_library( + name = "autoupdate", + testonly = 1, + srcs = ["autoupdate.cpp"], + hdrs = [ + "autoupdate.h", + "line.h", + ], + deps = [ + "//common:check", + "//common:ostream", + "@com_google_absl//absl/strings:string_view", + "@com_googlesource_code_re2//:re2", + "@llvm-project//llvm:Support", + ], +) + cc_library( name = "file_test_base", testonly = 1, srcs = ["file_test_base.cpp"], hdrs = ["file_test_base.h"], deps = [ + ":autoupdate", "//common:check", - "//testing/util:test_raw_ostream", + "//common:error", + "//common:ostream", "@com_google_absl//absl/flags:flag", "@com_google_absl//absl/flags:parse", "@com_google_googletest//:gtest", diff --git a/testing/file_test/README.md b/testing/file_test/README.md new file mode 100644 index 000000000000..d86e6c8292a7 --- /dev/null +++ b/testing/file_test/README.md @@ -0,0 +1,137 @@ +# file_test + + + +## BUILD + +A typical BUILD target will look like: + +``` +load("rules.bzl", "file_test") + +file_test( + name = "my_file_test", + srcs = ["my_file_test.cpp"], + tests = glob(["testdata/**"]), + deps = [ + ":my_lib", + "//testing/file_test:file_test_base", + "@com_google_googletest//:gtest", + "@llvm-project//llvm:Support", + ], +) +``` + +## Implementation + +A typical implementation will look like: + +``` +#include "my_library.h" + +#include "llvm/ADT/StringExtras.h" +#include "testing/file_test/file_test_base.h" + +namespace Carbon::Testing { +namespace { + +class MyFileTest : public FileTestBase { + public: + using FileTestBase::FileTestBase; + + // Called as part of individual test executions. + auto Run(const llvm::SmallVector& test_args, + const llvm::SmallVector& test_files, + llvm::raw_pwrite_stream& stdout, llvm::raw_pwrite_stream& stderr) + -> ErrorOr override { + MyFunctionality(test_args, stdout, stderr); + } + + // Provides arguments which are used in tests that don't provide ARGS. + auto GetDefaultArgs() -> llvm::SmallVector override { + return {"default_args", "%s"}; + } +}; + +} // namespace + +// Registers for the framework to construct the tests. +CARBON_FILE_TEST_FACTORY(MyFileTest); + +} // namespace Carbon::Testing +``` + +## Comment markers + +Settings in files are provided in comments, similar to `FileCheck` syntax. +`bazel run :file_test -- --autoupdate` automatically constructs compatible +CHECK:STDOUT: and CHECK:STDERR: lines. + +Supported comment markers are: + +- ``` + // AUTOUDPATE + // NOAUTOUPDATE + ``` + + Controls whether the checks in the file will be autoupdated if --autoupdate + is passed. Exactly one of these two markers must be present. If the file + uses splits, AUTOUPDATE must currently be before any splits. + + When autoupdating, CHECKs will be inserted starting below AUTOUPDATE. When a + CHECK has line information, autoupdate will try to insert the CHECK + immediately above the line it's associated with. When that happens, any + following CHECK lines without line information will immediately follow, + between the CHECK with line information and the associated line. + +- `// ARGS: ` + + Provides a space-separated list of arguments, which will be passed to + RunWithFiles as test_args. These are intended for use by the command as + arguments. + + Supported replacements within arguments are: + + - `%s` + + Replaced with the list of files. Currently only allowed as a standalone + argument, not a substring. + + - `%t` + + Replaced with `${TEST_TMPDIR}/temp_file`. + + ARGS can be specified at most once. If not provided, the FileTestBase child + is responsible for providing default arguments. + +- `// SET-CHECK-SUBSET` + + By default, all lines of output must have a CHECK match. Adding this as a + option sets it so that non-matching lines are ignored. All provided + CHECK:STDOUT: and CHECK:STDERR: lines must still have a match in output. + + SET-CHECK-SUBSET can be specified at most once. + +- `// --- ` + + By default, all file content is provided to the test as a single file in + test_files. Using this marker allows the file to be split into multiple + files which will all be passed to test_files. + + Files are not created on disk; it's expected the child will create an + InMemoryFilesystem if needed. + +- ``` + // CHECK:STDOUT: + // CHECK:STDERR: + ``` + + These provide a match for output from the command. See `SET-CHECK-SUBSET` + for how to change from full to subset matching of output. + + Output line matchers may contain `[[@LINE+offset]` and `{{regex}}` syntaxes, + similar to `FileCheck`. diff --git a/testing/file_test/autoupdate.cpp b/testing/file_test/autoupdate.cpp new file mode 100644 index 000000000000..6122c27ad4e3 --- /dev/null +++ b/testing/file_test/autoupdate.cpp @@ -0,0 +1,241 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "testing/file_test/autoupdate.h" + +#include + +#include "absl/strings/string_view.h" +#include "common/check.h" +#include "common/ostream.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/Support/FormatVariadic.h" +#include "re2/re2.h" + +namespace Carbon::Testing { + +// Put helper classes in an anonymous namespace. +namespace { + +class CheckLine : public FileTestLineBase { + public: + // RE2 is passed by a pointer because it doesn't support std::optional. + explicit CheckLine(int line_number, const RE2* line_number_re, + std::string line) + : FileTestLineBase(line_number), + line_number_re_(line_number_re), + line_(std::move(line)) {} + + auto Print(llvm::raw_ostream& out) const -> void override { + out << indent_ << line_; + } + + // When the location of the CHECK in output is known, we can set the indent + // and its line. + auto SetOutputLine(llvm::StringRef indent, int output_line_number) -> void { + indent_ = indent; + output_line_number_ = output_line_number; + } + + // When the location of all lines in a file are known, we can set the line + // offset based on the target line. + auto SetRemappedLine(const std::string& sub_for_formatv, + int target_line_number) -> void { + // Should only be called when we have a regex. + CARBON_CHECK(line_number_re_); + + int offset = target_line_number - output_line_number_; + const char* offset_prefix = offset < 0 ? "" : "+"; + std::string replacement = + llvm::formatv(sub_for_formatv.data(), + llvm::formatv("[[@LINE{0}{1}]]", offset_prefix, offset)); + RE2::Replace(&line_, *line_number_re_, replacement); + } + + private: + const RE2* line_number_re_; + std::string line_; + llvm::StringRef indent_; + int output_line_number_ = -1; +}; + +} // namespace + +// Adds output lines for autoupdate. +static auto AddCheckLines( + llvm::StringRef output, const char* label, + const llvm::SmallVector& filenames, + bool line_number_re_has_file, const RE2& line_number_re, + std::function do_extra_check_replacements, + llvm::SmallVector>& check_lines) -> void { + if (output.empty()) { + return; + } + + // Prepare to look for filenames in lines. + llvm::StringRef current_filename = filenames[0]; + const auto* remaining_filenames = filenames.begin() + 1; + + // %t substitution means we may see TEST_TMPDIR in output. + char* tmpdir_env = getenv("TEST_TMPDIR"); + CARBON_CHECK(tmpdir_env != nullptr); + llvm::StringRef tmpdir = tmpdir_env; + + llvm::SmallVector lines(llvm::split(output, '\n')); + // It's typical that output ends with a newline, but we don't want to add a + // blank CHECK for it. + if (lines.back().empty()) { + lines.pop_back(); + } + + int append_to = 0; + for (const auto& line : lines) { + std::string check_line = llvm::formatv("// CHECK:{0}:{1}{2}", label, + line.empty() ? "" : " ", line); + + // Ignore TEST_TMPDIR in output. + if (auto pos = check_line.find(tmpdir); pos != std::string::npos) { + check_line.replace(pos, tmpdir.size(), "{{.+}}"); + } + + do_extra_check_replacements(check_line); + + // Look for line information in the output. line_number is only set if the + // match is correct. + int line_number = -1; + int match_line_number; + if (line_number_re_has_file) { + absl::string_view match_filename; + if (RE2::PartialMatch(check_line, line_number_re, &match_filename, + &match_line_number)) { + llvm::StringRef match_filename_ref = match_filename; + if (match_filename_ref != current_filename) { + // If the filename doesn't match, it may be still usable if it refers + // to a later file. + const auto* pos = std::find(remaining_filenames, filenames.end(), + match_filename_ref); + if (pos != filenames.end()) { + remaining_filenames = pos + 1; + append_to = pos - filenames.begin(); + line_number = match_line_number; + } + } else { + // The line applies to the current file. + line_number = match_line_number; + } + } + } else { + // There's no file association, so we only look at the line. + if (RE2::PartialMatch(check_line, line_number_re, &match_line_number)) { + line_number = match_line_number; + } + } + check_lines[append_to].push_back( + CheckLine(line_number, line_number == -1 ? nullptr : &line_number_re, + check_line)); + } +} + +auto AutoupdateFileTest( + const std::filesystem::path& file_test_path, llvm::StringRef input_content, + const llvm::SmallVector& filenames, + int autoupdate_line_number, + llvm::SmallVector>& non_check_lines, + llvm::StringRef stdout, llvm::StringRef stderr, + FileTestLineNumberReplacement line_number_replacement, + std::function do_extra_check_replacements) -> bool { + // Prepare CHECK lines. + llvm::SmallVector> check_lines; + check_lines.resize(filenames.size()); + RE2 line_number_re(line_number_replacement.pattern); + CARBON_CHECK(line_number_re.ok()) << "Invalid line replacement RE2: `" + << line_number_replacement.pattern << "`"; + + AddCheckLines(stdout, "STDOUT", filenames, line_number_replacement.has_file, + line_number_re, do_extra_check_replacements, check_lines); + AddCheckLines(stderr, "STDERR", filenames, line_number_replacement.has_file, + line_number_re, do_extra_check_replacements, check_lines); + + // All CHECK lines are suppressed until we reach AUTOUPDATE. + bool reached_autoupdate = false; + + // Stitch together content. + llvm::SmallVector new_lines; + for (auto [filename, non_check_file, check_file] : + llvm::zip(filenames, non_check_lines, check_lines)) { + llvm::DenseMap output_line_remap; + int output_line_number = 0; + auto* check_line = check_file.begin(); + + // Looping through the original file, print check lines preceding each + // original line. + for (const auto& non_check_line : non_check_file) { + // If there are any non-check lines with an invalid line_number, it's + // something like a split directive which shouldn't increment + // output_line_number. + if (non_check_line.line_number() < 1) { + new_lines.push_back(&non_check_line); + continue; + } + + if (reached_autoupdate) { + for (; check_line != check_file.end() && + check_line->line_number() <= non_check_line.line_number(); + ++check_line) { + new_lines.push_back(check_line); + check_line->SetOutputLine(non_check_line.indent(), + ++output_line_number); + } + } else if (autoupdate_line_number == non_check_line.line_number()) { + // This is the AUTOUPDATE line, so we'll print it, then start printing + // CHECK lines. + reached_autoupdate = true; + } + new_lines.push_back(&non_check_line); + CARBON_CHECK( + output_line_remap + .insert({non_check_line.line_number(), ++output_line_number}) + .second); + } + + // This should always be true after the first file is processed. + CARBON_CHECK(reached_autoupdate); + + // Print remaining check lines which -- for whatever reason -- come after + // all original lines. + for (; check_line != check_file.end(); ++check_line) { + new_lines.push_back(check_line); + check_line->SetOutputLine("", ++output_line_number); + } + + // Update all remapped lines in CHECK output. + for (auto& offset_check_line : check_file) { + if (offset_check_line.line_number() >= 1) { + auto new_line = output_line_remap.find(offset_check_line.line_number()); + CARBON_CHECK(new_line != output_line_remap.end()); + offset_check_line.SetRemappedLine( + line_number_replacement.sub_for_formatv, new_line->second); + } + } + } + + // Generate the autoupdated file. + std::string new_content; + llvm::raw_string_ostream new_content_stream(new_content); + for (const auto& line : new_lines) { + line->Print(new_content_stream); + new_content_stream << '\n'; + } + + // Update the file on disk if needed. + if (new_content == input_content) { + return false; + } + std::ofstream out(file_test_path); + out << new_content; + return true; +} + +} // namespace Carbon::Testing diff --git a/testing/file_test/autoupdate.h b/testing/file_test/autoupdate.h new file mode 100644 index 000000000000..0683610f2272 --- /dev/null +++ b/testing/file_test/autoupdate.h @@ -0,0 +1,39 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef CARBON_TESTING_FILE_TEST_AUTOUPDATE_H_ +#define CARBON_TESTING_FILE_TEST_AUTOUPDATE_H_ + +#include + +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "testing/file_test/line.h" + +namespace Carbon::Testing { + +struct FileTestLineNumberReplacement { + bool has_file; + + // The line replacement. The pattern should match lines. If has_file, pattern + // should have a file and line group; otherwise, only a line group. + std::string pattern; + + // sub_for_formatv should provide {0} to substitute with [[@LINE...]] deltas. + std::string sub_for_formatv; +}; + +// Automatically updates CHECKs in the provided file. Returns true if updated. +auto AutoupdateFileTest( + const std::filesystem::path& file_test_path, llvm::StringRef input_content, + const llvm::SmallVector& filenames, + int autoupdate_line_number, + llvm::SmallVector>& non_check_lines, + llvm::StringRef stdout, llvm::StringRef stderr, + FileTestLineNumberReplacement line_number_replacement, + std::function do_extra_check_replacements) -> bool; + +} // namespace Carbon::Testing + +#endif // CARBON_TESTING_FILE_TEST_AUTOUPDATE_H_ diff --git a/testing/file_test/file_test_base.cpp b/testing/file_test/file_test_base.cpp index e7d764fe119f..7dd6becb674f 100644 --- a/testing/file_test/file_test_base.cpp +++ b/testing/file_test/file_test_base.cpp @@ -6,6 +6,7 @@ #include #include +#include #include "absl/flags/flag.h" #include "absl/flags/parse.h" @@ -14,10 +15,12 @@ #include "llvm/ADT/Twine.h" #include "llvm/Support/FormatVariadic.h" #include "llvm/Support/InitLLVM.h" -#include "testing/util/test_raw_ostream.h" ABSL_FLAG(std::vector, file_tests, {}, "A comma-separated list of tests for file_test infrastructure."); +ABSL_FLAG(bool, autoupdate, false, + "Instead of verifying files match test output, autoupdate files " + "based on test output."); namespace Carbon::Testing { @@ -59,65 +62,102 @@ auto FileTestBase::TestBody() -> void { llvm::errs() << "\nTo test this file alone, run:\n bazel test " << target << " --test_arg=--file_tests=" << test_file << "\n\n"; - // Store the file so that test_files can use references to content. - std::string test_content = ReadFile(path()); - - // Load expected output. - llvm::SmallVector test_args; - llvm::SmallVector test_files; - llvm::SmallVector> expected_stdout; - llvm::SmallVector> expected_stderr; - bool check_subset = false; - ProcessTestFile(test_content, test_args, test_files, expected_stdout, - expected_stderr, check_subset); - if (HasFailure()) { - return; - } - - // Process arguments. - if (test_args.empty()) { - test_args = GetDefaultArgs(); - } - DoArgReplacements(test_args, test_files); - if (HasFailure()) { - return; - } - - // Pass arguments as StringRef. - llvm::SmallVector test_args_ref; - test_args_ref.reserve(test_args.size()); - for (const auto& arg : test_args) { - test_args_ref.push_back(arg); - } - - // Capture trace streaming, but only when in debug mode. - TestRawOstream stdout; - TestRawOstream stderr; - bool run_succeeded = RunWithFiles(test_args_ref, test_files, stdout, stderr); - if (HasFailure()) { - return; - } + TestContext context; + auto run_result = ProcessTestFileAndRun(context); + ASSERT_TRUE(run_result.ok()) << run_result.error(); + ValidateRun(context.test_files); EXPECT_THAT(!llvm::StringRef(path().filename()).starts_with("fail_"), - Eq(run_succeeded)) + Eq(context.exit_with_success)) << "Tests should be prefixed with `fail_` if and only if running them " "is expected to fail."; // Check results. - if (check_subset) { - EXPECT_THAT(SplitOutput(stdout.TakeStr()), IsSupersetOf(expected_stdout)); - EXPECT_THAT(SplitOutput(stderr.TakeStr()), IsSupersetOf(expected_stderr)); + if (context.check_subset) { + EXPECT_THAT(SplitOutput(context.stdout), + IsSupersetOf(context.expected_stdout)); + EXPECT_THAT(SplitOutput(context.stderr), + IsSupersetOf(context.expected_stderr)); } else { - EXPECT_THAT(SplitOutput(stdout.TakeStr()), - ElementsAreArray(expected_stdout)); - EXPECT_THAT(SplitOutput(stderr.TakeStr()), - ElementsAreArray(expected_stderr)); + EXPECT_THAT(SplitOutput(context.stdout), + ElementsAreArray(context.expected_stdout)); + EXPECT_THAT(SplitOutput(context.stderr), + ElementsAreArray(context.expected_stderr)); } } +auto FileTestBase::Autoupdate() -> bool { + TestContext context; + auto run_result = ProcessTestFileAndRun(context); + CARBON_CHECK(run_result.ok()) << run_result.error(); + if (!context.autoupdate_line_number) { + return false; + } + + llvm::SmallVector filenames; + filenames.reserve(context.non_check_lines.size()); + if (context.non_check_lines.size() > 1) { + // There are splits, so we provide an empty name for the first file. + filenames.push_back({}); + } + for (const auto& file : context.test_files) { + filenames.push_back(file.filename); + } + + llvm::ArrayRef filenames_for_line_number = filenames; + if (filenames.size() > 1) { + filenames_for_line_number = filenames_for_line_number.drop_front(); + } + + return AutoupdateFileTest( + path(), context.input_content, filenames, *context.autoupdate_line_number, + context.non_check_lines, context.stdout, context.stderr, + GetLineNumberReplacement(filenames_for_line_number), + [&](std::string& line) { DoExtraCheckReplacements(line); }); +} + +auto FileTestBase::GetLineNumberReplacement( + llvm::ArrayRef filenames) -> LineNumberReplacement { + return { + .has_file = true, + .pattern = llvm::formatv(R"(({0}):(\d+):)", llvm::join(filenames, "|")), + .sub_for_formatv = R"(\1:{0}:)"}; +} + +auto FileTestBase::ProcessTestFileAndRun(TestContext& context) + -> ErrorOr { + // Store the file so that test_files can use references to content. + context.input_content = ReadFile(path()); + + // Load expected output. + CARBON_RETURN_IF_ERROR(ProcessTestFile(context)); + + // Process arguments. + if (context.test_args.empty()) { + context.test_args = GetDefaultArgs(); + } + CARBON_RETURN_IF_ERROR( + DoArgReplacements(context.test_args, context.test_files)); + + // Pass arguments as StringRef. + llvm::SmallVector test_args_ref; + test_args_ref.reserve(context.test_args.size()); + for (const auto& arg : context.test_args) { + test_args_ref.push_back(arg); + } + + // Capture trace streaming, but only when in debug mode. + llvm::raw_svector_ostream stdout(context.stdout); + llvm::raw_svector_ostream stderr(context.stderr); + CARBON_ASSIGN_OR_RETURN( + context.exit_with_success, + Run(test_args_ref, context.test_files, stdout, stderr)); + return Success(); +} + auto FileTestBase::DoArgReplacements( llvm::SmallVector& test_args, - const llvm::SmallVector& test_files) -> void { + const llvm::SmallVector& test_files) -> ErrorOr { for (auto* it = test_args.begin(); it != test_args.end(); ++it) { auto percent = it->find("%"); if (percent == std::string::npos) { @@ -125,13 +165,13 @@ auto FileTestBase::DoArgReplacements( } if (percent + 1 >= it->size()) { - FAIL() << "% is not allowed on its own: " << *it; + return ErrorBuilder() << "% is not allowed on its own: " << *it; } char c = (*it)[percent + 1]; switch (c) { case 's': { if (*it != "%s") { - FAIL() << "%s must be the full argument: " << *it; + return ErrorBuilder() << "%s must be the full argument: " << *it; } it = test_args.erase(it); for (const auto& file : test_files) { @@ -143,107 +183,150 @@ auto FileTestBase::DoArgReplacements( break; } case 't': { - char* temp = getenv("TEST_TMPDIR"); - CARBON_CHECK(temp != nullptr); - it->replace(percent, 2, llvm::formatv("{0}/temp_file", temp)); + char* tmpdir = getenv("TEST_TMPDIR"); + CARBON_CHECK(tmpdir != nullptr); + it->replace(percent, 2, llvm::formatv("{0}/temp_file", tmpdir)); break; } default: - FAIL() << "%" << c << " is not supported: " << *it; + return ErrorBuilder() << "%" << c << " is not supported: " << *it; } } + return Success(); } -auto FileTestBase::ProcessTestFile( - llvm::StringRef file_content, llvm::SmallVector& test_args, - llvm::SmallVector& test_files, - llvm::SmallVector>& expected_stdout, - llvm::SmallVector>& expected_stderr, - bool& check_subset) -> void { +auto FileTestBase::ProcessTestFile(TestContext& context) -> ErrorOr { + // Original file content, and a cursor for walking through it. + llvm::StringRef file_content = context.input_content; llvm::StringRef cursor = file_content; + + // Whether content has been found, only updated before a file split is found + // (which may be never). bool found_content_pre_split = false; + + // Whether either AUTOUDPATE or NOAUTOUPDATE was found. + bool found_autoupdate = false; + + // The index in the current test file. Will be reset on splits. int line_index = 0; + + // The current file name, considering splits. Not set for the default file. llvm::StringRef current_file_name; + + // The current file's start. const char* current_file_start = nullptr; + + context.non_check_lines.resize(1); while (!cursor.empty()) { auto [line, next_cursor] = cursor.split("\n"); cursor = next_cursor; + auto line_trimmed = line.ltrim(); static constexpr llvm::StringLiteral SplitPrefix = "// ---"; - if (line.consume_front(SplitPrefix)) { + if (line_trimmed.consume_front(SplitPrefix)) { + if (!found_autoupdate) { + // If there's a split, all output is appended at the end of each file + // before AUTOUPDATE. We may want to change that, but it's not necessary + // to handle right now. + return ErrorBuilder() + << "AUTOUPDATE/NOAUTOUPDATE setting must be in the first file."; + } + + context.non_check_lines.push_back({FileTestLine(0, line)}); // On a file split, add the previous file, then start a new one. if (current_file_start) { - test_files.push_back(TestFile( + context.test_files.push_back(TestFile( current_file_name.str(), - llvm::StringRef( - current_file_start, - line.begin() - current_file_start - SplitPrefix.size()))); - } else { + llvm::StringRef(current_file_start, line_trimmed.begin() - + current_file_start - + SplitPrefix.size()))); + } else if (found_content_pre_split) { // For the first split, we make sure there was no content prior. - ASSERT_FALSE(found_content_pre_split) - << "When using split files, there must be no content before the " - "first split file."; + return ErrorBuilder() + << "When using split files, there must be no content before the " + "first split file."; } - current_file_name = line.trim(); + current_file_name = line_trimmed.trim(); current_file_start = cursor.begin(); line_index = 0; continue; - } else if (!current_file_start && !line.starts_with("//") && - !line.trim().empty()) { + } else if (!current_file_start && !line_trimmed.starts_with("//") && + !line_trimmed.empty()) { found_content_pre_split = true; } ++line_index; // Process expectations when found. - auto line_trimmed = line.ltrim(); - if (line_trimmed.consume_front("// ARGS: ")) { - if (test_args.empty()) { - // Split the line into arguments. - std::pair cursor = - llvm::getToken(line_trimmed); - while (!cursor.first.empty()) { - test_args.push_back(std::string(cursor.first)); - cursor = llvm::getToken(cursor.second); - } - } else { - FAIL() << "ARGS was specified multiple times: " << line.str(); - } - } else if (line_trimmed == "// SET-CHECK-SUBSET") { - if (!check_subset) { - check_subset = true; - } else { - FAIL() << "SET-CHECK-SUBSET was specified multiple times"; - } - } else if (line_trimmed.consume_front("// CHECK")) { + if (line_trimmed.consume_front("// CHECK")) { + llvm::SmallVector>* expected = nullptr; if (line_trimmed.consume_front(":STDOUT:")) { - expected_stdout.push_back( - TransformExpectation(line_index, line_trimmed)); + expected = &context.expected_stdout; } else if (line_trimmed.consume_front(":STDERR:")) { - expected_stderr.push_back( - TransformExpectation(line_index, line_trimmed)); + expected = &context.expected_stderr; } else { - FAIL() << "Unexpected CHECK in input: " << line.str(); + return ErrorBuilder() << "Unexpected CHECK in input: " << line.str(); + } + expected->push_back(TransformExpectation(line_index, line_trimmed)); + } else { + context.non_check_lines.back().push_back(FileTestLine(line_index, line)); + if (line_trimmed.consume_front("// ARGS: ")) { + if (context.test_args.empty()) { + // Split the line into arguments. + std::pair cursor = + llvm::getToken(line_trimmed); + while (!cursor.first.empty()) { + context.test_args.push_back(std::string(cursor.first)); + cursor = llvm::getToken(cursor.second); + } + } else { + return ErrorBuilder() + << "ARGS was specified multiple times: " << line.str(); + } + } else if (line_trimmed == "// AUTOUPDATE" || + line_trimmed == "// NOAUTOUPDATE") { + if (found_autoupdate) { + return ErrorBuilder() + << "Multiple AUTOUPDATE/NOAUTOUPDATE settings found"; + } + found_autoupdate = true; + if (line_trimmed == "// AUTOUPDATE") { + context.autoupdate_line_number = line_index; + } + } else if (line_trimmed == "// SET-CHECK-SUBSET") { + if (!context.check_subset) { + context.check_subset = true; + } else { + return ErrorBuilder() + << "SET-CHECK-SUBSET was specified multiple times"; + } } } } + if (!found_autoupdate) { + return ErrorBuilder() << "Missing AUTOUPDATE/NOAUTOUPDATE setting"; + } + if (current_file_start) { - test_files.push_back( + context.test_files.push_back( TestFile(current_file_name.str(), llvm::StringRef(current_file_start, file_content.end() - current_file_start))); } else { // If no file splitting happened, use the main file as the test file. - test_files.push_back(TestFile(path().filename().string(), file_content)); + context.test_files.push_back( + TestFile(path().filename().string(), file_content)); } // Assume there is always a suffix `\n` in output. - if (!expected_stdout.empty()) { - expected_stdout.push_back(StrEq("")); + if (!context.expected_stdout.empty()) { + context.expected_stdout.push_back(StrEq("")); } - if (!expected_stderr.empty()) { - expected_stderr.push_back(StrEq("")); + if (!context.expected_stderr.empty()) { + context.expected_stderr.push_back(StrEq("")); } + + return Success(); } auto FileTestBase::TransformExpectation(int line_index, llvm::StringRef in) @@ -358,17 +441,30 @@ auto main(int argc, char** argv) -> int { auto test_factory = Carbon::Testing::GetFileTestFactory(); - // Register tests based on their absolute path. for (const auto& file_test : absl::GetFlag(FLAGS_file_tests)) { + // Pass the absolute path to the factory function. auto path = std::filesystem::absolute(file_test, ec); CARBON_CHECK(!ec) << file_test << ": " << ec.message(); CARBON_CHECK(llvm::StringRef(path.string()).starts_with(base_dir)) << "\n " << path << "\n should start with\n " << base_dir; - std::string test_name = path.string().substr(base_dir.size()); - testing::RegisterTest(test_factory.name, test_name.c_str(), nullptr, - test_name.c_str(), __FILE__, __LINE__, - [=]() { return test_factory.factory_fn(path); }); + if (absl::GetFlag(FLAGS_autoupdate)) { + std::unique_ptr test( + test_factory.factory_fn(path)); + llvm::errs() << (test->Autoupdate() ? "!" : "."); + } else { + std::string test_name = path.string().substr(base_dir.size()); + testing::RegisterTest(test_factory.name, test_name.c_str(), nullptr, + test_name.c_str(), __FILE__, __LINE__, + [=]() { return test_factory.factory_fn(path); }); + } + } + if (absl::GetFlag(FLAGS_autoupdate)) { + llvm::errs() << "\nDone!\n"; } - return RUN_ALL_TESTS(); + if (absl::GetFlag(FLAGS_autoupdate)) { + return EXIT_SUCCESS; + } else { + return RUN_ALL_TESTS(); + } } diff --git a/testing/file_test/file_test_base.h b/testing/file_test/file_test_base.h index 919b6ae70265..93667e8cc017 100644 --- a/testing/file_test/file_test_base.h +++ b/testing/file_test/file_test_base.h @@ -10,72 +10,17 @@ #include #include -#include +#include "common/error.h" +#include "common/ostream.h" +#include "llvm/ADT/SmallString.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" -#include "llvm/Support/raw_os_ostream.h" -#include "llvm/Support/raw_ostream.h" +#include "testing/file_test/autoupdate.h" namespace Carbon::Testing { -// A framework for testing files. Children write -// `CARBON_FILE_TEST_FACTORY(MyTest)` which is used to construct the tests. -// `RunWithFiles` must also be implemented and will be called as part of -// individual test executions. This framework includes a `main` implementation, -// so users must not provide one. -// -// Settings in files are provided in comments, similar to `FileCheck` syntax. -// `autoupdate_testdata.py` automatically constructs compatible CHECK:STDOUT: -// and CHECK:STDERR: lines. -// -// Supported comment markers are: -// -// - // ARGS: -// -// Provides a space-separated list of arguments, which will be passed to -// RunWithFiles as test_args. These are intended for use by the command as -// arguments. -// -// Supported replacements within arguments are: -// -// - %s -// -// Replaced with the list of files. Currently only allowed as a standalone -// argument, not a substring. -// -// - %t -// -// Replaced with `${TEST_TMPDIR}/temp_file`. -// -// ARGS can be specified at most once. If not provided, the FileTestBase child -// is responsible for providing default arguments. -// -// - // SET-CHECK-SUBSET -// -// By default, all lines of output must have a CHECK match. Adding this as a -// flag sets it so that non-matching lines are ignored. All provided -// CHECK:STDOUT: and CHECK:STDERR: lines must still have a match in output. -// -// SET-CHECK-SUBSET can be specified at most once. -// -// - // --- -// -// By default, all file content is provided to the test as a single file in -// test_files. Using this marker allows the file to be split into multiple -// files which will all be passed to test_files. -// -// Files are not created on disk; it's expected the child will create an -// InMemoryFilesystem if needed. -// -// - // CHECK:STDOUT: -// // CHECK:STDERR: -// -// These provides a match for output from the command. See SET-CHECK-SUBSET -// for how to change from full to subset matching of output. -// -// Output line matchers may contain `[[@LINE+offset]` and -// `{{regex}}` syntaxes, similar to `FileCheck`. +// A framework for testing files. See README.md for documentation. class FileTestBase : public testing::Test { public: struct TestFile { @@ -94,38 +39,100 @@ class FileTestBase : public testing::Test { llvm::StringRef content; }; + // Provided for child class convenience. + using LineNumberReplacement = FileTestLineNumberReplacement; + explicit FileTestBase(std::filesystem::path path) : path_(std::move(path)) {} - // Implemented by children to run the test. Called by the TestBody - // implementation, which will validate stdout and stderr. The return value - // should be false when "fail_" is in the filename. - virtual auto RunWithFiles(const llvm::SmallVector& test_args, - const llvm::SmallVector& test_files, - llvm::raw_pwrite_stream& stdout, - llvm::raw_pwrite_stream& stderr) -> bool = 0; + // Implemented by children to run the test. For example, TestBody validates + // stdout and stderr. + // + // Any test expectations should be called from ValidateRun, not Run. + // + // The return value should be an error if there was an abnormal error. It + // should be true if a binary would return EXIT_SUCCESS, and false for + // EXIT_FAILURE (which is a test success for `fail_*` tests). + virtual auto Run(const llvm::SmallVector& test_args, + const llvm::SmallVector& test_files, + llvm::raw_pwrite_stream& stdout, + llvm::raw_pwrite_stream& stderr) -> ErrorOr = 0; + + // Implemented by children to do post-Run test expectations. Only called when + // testing. Does not need to be provided if only CHECK test expectations are + // used. + virtual auto ValidateRun(const llvm::SmallVector& /*test_files*/) + -> void {} // Returns default arguments. Only called when a file doesn't set ARGS. virtual auto GetDefaultArgs() -> llvm::SmallVector = 0; + // Returns replacement information for line numbers. See LineReplacement for + // construction. + virtual auto GetLineNumberReplacement( + llvm::ArrayRef filenames) -> LineNumberReplacement; + + // Optionally allows children to provide extra replacements for autoupdate. + virtual auto DoExtraCheckReplacements(std::string& /*check_line*/) -> void {} + // Runs a test and compares output. This keeps output split by line so that // issues are a little easier to identify by the different line. auto TestBody() -> void final; + // Runs the test and autoupdates checks. Returns true if updated. + auto Autoupdate() -> bool; + // Returns the full path of the file being tested. auto path() -> const std::filesystem::path& { return path_; }; private: + // Encapsulates test context generated by processing and running. + struct TestContext { + // The input test file content. Other parts may reference this. + std::string input_content; + + // Lines which don't contain CHECKs, and thus need to be retained by + // autoupdate. Their line number in the file is attached. + // + // If there are splits, then the line is in the respective file. For N + // splits, there will be one vector for the parts of the input file which + // are not in any split, plus one vector per split file. + llvm::SmallVector> non_check_lines; + + // Arguments for the test, generated from ARGS. + llvm::SmallVector test_args; + + // Files in the test, generated by content and splits. + llvm::SmallVector test_files; + + // The location of the autoupdate marker, for autoupdated files. + std::optional autoupdate_line_number; + + // Whether checks are a subset, generated from SET-CHECK-SUBSET. + bool check_subset = false; + + // stdout and stderr based on CHECK lines in the file. + llvm::SmallVector> expected_stdout; + llvm::SmallVector> expected_stderr; + + // stdout and stderr from Run. 16 is arbitrary but a required value. + llvm::SmallString<16> stdout; + llvm::SmallString<16> stderr; + + // Whether Run exited with success. + bool exit_with_success = false; + }; + + // Processes the test file and runs the test. Returns an error if something + // went wrong. + auto ProcessTestFileAndRun(TestContext& context) -> ErrorOr; + // Does replacements in ARGS for %s and %t. auto DoArgReplacements(llvm::SmallVector& test_args, - const llvm::SmallVector& test_files) -> void; + const llvm::SmallVector& test_files) + -> ErrorOr; // Processes the test input, producing test files and expected output. - auto ProcessTestFile( - llvm::StringRef file_content, llvm::SmallVector& test_args, - llvm::SmallVector& test_files, - llvm::SmallVector>& expected_stdout, - llvm::SmallVector>& expected_stderr, - bool& check_subset) -> void; + auto ProcessTestFile(TestContext& context) -> ErrorOr; // Transforms an expectation on a given line from `FileCheck` syntax into a // standard regex matcher. diff --git a/testing/file_test/file_test_base_test.cpp b/testing/file_test/file_test_base_test.cpp index 6caecd198244..d9d58b2f0504 100644 --- a/testing/file_test/file_test_base_test.cpp +++ b/testing/file_test/file_test_base_test.cpp @@ -7,11 +7,7 @@ #include #include -#include -#include - #include "llvm/ADT/StringExtras.h" -#include "llvm/Support/raw_ostream.h" namespace Carbon::Testing { namespace { @@ -34,10 +30,10 @@ class FileTestBaseTest : public FileTestBase { return Field("content", &TestFile::content, Eq(content)); } - auto RunWithFiles(const llvm::SmallVector& test_args, - const llvm::SmallVector& test_files, - llvm::raw_pwrite_stream& stdout, - llvm::raw_pwrite_stream& stderr) -> bool override { + auto Run(const llvm::SmallVector& test_args, + const llvm::SmallVector& test_files, + llvm::raw_pwrite_stream& stdout, llvm::raw_pwrite_stream& stderr) + -> ErrorOr override { if (!test_args.empty()) { llvm::ListSeparator sep; stdout << test_args.size() << " args: "; @@ -49,38 +45,46 @@ class FileTestBaseTest : public FileTestBase { auto filename = path().filename(); if (filename == "args.carbon") { - EXPECT_THAT(test_files, ElementsAre(HasFilename("args.carbon"))); return true; } else if (filename == "example.carbon") { - EXPECT_THAT(test_files, ElementsAre(HasFilename("example.carbon"))); + int delta_line = 10; stdout << "something\n" - "\n" - "9: Line delta\n" - "8: Negative line delta\n" - "+*[]{}\n" - "Foo baz\n"; + << "\n" + << "example.carbon:" << delta_line + 1 << ": Line delta\n" + << "example.carbon:" << delta_line << ": Negative line delta\n" + << "+*[]{}\n" + << "Foo baz\n"; return true; } else if (filename == "fail_example.carbon") { - EXPECT_THAT(test_files, ElementsAre(HasFilename("fail_example.carbon"))); stderr << "Oops\n"; return false; } else if (filename == "two_files.carbon") { int i = 0; for (const auto& file : test_files) { // Prints line numbers to validate per-file. - stdout << file.filename << ": " << ++i << "\n"; + stdout << file.filename << ":2: " << ++i << "\n"; } + return true; + } else { + return ErrorBuilder() << "Unexpected file: " << filename; + } + } + + auto ValidateRun(const llvm::SmallVector& test_files) + -> void override { + auto filename = path().filename(); + if (filename == "two_files.carbon") { EXPECT_THAT( test_files, ElementsAre( AllOf(HasFilename("a.carbon"), - HasContent("// CHECK:STDOUT: a.carbon: [[@LINE+0]]\n\n")), + HasContent( + "// CHECK:STDOUT: a.carbon:[[@LINE+1]]: 1\naaa\n\n")), AllOf(HasFilename("b.carbon"), - HasContent("// CHECK:STDOUT: b.carbon: [[@LINE+1]]\n")))); - return true; + HasContent( + "// CHECK:STDOUT: b.carbon:[[@LINE+1]]: 2\nbbb\n")))); } else { - ADD_FAILURE() << "Unexpected file: " << filename; - return false; + EXPECT_THAT(test_files, ElementsAre(HasFilename(filename))); } } diff --git a/testing/file_test/line.h b/testing/file_test/line.h new file mode 100644 index 000000000000..379d209de074 --- /dev/null +++ b/testing/file_test/line.h @@ -0,0 +1,46 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef CARBON_TESTING_FILE_TEST_LINE_H_ +#define CARBON_TESTING_FILE_TEST_LINE_H_ + +#include "common/ostream.h" +#include "llvm/ADT/StringRef.h" + +namespace Carbon::Testing { + +// Interface for lines. +class FileTestLineBase { + public: + explicit FileTestLineBase(int line_number) : line_number_(line_number) {} + virtual ~FileTestLineBase() {} + + // Prints the autoupdated line. + virtual auto Print(llvm::raw_ostream& out) const -> void = 0; + + auto line_number() const -> int { return line_number_; } + + private: + int line_number_; +}; + +// A line in the original file test. +class FileTestLine : public FileTestLineBase { + public: + explicit FileTestLine(int line_number, llvm::StringRef line) + : FileTestLineBase(line_number), line_(line) {} + + auto Print(llvm::raw_ostream& out) const -> void override { out << line_; } + + auto indent() const -> llvm::StringRef { + return line_.substr(0, line_.find_first_not_of(" \n")); + } + + private: + llvm::StringRef line_; +}; + +} // namespace Carbon::Testing + +#endif // CARBON_TESTING_FILE_TEST_LINE_H_ diff --git a/testing/file_test/testdata/args.carbon b/testing/file_test/testdata/args.carbon index 696dd1ffb4a6..ad267d2183f6 100644 --- a/testing/file_test/testdata/args.carbon +++ b/testing/file_test/testdata/args.carbon @@ -3,4 +3,5 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // ARGS: abc file=%t %s +// AUTOUPDATE // CHECK:STDOUT: 3 args: `abc`, `file={{.+}}/temp_file`, `args.carbon` diff --git a/testing/file_test/testdata/example.carbon b/testing/file_test/testdata/example.carbon index 85b66ecf2ae4..8a431224072d 100644 --- a/testing/file_test/testdata/example.carbon +++ b/testing/file_test/testdata/example.carbon @@ -2,10 +2,12 @@ // Exceptions. See /LICENSE for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// Can't autoupdate due to the `{{.+}}` check. +// NOAUTOUPDATE // CHECK:STDOUT: 2 args: `default_args`, `example.carbon` // CHECK:STDOUT: something // CHECK:STDOUT: -// CHECK:STDOUT: [[@LINE+1]]: Line delta -// CHECK:STDOUT: [[@LINE-1]]: Negative line delta +// CHECK:STDOUT: example.carbon:[[@LINE+1]]: Line delta +// CHECK:STDOUT: example.carbon:[[@LINE-1]]: Negative line delta // CHECK:STDOUT: +*[]{} // CHECK:STDOUT: F{{.+}}z diff --git a/testing/file_test/testdata/fail_example.carbon b/testing/file_test/testdata/fail_example.carbon index de4130b67915..7d14a9e03290 100644 --- a/testing/file_test/testdata/fail_example.carbon +++ b/testing/file_test/testdata/fail_example.carbon @@ -2,5 +2,6 @@ // Exceptions. See /LICENSE for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// AUTOUPDATE // CHECK:STDOUT: 2 args: `default_args`, `fail_example.carbon` // CHECK:STDERR: Oops diff --git a/testing/file_test/testdata/two_files.carbon b/testing/file_test/testdata/two_files.carbon index 9d826c0f711f..01955aa857dc 100644 --- a/testing/file_test/testdata/two_files.carbon +++ b/testing/file_test/testdata/two_files.carbon @@ -2,10 +2,13 @@ // Exceptions. See /LICENSE for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// AUTOUPDATE // CHECK:STDOUT: 3 args: `default_args`, `a.carbon`, `b.carbon` // --- a.carbon -// CHECK:STDOUT: a.carbon: [[@LINE+0]] +// CHECK:STDOUT: a.carbon:[[@LINE+1]]: 1 +aaa // --- b.carbon -// CHECK:STDOUT: b.carbon: [[@LINE+1]] +// CHECK:STDOUT: b.carbon:[[@LINE+1]]: 2 +bbb diff --git a/toolchain/autoupdate_testdata.py b/toolchain/autoupdate_testdata.py new file mode 100755 index 000000000000..6b2d20f56847 --- /dev/null +++ b/toolchain/autoupdate_testdata.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 + +"""Autoupdates testdata in toolchain.""" + +__copyright__ = """ +Part of the Carbon Language project, under the Apache License v2.0 with LLVM +Exceptions. See /LICENSE for license information. +SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +""" + +import argparse +import subprocess + +TARGETS = { + "codegen": "//toolchain/codegen:codegen_file_test", + "driver": "//toolchain/driver:driver_file_test", + "lexer": "//toolchain/lexer:lexer_file_test", + "lowering": "//toolchain/lowering:lowering_file_test", + "parser": "//toolchain/parser:parse_tree_file_test", + "semantics": "//toolchain/semantics:semantics_file_test", +} + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "dirs", + # We don't use `choices` because it seems to conflict with "*". + nargs="*", + default=TARGETS.keys(), + help="Optionally restrict directories to update. Defaults to all.", + ) + parsed_args = parser.parse_args() + + # Deduplicate and validate arguments. + dirs = set(parsed_args.dirs) + invalid_dirs = dirs.difference(TARGETS.keys()) + if invalid_dirs: + exit( + f"Invalid dirs: {', '.join(invalid_dirs)}; " + f"allowed dirs are {', '.join(TARGETS.keys())}." + ) + + # Build the targets together if there's more than one. Otherwise, we may as + # well build and run together. + if len(dirs) > 1: + subprocess.check_call( + ["bazel", "build", "-c", "opt"] + [TARGETS[d] for d in dirs] + ) + for d in dirs: + subprocess.check_call( + ["bazel", "run", "-c", "opt", TARGETS[d], "--", "--autoupdate"] + ) + + +if __name__ == "__main__": + main() diff --git a/toolchain/codegen/testdata/assembly/fail_target_triple.carbon b/toolchain/codegen/testdata/assembly/fail_target_triple.carbon index 57527c26b916..882720f049fb 100644 --- a/toolchain/codegen/testdata/assembly/fail_target_triple.carbon +++ b/toolchain/codegen/testdata/assembly/fail_target_triple.carbon @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // // ARGS: dump assembly --target_triple=x86_687-unknown-linux-gnu %s -// TODO: Support autoupdate with ARGS. +// No autoupdate because the message comes from LLVM. // NOAUTOUPDATE // CHECK:STDERR: ERROR: Invalid -target_triple:{{.*}} diff --git a/toolchain/codegen/testdata/objcode/basic.carbon b/toolchain/codegen/testdata/objcode/basic.carbon index 28bcd2585486..ba5c6231fd63 100644 --- a/toolchain/codegen/testdata/objcode/basic.carbon +++ b/toolchain/codegen/testdata/objcode/basic.carbon @@ -3,8 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // // ARGS: dump objcode --target_triple=x86_64-unknown-linux-gnu --output_file=%t %s -// TODO: Support autoupdate with ARGS. -// NOAUTOUPDATE +// AUTOUPDATE // CHECK:STDOUT: Success: Object file is generated! fn Main() -> i32 { return 0; } diff --git a/toolchain/codegen/testdata/objcode/fail_no_input_file.carbon b/toolchain/codegen/testdata/objcode/fail_no_input_file.carbon index 73957f6a3c09..db0479d40f34 100644 --- a/toolchain/codegen/testdata/objcode/fail_no_input_file.carbon +++ b/toolchain/codegen/testdata/objcode/fail_no_input_file.carbon @@ -3,8 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // // ARGS: dump objcode --output_file=%t --target_triple=x86_64-unknown-linux-gnu -// TODO: Support autoupdate with ARGS. -// NOAUTOUPDATE +// AUTOUPDATE // CHECK:STDERR: ERROR: No input file specified. fn Main() -> i32 { return 0; } diff --git a/toolchain/codegen/testdata/objcode/fail_no_output_file.carbon b/toolchain/codegen/testdata/objcode/fail_no_output_file.carbon index 95b6d8407edf..b0deeab576c3 100644 --- a/toolchain/codegen/testdata/objcode/fail_no_output_file.carbon +++ b/toolchain/codegen/testdata/objcode/fail_no_output_file.carbon @@ -3,8 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // // ARGS: dump objcode --target_triple=x86_64-unknown-linux-gnu %s -// TODO: Support autoupdate with ARGS. -// NOAUTOUPDATE +// AUTOUPDATE // CHECK:STDERR: ERROR: Must provide an output file. fn Main() -> i32 { return 0; } diff --git a/toolchain/codegen/testdata/objcode/fail_target_triple.carbon b/toolchain/codegen/testdata/objcode/fail_target_triple.carbon index df34fb3e99f7..a961c1a601c5 100644 --- a/toolchain/codegen/testdata/objcode/fail_target_triple.carbon +++ b/toolchain/codegen/testdata/objcode/fail_target_triple.carbon @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // // ARGS: dump objcode --target_triple=x86_684-unknown-linux-gnu --output_file=%t %s -// TODO: Support autoupdate with ARGS. +// No autoupdate because the message comes from LLVM. // NOAUTOUPDATE // CHECK:STDERR: ERROR: Invalid -target_triple:{{.*}} diff --git a/toolchain/driver/driver_file_test.cpp b/toolchain/driver/driver_file_test.cpp index 16f50adaea24..b388c524984f 100644 --- a/toolchain/driver/driver_file_test.cpp +++ b/toolchain/driver/driver_file_test.cpp @@ -17,6 +17,13 @@ class DriverFileTest : public DriverFileTestBase { auto GetDefaultArgs() -> llvm::SmallVector override { CARBON_FATAL() << "ARGS is always set in these tests"; } + + auto DoExtraCheckReplacements(std::string& check_line) -> void override { + // TODO: Disable token output, it's not interesting for these tests. + if (llvm::StringRef(check_line).starts_with("// CHECK:STDOUT: {")) { + check_line = "// CHECK:STDOUT: {{.*}}"; + } + } }; } // namespace diff --git a/toolchain/driver/driver_file_test_base.h b/toolchain/driver/driver_file_test_base.h index 7247c4a337bd..b6cf4de85059 100644 --- a/toolchain/driver/driver_file_test_base.h +++ b/toolchain/driver/driver_file_test_base.h @@ -23,17 +23,16 @@ class DriverFileTestBase : public FileTestBase { public: using FileTestBase::FileTestBase; - auto RunWithFiles(const llvm::SmallVector& test_args, - const llvm::SmallVector& test_files, - llvm::raw_pwrite_stream& stdout, - llvm::raw_pwrite_stream& stderr) -> bool override { + auto Run(const llvm::SmallVector& test_args, + const llvm::SmallVector& test_files, + llvm::raw_pwrite_stream& stdout, llvm::raw_pwrite_stream& stderr) + -> ErrorOr override { // Create the files in-memory. llvm::vfs::InMemoryFileSystem fs; for (const auto& test_file : test_files) { if (!fs.addFile(test_file.filename, /*ModificationTime=*/0, llvm::MemoryBuffer::getMemBuffer(test_file.content))) { - ADD_FAILURE() << "File is repeated: " << test_file.filename; - return false; + return ErrorBuilder() << "File is repeated: " << test_file.filename; } } diff --git a/toolchain/driver/testdata/fail_errors_sorted.carbon b/toolchain/driver/testdata/fail_errors_sorted.carbon index 4b54b287796c..e67f31246be6 100644 --- a/toolchain/driver/testdata/fail_errors_sorted.carbon +++ b/toolchain/driver/testdata/fail_errors_sorted.carbon @@ -4,9 +4,8 @@ // // ARGS: dump tokens %s // -// TODO: Support autoupdate with ARGS. -// NOAUTOUPDATE // TODO: Disable token output, it's not interesting for these tests. +// AUTOUPDATE // CHECK:STDOUT: [ // CHECK:STDOUT: {{.*}} // CHECK:STDOUT: {{.*}} diff --git a/toolchain/driver/testdata/fail_errors_streamed.carbon b/toolchain/driver/testdata/fail_errors_streamed.carbon index b2ea7e0baee7..7efbcfadf9f4 100644 --- a/toolchain/driver/testdata/fail_errors_streamed.carbon +++ b/toolchain/driver/testdata/fail_errors_streamed.carbon @@ -4,9 +4,8 @@ // // ARGS: --print-errors=streamed dump tokens %s // -// TODO: Support autoupdate with ARGS. -// NOAUTOUPDATE // TODO: Disable token output, it's not interesting for these tests. +// AUTOUPDATE // CHECK:STDOUT: [ // CHECK:STDOUT: {{.*}} // CHECK:STDOUT: {{.*}} @@ -30,11 +29,10 @@ fn run(String program) { return True; -var x = 3a; - -// CHECK:STDERR: fail_errors_streamed.carbon:[[@LINE-2]]:10: Invalid digit 'a' in decimal numeric literal. +// CHECK:STDERR: fail_errors_streamed.carbon:[[@LINE+6]]:10: Invalid digit 'a' in decimal numeric literal. // CHECK:STDERR: var x = 3a; // CHECK:STDERR: ^ -// CHECK:STDERR: fail_errors_streamed.carbon:[[@LINE-8]]:24: Closing symbol does not match most recent opening symbol. +// CHECK:STDERR: fail_errors_streamed.carbon:[[@LINE-6]]:24: Closing symbol does not match most recent opening symbol. // CHECK:STDERR: fn run(String program) { // CHECK:STDERR: ^ +var x = 3a; diff --git a/toolchain/lexer/BUILD b/toolchain/lexer/BUILD index 571af41b4fa8..aa6d804e62f5 100644 --- a/toolchain/lexer/BUILD +++ b/toolchain/lexer/BUILD @@ -238,6 +238,7 @@ file_test( tests = glob(["testdata/**/*.carbon"]), deps = [ "//toolchain/driver:driver_file_test_base", + "@com_googlesource_code_re2//:re2", "@llvm-project//llvm:Support", ], ) diff --git a/toolchain/lexer/autoupdate_testdata.py b/toolchain/lexer/autoupdate_testdata.py deleted file mode 100755 index 54488451b794..000000000000 --- a/toolchain/lexer/autoupdate_testdata.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python3 - -"""Updates the CHECK: lines in tests with an AUTOUPDATE line.""" - -__copyright__ = """ -Part of the Carbon Language project, under the Apache License v2.0 with LLVM -Exceptions. See /LICENSE for license information. -SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -""" - -import subprocess -import sys -from pathlib import Path - - -def main() -> None: - # Subprocess to the main script in order to avoid Python import behaviors. - this_py = Path(__file__).resolve() - autoupdate_py = this_py.parent.parent.parent.joinpath( - "testing", "scripts", "autoupdate_testdata_base.py" - ) - args = [ - str(autoupdate_py), - # Flags to configure for lexer testing. - "--tool=carbon", - "--autoupdate_arg=dump", - "--autoupdate_arg=tokens", - # Ignore the resulting column of EndOfFile because it's typically the - # end of the CHECK comment. - "--extra_check_replacement", - ".*'EndOfFile'", - r"column: (?:\d+)", - "column: {{[0-9]+}}", - # Ignore spaces that are used to columnize lines. - "--line_number_delta_prefix={{ *}}", - "--line_number_pattern=" - r"(?P line: )(?P *\d+)(?P,)", - "--testdata=toolchain/lexer/testdata", - ] + sys.argv[1:] - exit(subprocess.call(args)) - - -if __name__ == "__main__": - main() diff --git a/toolchain/lexer/lexer_file_test.cpp b/toolchain/lexer/lexer_file_test.cpp index a70f1090c814..0ef0720c4671 100644 --- a/toolchain/lexer/lexer_file_test.cpp +++ b/toolchain/lexer/lexer_file_test.cpp @@ -5,6 +5,7 @@ #include #include "llvm/ADT/SmallVector.h" +#include "re2/re2.h" #include "toolchain/driver/driver_file_test_base.h" namespace Carbon::Testing { @@ -12,11 +13,30 @@ namespace { class LexerFileTest : public DriverFileTestBase { public: - using DriverFileTestBase::DriverFileTestBase; + explicit LexerFileTest(std::filesystem::path path) + : DriverFileTestBase(std::move(path)), + end_of_file_re_((R"((EndOfFile.*column: )( *\d+))")) {} auto GetDefaultArgs() -> llvm::SmallVector override { return {"dump", "tokens", "%s"}; } + + auto GetLineNumberReplacement(llvm::ArrayRef /*filenames*/) + -> LineNumberReplacement override { + return {.has_file = false, + .pattern = R"(line: +(\d+))", + // The `{{{{` becomes `{{`. + .sub_for_formatv = "line: {{{{ *}}{0}"}; + } + + auto DoExtraCheckReplacements(std::string& check_line) -> void override { + // Ignore the resulting column of EndOfFile because it's often the end of + // the CHECK comment. + RE2::Replace(&check_line, end_of_file_re_, R"(\1{{ *\\d+}})"); + } + + private: + RE2 end_of_file_re_; }; } // namespace diff --git a/toolchain/lexer/testdata/basic_syntax.carbon b/toolchain/lexer/testdata/basic_syntax.carbon index 1572a308decd..8de3f625c723 100644 --- a/toolchain/lexer/testdata/basic_syntax.carbon +++ b/toolchain/lexer/testdata/basic_syntax.carbon @@ -18,6 +18,6 @@ fn run(String program) { // CHECK:STDOUT: { index: 9, kind: 'Semi', line: {{ *}}[[@LINE+1]], column: 14, indent: 3, spelling: ';', has_trailing_space: true }, return True; // CHECK:STDOUT: { index: 10, kind: 'CloseCurlyBrace', line: {{ *}}[[@LINE+3]], column: 1, indent: 1, spelling: '}', opening_token: 6, has_trailing_space: true }, -// CHECK:STDOUT: { index: 11, kind: 'EndOfFile', line: {{ *}}[[@LINE+2]], column: 2, indent: 1, spelling: '' }, +// CHECK:STDOUT: { index: 11, kind: 'EndOfFile', line: {{ *}}[[@LINE+2]], column: {{ *\d+}}, indent: 1, spelling: '' }, // CHECK:STDOUT: ] } diff --git a/toolchain/lexer/testdata/keywords.carbon b/toolchain/lexer/testdata/keywords.carbon index 54611b2fc2c8..118e99d950a6 100644 --- a/toolchain/lexer/testdata/keywords.carbon +++ b/toolchain/lexer/testdata/keywords.carbon @@ -130,6 +130,6 @@ where while // CHECK:STDOUT: { index: 61, kind: 'Identifier', line: {{ *}}[[@LINE+3]], column: 1, indent: 1, spelling: 'notakeyword', identifier: 0, has_trailing_space: true }, -// CHECK:STDOUT: { index: 62, kind: 'EndOfFile', line: {{ *}}[[@LINE+2]], column: {{[0-9]+}}, indent: 1, spelling: '' }, +// CHECK:STDOUT: { index: 62, kind: 'EndOfFile', line: {{ *}}[[@LINE+2]], column: {{ *\d+}}, indent: 1, spelling: '' }, // CHECK:STDOUT: ] notakeyword diff --git a/toolchain/lexer/testdata/printing_digit_padding.carbon b/toolchain/lexer/testdata/printing_digit_padding.carbon index 1948f48dbb77..43451c562e0b 100644 --- a/toolchain/lexer/testdata/printing_digit_padding.carbon +++ b/toolchain/lexer/testdata/printing_digit_padding.carbon @@ -12,6 +12,6 @@ // CHECK:STDOUT: { index: 1, kind: 'Semi', line: {{ *}}[[@LINE+5]], column: 9, indent: 9, spelling: ';' }, // CHECK:STDOUT: { index: 2, kind: 'Semi', line: {{ *}}[[@LINE+4]], column: 10, indent: 9, spelling: ';' }, // CHECK:STDOUT: { index: 3, kind: 'Semi', line: {{ *}}[[@LINE+3]], column: 11, indent: 9, spelling: ';', has_trailing_space: true }, - // CHECK:STDOUT: { index: 4, kind: 'EndOfFile', line: {{ *}}[[@LINE+2]], column: {{[0-9]+}}, indent: 9, spelling: '' }, + // CHECK:STDOUT: { index: 4, kind: 'EndOfFile', line: {{ *}}[[@LINE+2]], column: {{ *\d+}}, indent: 9, spelling: '' }, // CHECK:STDOUT: ] ;;; diff --git a/toolchain/lexer/testdata/printing_integer_literal.carbon b/toolchain/lexer/testdata/printing_integer_literal.carbon index 1719a78e2cc6..95f434d247bc 100644 --- a/toolchain/lexer/testdata/printing_integer_literal.carbon +++ b/toolchain/lexer/testdata/printing_integer_literal.carbon @@ -6,6 +6,6 @@ // CHECK:STDOUT: [ // CHECK:STDOUT: { index: 0, kind: 'IntegerLiteral', line: {{ *}}[[@LINE+3]], column: 1, indent: 1, spelling: '123', value: `123`, has_trailing_space: true }, -// CHECK:STDOUT: { index: 1, kind: 'EndOfFile', line: {{ *}}[[@LINE+2]], column: {{[0-9]+}}, indent: 1, spelling: '' }, +// CHECK:STDOUT: { index: 1, kind: 'EndOfFile', line: {{ *}}[[@LINE+2]], column: {{ *\d+}}, indent: 1, spelling: '' }, // CHECK:STDOUT: ] 123 diff --git a/toolchain/lexer/testdata/printing_real_literal.carbon b/toolchain/lexer/testdata/printing_real_literal.carbon index 6411f615ffa7..35b597e9e4b3 100644 --- a/toolchain/lexer/testdata/printing_real_literal.carbon +++ b/toolchain/lexer/testdata/printing_real_literal.carbon @@ -6,6 +6,6 @@ // CHECK:STDOUT: [ // CHECK:STDOUT: { index: 0, kind: 'RealLiteral', line: {{ *}}[[@LINE+3]], column: 1, indent: 1, spelling: '2.5', value: `25*10^-1`, has_trailing_space: true }, -// CHECK:STDOUT: { index: 1, kind: 'EndOfFile', line: {{ *}}[[@LINE+2]], column: {{[0-9]+}}, indent: 1, spelling: '' }, +// CHECK:STDOUT: { index: 1, kind: 'EndOfFile', line: {{ *}}[[@LINE+2]], column: {{ *\d+}}, indent: 1, spelling: '' }, // CHECK:STDOUT: ] 2.5 diff --git a/toolchain/lexer/testdata/printing_token.carbon b/toolchain/lexer/testdata/printing_token.carbon index 08dff3771a71..23d012c01410 100644 --- a/toolchain/lexer/testdata/printing_token.carbon +++ b/toolchain/lexer/testdata/printing_token.carbon @@ -6,6 +6,6 @@ // CHECK:STDOUT: [ // CHECK:STDOUT: { index: 0, kind: 'IntegerLiteral', line: {{ *}}[[@LINE+3]], column: 1, indent: 1, spelling: '0x9', value: `9`, has_trailing_space: true }, -// CHECK:STDOUT: { index: 1, kind: 'EndOfFile', line: {{ *}}[[@LINE+2]], column: {{[0-9]+}}, indent: 1, spelling: '' }, +// CHECK:STDOUT: { index: 1, kind: 'EndOfFile', line: {{ *}}[[@LINE+2]], column: {{ *\d+}}, indent: 1, spelling: '' }, // CHECK:STDOUT: ] 0x9 diff --git a/toolchain/lowering/autoupdate_testdata.py b/toolchain/lowering/autoupdate_testdata.py deleted file mode 100755 index ea3e2aa5119b..000000000000 --- a/toolchain/lowering/autoupdate_testdata.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python3 - -"""Updates the CHECK: lines in tests with an AUTOUPDATE line.""" - -__copyright__ = """ -Part of the Carbon Language project, under the Apache License v2.0 with LLVM -Exceptions. See /LICENSE for license information. -SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -""" - -import subprocess -import sys -from pathlib import Path - - -def main() -> None: - # Subprocess to the main script in order to avoid Python import behaviors. - this_py = Path(__file__).resolve() - autoupdate_py = this_py.parent.parent.parent.joinpath( - "testing", "scripts", "autoupdate_testdata_base.py" - ) - args = [ - str(autoupdate_py), - # Flags to configure for lowering testing. - "--tool=carbon", - "--autoupdate_arg=dump", - "--autoupdate_arg=llvm-ir", - "--testdata=toolchain/lowering/testdata", - ] + sys.argv[1:] - exit(subprocess.call(args)) - - -if __name__ == "__main__": - main() diff --git a/toolchain/parser/autoupdate_testdata.py b/toolchain/parser/autoupdate_testdata.py deleted file mode 100755 index f783a5dfb1bc..000000000000 --- a/toolchain/parser/autoupdate_testdata.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python3 - -"""Updates the CHECK: lines in tests with an AUTOUPDATE line.""" - -__copyright__ = """ -Part of the Carbon Language project, under the Apache License v2.0 with LLVM -Exceptions. See /LICENSE for license information. -SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -""" - -import subprocess -import sys -from pathlib import Path - - -def main() -> None: - # Subprocess to the main script in order to avoid Python import behaviors. - this_py = Path(__file__).resolve() - autoupdate_py = this_py.parent.parent.parent.joinpath( - "testing", "scripts", "autoupdate_testdata_base.py" - ) - args = [ - str(autoupdate_py), - # Flags to configure for parser testing. - "--tool=carbon", - "--autoupdate_arg=dump", - "--autoupdate_arg=parse-tree", - "--testdata=toolchain/parser/testdata", - ] + sys.argv[1:] - exit(subprocess.call(args)) - - -if __name__ == "__main__": - main() diff --git a/toolchain/semantics/autoupdate_testdata.py b/toolchain/semantics/autoupdate_testdata.py deleted file mode 100755 index c056fec76dc3..000000000000 --- a/toolchain/semantics/autoupdate_testdata.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python3 - -"""Updates the CHECK: lines in tests with an AUTOUPDATE line.""" - -__copyright__ = """ -Part of the Carbon Language project, under the Apache License v2.0 with LLVM -Exceptions. See /LICENSE for license information. -SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -""" - -import subprocess -import sys -from pathlib import Path - - -def main() -> None: - # Subprocess to the main script in order to avoid Python import behaviors. - this_py = Path(__file__).resolve() - autoupdate_py = this_py.parent.parent.parent.joinpath( - "testing", "scripts", "autoupdate_testdata_base.py" - ) - args = [ - str(autoupdate_py), - # Flags to configure for explorer testing. - "--tool=carbon", - "--autoupdate_arg=dump", - "--autoupdate_arg=semantics-ir", - "--testdata=toolchain/semantics/testdata", - ] + sys.argv[1:] - exit(subprocess.call(args)) - - -if __name__ == "__main__": - main() diff --git a/toolchain/semantics/testdata/basics/builtin_nodes.carbon b/toolchain/semantics/testdata/basics/builtin_nodes.carbon index b37c0b449125..dfba89002eb9 100644 --- a/toolchain/semantics/testdata/basics/builtin_nodes.carbon +++ b/toolchain/semantics/testdata/basics/builtin_nodes.carbon @@ -4,8 +4,7 @@ // // ARGS: dump semantics-ir --include_builtins %s // -// TODO: Support autoupdate with ARGS -// NOAUTOUPDATE +// AUTOUPDATE // CHECK:STDOUT: cross_reference_irs_size: 1 // CHECK:STDOUT: functions: [ // CHECK:STDOUT: ]