diff --git a/testing/file_test/README.md b/testing/file_test/README.md index ca9513cde650..7f39679c7959 100644 --- a/testing/file_test/README.md +++ b/testing/file_test/README.md @@ -80,6 +80,21 @@ The main test file and any split-files must have a `fail_` prefix if and only if they have an associated error. An exception is that the main test file may omit `fail_` when it contains split-files that have a `fail_` prefix. +## Content replacement + +Some keywords can be inserted for content: + +- ``` + [[@TEST_NAME]] + ``` + + Replaces with the test name, which is the filename with the extension and + any `fail_` or `todo_` prefixes removed. For split files, this is based on + the split filename. + +The `[[@` string is reserved for future replacements, but `[[` is allowed in +content (comment markers don't allow `[[`). + ## Comment markers Settings in files are provided in comments, similar to `FileCheck` syntax. @@ -106,7 +121,9 @@ Supported comment markers are: check line refers to any line in the test, all STDOUT check lines are placed at the end of the file instead of immediately after AUTOUPDATE. -- `// ARGS: ` +- ``` + // ARGS: + ``` Provides a space-separated list of arguments, which will be passed to RunWithFiles as test_args. These are intended for use by the command as @@ -131,7 +148,9 @@ Supported comment markers are: ARGS can be specified at most once. If not provided, the FileTestBase child is responsible for providing default arguments. -- `// SET-CHECK-SUBSET` +- ``` + // SET-CHECK-SUBSET + ``` By default, all lines of output must have a CHECK match. Adding this as a option sets it so that non-matching lines are ignored. All provided @@ -139,7 +158,9 @@ Supported comment markers are: SET-CHECK-SUBSET can be specified at most once. -- `// --- ` +- ``` + // --- + ``` By default, all file content is provided to the test as a single file in test_files. Using this marker allows the file to be split into multiple diff --git a/testing/file_test/file_test_base.cpp b/testing/file_test/file_test_base.cpp index 32627c6964c2..d6777b483e7b 100644 --- a/testing/file_test/file_test_base.cpp +++ b/testing/file_test/file_test_base.cpp @@ -443,7 +443,7 @@ struct SplitState { auto has_splits() const -> bool { return file_index > 0; } auto add_content(llvm::StringRef line) -> void { - content.append(line); + content.append(line.str()); content.append("\n"); } @@ -462,13 +462,63 @@ struct SplitState { int file_index = 0; }; +// Replaces the content keywords. +// +// TEST_NAME is the only content keyword at present, but we do validate that +// other names are reserved. +static auto ReplaceContentKeywords(llvm::StringRef filename, + std::string* content) -> ErrorOr { + static constexpr llvm::StringLiteral Prefix = "[[@"; + + auto keyword_pos = content->find(Prefix); + // Return early if not finding anything. + if (keyword_pos == std::string::npos) { + return Success(); + } + + // Construct the test name by getting the base name without the extension, + // then removing any "fail_" or "todo_" prefixes. + llvm::StringRef test_name = filename; + if (auto last_slash = test_name.rfind("/"); + last_slash != llvm::StringRef::npos) { + test_name = test_name.substr(last_slash + 1); + } + if (auto ext_dot = test_name.find("."); ext_dot != llvm::StringRef::npos) { + test_name = test_name.substr(0, ext_dot); + } + // Note this also handles `fail_todo_` and `todo_fail_`. + test_name.consume_front("todo_"); + test_name.consume_front("fail_"); + test_name.consume_front("todo_"); + + while (keyword_pos != std::string::npos) { + static constexpr llvm::StringLiteral TestName = "[[@TEST_NAME]]"; + auto keyword = llvm::StringRef(*content).substr(keyword_pos); + if (keyword.starts_with(TestName)) { + content->replace(keyword_pos, TestName.size(), test_name); + keyword_pos += test_name.size(); + } else if (keyword.starts_with("[[@LINE")) { + // Just move past the prefix to find the next one. + keyword_pos += Prefix.size(); + } else { + return ErrorBuilder() + << "Unexpected use of `[[@` at `" << keyword.substr(0, 5) << "`"; + } + keyword_pos = content->find(Prefix, keyword_pos); + } + return Success(); +} + // Adds a file. Used for both split and unsplit test files. static auto AddTestFile(llvm::StringRef filename, std::string* content, llvm::SmallVector* test_files) - -> void { + -> ErrorOr { + CARBON_RETURN_IF_ERROR(ReplaceContentKeywords(filename, content)); + test_files->push_back( {.filename = filename.str(), .content = std::move(*content)}); content->clear(); + return Success(); } // Process file split ("---") lines when found. Returns true if the line is @@ -500,7 +550,8 @@ static auto TryConsumeSplit( // On a file split, add the previous file, then start a new one. if (split->has_splits()) { - AddTestFile(split->filename, &split->content, test_files); + CARBON_RETURN_IF_ERROR( + AddTestFile(split->filename, &split->content, test_files)); } else { split->content.clear(); if (split->found_code_pre_split) { @@ -646,14 +697,14 @@ static auto TransformExpectation(int line_index, llvm::StringRef in) // Once all content is processed, do any remaining split processing. static auto FinishSplit(llvm::StringRef test_name, SplitState* split, llvm::SmallVector* test_files) - -> void { + -> ErrorOr { if (split->has_splits()) { - AddTestFile(split->filename, &split->content, test_files); + return AddTestFile(split->filename, &split->content, test_files); } else { // If no file splitting happened, use the main file as the test file. // There will always be a `/` unless tests are in the repo root. - AddTestFile(test_name.drop_front(test_name.rfind("/") + 1), &split->content, - test_files); + return AddTestFile(test_name.drop_front(test_name.rfind("/") + 1), + &split->content, test_files); } } @@ -826,7 +877,7 @@ auto FileTestBase::ProcessTestFile(TestContext& context) -> ErrorOr { } context.has_splits = split.has_splits(); - FinishSplit(test_name_, &split, &context.test_files); + CARBON_RETURN_IF_ERROR(FinishSplit(test_name_, &split, &context.test_files)); // Assume there is always a suffix `\n` in output. if (!context.expected_stdout.empty()) { diff --git a/testing/file_test/file_test_base_test.cpp b/testing/file_test/file_test_base_test.cpp index 3934b1e1a1dc..3e4ef6887934 100644 --- a/testing/file_test/file_test_base_test.cpp +++ b/testing/file_test/file_test_base_test.cpp @@ -9,6 +9,7 @@ #include "common/ostream.h" #include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringSwitch.h" #include "llvm/Support/FormatVariadic.h" namespace Carbon::Testing { @@ -21,121 +22,8 @@ class FileTestBaseTest : public FileTestBase { auto Run(const llvm::SmallVector& test_args, llvm::vfs::InMemoryFileSystem& fs, llvm::raw_pwrite_stream& stdout, - llvm::raw_pwrite_stream& stderr) -> ErrorOr override { - llvm::ArrayRef args = test_args; + llvm::raw_pwrite_stream& stderr) -> ErrorOr override; - llvm::ListSeparator sep; - stdout << args.size() << " args: "; - for (auto arg : args) { - stdout << sep << "`" << arg << "`"; - } - stdout << "\n"; - - auto filename = std::filesystem::path(test_name().str()).filename(); - if (filename == "args.carbon") { - // 'args.carbon' has custom arguments, so don't do regular argument - // validation for it. - return {{.success = true}}; - } - - if (args.empty() || args.front() != "default_args") { - return ErrorBuilder() << "missing `default_args` argument"; - } - args = args.drop_front(); - - for (auto arg : args) { - if (!fs.exists(arg)) { - return ErrorBuilder() << "Missing file: " << arg; - } - } - - if (filename == "example.carbon") { - int delta_line = 10; - stdout << "something\n" - << "\n" - << "example.carbon:" << delta_line + 1 << ": Line delta\n" - << "example.carbon:" << delta_line << ": Negative line delta\n" - << "+*[]{}\n" - << "Foo baz\n"; - return {{.success = true}}; - } else if (filename == "fail_example.carbon") { - stderr << "Oops\n"; - return {{.success = false}}; - } else if (filename == "two_files.carbon" || - filename == "not_split.carbon") { - for (auto arg : args) { - // Describe file contents to stdout to validate splitting. - auto file = fs.getBufferForFile(arg, /*FileSize=*/-1, - /*RequiresNullTerminator=*/false); - if (file.getError()) { - return Error(file.getError().message()); - } - llvm::StringRef content = file.get()->getBuffer(); - stdout << arg << ":1: starts with \""; - stdout.write_escaped(content.take_front(40)); - stdout << "\", length " << content.count('\n') << " lines\n"; - } - return {{.success = true}}; - } else if (filename == "alternating_files.carbon") { - stdout << "unattached message 1\n" - << "a.carbon:2: message 2\n" - << "b.carbon:5: message 3\n" - << "a.carbon:2: message 4\n" - << "b.carbon:5: message 5\n" - << "unattached message 6\n"; - stderr << "unattached message 1\n" - << "a.carbon:2: message 2\n" - << "b.carbon:5: message 3\n" - << "a.carbon:2: message 4\n" - << "b.carbon:5: message 5\n" - << "unattached message 6\n"; - return {{.success = true}}; - } else if (filename == "unattached_multi_file.carbon") { - stdout << "unattached message 1\n" - << "unattached message 2\n"; - stderr << "unattached message 3\n" - << "unattached message 4\n"; - return {{.success = true}}; - } else if (filename == "file_only_re_one_file.carbon") { - stdout << "unattached message 1\n" - << "file: file_only_re_one_file.carbon\n" - << "line: 1\n" - << "unattached message 2\n"; - return {{.success = true}}; - } else if (filename == "file_only_re_multi_file.carbon") { - int msg_count = 0; - stdout << "unattached message " << ++msg_count << "\n" - << "file: a.carbon\n" - << "unattached message " << ++msg_count << "\n" - << "line: 3: attached message " << ++msg_count << "\n" - << "unattached message " << ++msg_count << "\n" - << "line: 8: late message " << ++msg_count << "\n" - << "unattached message " << ++msg_count << "\n" - << "file: b.carbon\n" - << "line: 2: attached message " << ++msg_count << "\n" - << "unattached message " << ++msg_count << "\n" - << "line: 7: late message " << ++msg_count << "\n" - << "unattached message " << ++msg_count << "\n"; - return {{.success = true}}; - } else if (filename == "fail_multi_success_overall_fail.carbon") { - RunResult result = {.success = false}; - result.per_file_success.push_back({"a.carbon", true}); - result.per_file_success.push_back({"b.carbon", true}); - return result; - } else if (filename == "multi_success.carbon") { - RunResult result = {.success = true}; - result.per_file_success.push_back({"a.carbon", true}); - result.per_file_success.push_back({"b.carbon", true}); - return result; - } else if (filename == "multi_success_and_fail.carbon") { - RunResult result = {.success = false}; - result.per_file_success.push_back({"a.carbon", true}); - result.per_file_success.push_back({"fail_b.carbon", false}); - return result; - } else { - return ErrorBuilder() << "Unexpected file: " << filename; - } - } auto GetArgReplacements() -> llvm::StringMap override { return {{"replacement", "replaced"}}; } @@ -163,6 +51,208 @@ class FileTestBaseTest : public FileTestBase { } }; +// Prints arguments so that they can be validated in tests. +static auto PrintArgs(llvm::ArrayRef args, + llvm::raw_pwrite_stream& stdout) -> void { + llvm::ListSeparator sep; + stdout << args.size() << " args: "; + for (auto arg : args) { + stdout << sep << "`" << arg << "`"; + } + stdout << "\n"; +} + +// Verifies arguments are well-structured, and returns the files in them. +static auto GetFilesFromArgs(llvm::ArrayRef args, + llvm::vfs::InMemoryFileSystem& fs) + -> ErrorOr> { + if (args.empty() || args.front() != "default_args") { + return ErrorBuilder() << "missing `default_args` argument"; + } + args = args.drop_front(); + + for (auto arg : args) { + if (!fs.exists(arg)) { + return ErrorBuilder() << "Missing file: " << arg; + } + } + return args; +} + +// Parameters used to by individual test handlers, for easy value forwarding. +struct TestParams { + // These are the arguments to `Run()`. + llvm::vfs::InMemoryFileSystem& fs; + llvm::raw_pwrite_stream& stdout; + llvm::raw_pwrite_stream& stderr; + + // This is assigned after construction. + llvm::ArrayRef files; +}; + +// Does printing and returns expected results for alternating_files.carbon. +static auto TestAlternatingFiles(TestParams& params) + -> ErrorOr { + params.stdout << "unattached message 1\n" + << "a.carbon:2: message 2\n" + << "b.carbon:5: message 3\n" + << "a.carbon:2: message 4\n" + << "b.carbon:5: message 5\n" + << "unattached message 6\n"; + params.stderr << "unattached message 1\n" + << "a.carbon:2: message 2\n" + << "b.carbon:5: message 3\n" + << "a.carbon:2: message 4\n" + << "b.carbon:5: message 5\n" + << "unattached message 6\n"; + return {{.success = true}}; +} + +// Does printing and returns expected results for example.carbon. +static auto TestExample(TestParams& params) + -> ErrorOr { + int delta_line = 10; + params.stdout << "something\n" + << "\n" + << "example.carbon:" << delta_line + 1 << ": Line delta\n" + << "example.carbon:" << delta_line << ": Negative line delta\n" + << "+*[]{}\n" + << "Foo baz\n"; + return {{.success = true}}; +} + +// Does printing and returns expected results for fail_example.carbon. +static auto TestFailExample(TestParams& params) + -> ErrorOr { + params.stderr << "Oops\n"; + return {{.success = false}}; +} + +// Does printing and returns expected results for +// file_only_re_multi_file.carbon. +static auto TestFileOnlyREMultiFile(TestParams& params) + -> ErrorOr { + int msg_count = 0; + params.stdout << "unattached message " << ++msg_count << "\n" + << "file: a.carbon\n" + << "unattached message " << ++msg_count << "\n" + << "line: 3: attached message " << ++msg_count << "\n" + << "unattached message " << ++msg_count << "\n" + << "line: 8: late message " << ++msg_count << "\n" + << "unattached message " << ++msg_count << "\n" + << "file: b.carbon\n" + << "line: 2: attached message " << ++msg_count << "\n" + << "unattached message " << ++msg_count << "\n" + << "line: 7: late message " << ++msg_count << "\n" + << "unattached message " << ++msg_count << "\n"; + return {{.success = true}}; +} + +// Does printing and returns expected results for file_only_re_one_file.carbon. +static auto TestFileOnlyREOneFile(TestParams& params) + -> ErrorOr { + params.stdout << "unattached message 1\n" + << "file: file_only_re_one_file.carbon\n" + << "line: 1\n" + << "unattached message 2\n"; + return {{.success = true}}; +} + +// Does printing and returns expected results for unattached_multi_file.carbon. +static auto TestUnattachedMultiFile(TestParams& params) + -> ErrorOr { + params.stdout << "unattached message 1\n" + << "unattached message 2\n"; + params.stderr << "unattached message 3\n" + << "unattached message 4\n"; + return {{.success = true}}; +} + +// Does printing and returns expected results for: +// - fail_multi_success_overall_fail.carbon +// - multi_success.carbon +// - multi_success_and_fail.carbon +// +// Parameters indicate overall and per-file success. +static auto HandleMultiSuccessTests(bool overall, bool a, bool b) + -> ErrorOr { + FileTestBaseTest::RunResult result = {.success = overall}; + result.per_file_success.push_back({a ? "a.carbon" : "fail_a.carbon", a}); + result.per_file_success.push_back({b ? "b.carbon" : "fail_b.carbon", b}); + return result; +} + +// Echoes back non-comment file content. Used for default file handling. +static auto EchoFileContent(TestParams& params) + -> ErrorOr { + // By default, echo non-comment content of files back. + for (auto test_file : params.files) { + // Describe file contents to stdout to validate splitting. + auto file = params.fs.getBufferForFile(test_file, /*FileSize=*/-1, + /*RequiresNullTerminator=*/false); + if (file.getError()) { + return Error(file.getError().message()); + } + llvm::StringRef buffer = file.get()->getBuffer(); + for (int line_number = 1; !buffer.empty(); ++line_number) { + auto [line, remainder] = buffer.split('\n'); + if (!line.empty() && !line.starts_with("//")) { + params.stdout << test_file << ":" << line_number << ": " << line + << "\n"; + } + buffer = remainder; + } + } + return {{.success = true}}; +} + +auto FileTestBaseTest::Run(const llvm::SmallVector& test_args, + llvm::vfs::InMemoryFileSystem& fs, + llvm::raw_pwrite_stream& stdout, + llvm::raw_pwrite_stream& stderr) + -> ErrorOr { + PrintArgs(test_args, stdout); + + auto filename = std::filesystem::path(test_name().str()).filename(); + if (filename == "args.carbon") { + // 'args.carbon' has custom arguments, so don't do regular argument + // validation for it. + return {{.success = true}}; + } + + // Choose the test function based on filename. + auto test_fn = + llvm::StringSwitch(TestParams&)>>( + filename.string()) + .Case("alternating_files.carbon", &TestAlternatingFiles) + .Case("example.carbon", &TestExample) + .Case("fail_example.carbon", &TestFailExample) + .Case("file_only_re_one_file.carbon", &TestFileOnlyREOneFile) + .Case("file_only_re_multi_file.carbon", &TestFileOnlyREMultiFile) + .Case("unattached_multi_file.carbon", &TestUnattachedMultiFile) + .Case("fail_multi_success_overall_fail.carbon", + [&](TestParams&) { + return HandleMultiSuccessTests(/*overall=*/false, /*a=*/true, + /*b=*/true); + }) + .Case("multi_success.carbon", + [&](TestParams&) { + return HandleMultiSuccessTests(/*overall=*/true, /*a=*/true, + /*b=*/true); + }) + .Case("multi_success_and_fail.carbon", + [&](TestParams&) { + return HandleMultiSuccessTests(/*overall=*/false, /*a=*/true, + /*b=*/false); + }) + .Default(&EchoFileContent); + + // Call the appropriate test function for the file. + TestParams params = {.fs = fs, .stdout = stdout, .stderr = stderr}; + CARBON_ASSIGN_OR_RETURN(params.files, GetFilesFromArgs(test_args, fs)); + return test_fn(params); +} + } // namespace CARBON_FILE_TEST_FACTORY(FileTestBaseTest) diff --git a/testing/file_test/testdata/not_split.carbon b/testing/file_test/testdata/not_split.carbon index 627c881d0691..21a0f8040112 100644 --- a/testing/file_test/testdata/not_split.carbon +++ b/testing/file_test/testdata/not_split.carbon @@ -8,4 +8,6 @@ // TIP: To dump output, run: // TIP: bazel run //testing/file_test:file_test_base_test -- --dump_output --file_tests=testing/file_test/testdata/not_split.carbon // CHECK:STDOUT: 2 args: `default_args`, `not_split.carbon` -// CHECK:STDOUT: not_split.carbon:[[@LINE-10]]: starts with "// Part of the Carbon Language project, ", length 11 lines + +not split +// CHECK:STDOUT: not_split.carbon:[[@LINE-1]]: not split diff --git a/testing/file_test/testdata/replace_content.carbon b/testing/file_test/testdata/replace_content.carbon new file mode 100644 index 000000000000..11f96c5397c9 --- /dev/null +++ b/testing/file_test/testdata/replace_content.carbon @@ -0,0 +1,13 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// AUTOUPDATE +// TIP: To test this file alone, run: +// TIP: bazel test //testing/file_test:file_test_base_test --test_arg=--file_tests=testing/file_test/testdata/replace_content.carbon +// TIP: To dump output, run: +// TIP: bazel run //testing/file_test:file_test_base_test -- --dump_output --file_tests=testing/file_test/testdata/replace_content.carbon +// CHECK:STDOUT: 2 args: `default_args`, `replace_content.carbon` + +library "[[@TEST_NAME]]"; +// CHECK:STDOUT: replace_content.carbon:[[@LINE-1]]: library "replace_content"; diff --git a/testing/file_test/testdata/replace_split_content.carbon b/testing/file_test/testdata/replace_split_content.carbon new file mode 100644 index 000000000000..f13fe92cb9b9 --- /dev/null +++ b/testing/file_test/testdata/replace_split_content.carbon @@ -0,0 +1,35 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// AUTOUPDATE +// TIP: To test this file alone, run: +// TIP: bazel test //testing/file_test:file_test_base_test --test_arg=--file_tests=testing/file_test/testdata/replace_split_content.carbon +// TIP: To dump output, run: +// TIP: bazel run //testing/file_test:file_test_base_test -- --dump_output --file_tests=testing/file_test/testdata/replace_split_content.carbon +// CHECK:STDOUT: 6 args: `default_args`, `a.carbon`, `b.impl.carbon`, `todo_c.carbon`, `todo_fail_d.carbon`, `triplicate.carbon` + +// --- a.carbon + +library "[[@TEST_NAME]]"; +// CHECK:STDOUT: a.carbon:[[@LINE-1]]: library "a"; + +// --- b.impl.carbon + +library "[[@TEST_NAME]]"; +// CHECK:STDOUT: b.impl.carbon:[[@LINE-1]]: library "b"; + +// --- todo_c.carbon + +library "[[@TEST_NAME]]"; +// CHECK:STDOUT: todo_c.carbon:[[@LINE-1]]: library "c"; + +// --- todo_fail_d.carbon + +library "[[@TEST_NAME]]"; +// CHECK:STDOUT: todo_fail_d.carbon:[[@LINE-1]]: library "d"; + +// --- triplicate.carbon + +[[@TEST_NAME]][[@TEST_NAME]][[@TEST_NAME]] +// CHECK:STDOUT: triplicate.carbon:[[@LINE-1]]: triplicatetriplicatetriplicate diff --git a/testing/file_test/testdata/two_files.carbon b/testing/file_test/testdata/two_files.carbon index b3399ffb2632..2f6cad4e88f3 100644 --- a/testing/file_test/testdata/two_files.carbon +++ b/testing/file_test/testdata/two_files.carbon @@ -11,8 +11,9 @@ // --- a.carbon aaa -// CHECK:STDOUT: a.carbon:[[@LINE-1]]: starts with "aaa\n// CHECK:STDOUT: a.carbon:{{\[\[}}@LINE-1]", length 3 lines +// CHECK:STDOUT: a.carbon:[[@LINE-1]]: aaa + // --- b.carbon bbb -// CHECK:STDOUT: b.carbon:[[@LINE-1]]: starts with "bbb\n// CHECK:STDOUT: b.carbon:{{\[\[}}@LINE-1]", length 2 lines +// CHECK:STDOUT: b.carbon:[[@LINE-1]]: bbb