diff --git a/bazel/testing/lit_autoupdate_base.py b/bazel/testing/lit_autoupdate_base.py index 013b656f581e..b067c2916cb1 100755 --- a/bazel/testing/lit_autoupdate_base.py +++ b/bazel/testing/lit_autoupdate_base.py @@ -111,7 +111,8 @@ def parse_args() -> ParsedArgs: parser.add_argument( "--lit_run", metavar="COMMAND", - required=True, + default=[], + required=False, action="append", help="RUN lines to set.", ) @@ -294,6 +295,7 @@ def replace_all(s: str, replacements: List[Tuple[str, str]]) -> str: def get_matchable_test_output( autoupdate_args: List[str], + for_lit: bool, extra_check_replacements: List[Tuple[Pattern, Pattern, str]], tool: str, bazel_runfiles: Pattern, @@ -311,21 +313,17 @@ def get_matchable_test_output( encoding="utf-8", ).stdout - # `lit` uses full paths to the test file, so use a regex to ignore paths - # when used. - out = replace_all( - out, - [ - ("{{", "{{[{][{]}}"), - ("[[", "{{[[][[]}}"), - # TODO: Maybe revisit and see if lit can be convinced to give a - # root-relative path. - (test, f"{{{{.*}}}}/{test}"), - ], - ) - # Replacing runfiles is a more complex replacement. - # We have some things show up under runfiles; this removes them. - out = bazel_runfiles.sub("{{.*}}/", out) + # Escape things that mirror FileCheck special characters. + out = out.replace("{{", "{{[{][{]}}") + out = out.replace("[[", "{{[[][[]}}") + if for_lit: + # `lit` uses full paths to the test file, so use a regex to ignore paths + # when used. + out = out.replace(test, f"{{{{.*}}}}/{test}") + out = bazel_runfiles.sub("{{.*}}/", out) + else: + # When not using `lit`, the runfiles path is removed. + out = bazel_runfiles.sub("", out) out_lines = out.splitlines() for i, line in enumerate(out_lines): @@ -412,6 +410,7 @@ def update_check( # Determine the merged output lines. out_lines = get_matchable_test_output( parsed_args.autoupdate_args, + bool(parsed_args.lit_run), parsed_args.extra_check_replacements, parsed_args.tool, bazel_runfiles, diff --git a/bazel/testing/merge_output.py b/bazel/testing/merge_output.py index d5520fab7a0f..5be6012efd94 100644 --- a/bazel/testing/merge_output.py +++ b/bazel/testing/merge_output.py @@ -8,7 +8,15 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception import subprocess import sys -import textwrap + + +def _print(output: str, label: str) -> None: + if output: + for line in output.splitlines(): + if line: + print(f"{label}: {line}") + else: + print(f"{label}:") def main() -> None: @@ -18,10 +26,9 @@ def main() -> None: stderr=subprocess.PIPE, encoding="utf-8", ) - if p.stdout: - print(textwrap.indent(p.stdout, "STDOUT: "), end="") - if p.stderr: - print(textwrap.indent(p.stderr, "STDERR: "), end="") + # The `lambda line` forces prefixes on empty lines. + _print(p.stdout, "STDOUT") + _print(p.stderr, "STDERR") exit(p.returncode) diff --git a/explorer/testdata/string/fail_newline.carbon b/explorer/testdata/string/fail_newline.carbon index 53a1066399fc..66c00629db3d 100644 --- a/explorer/testdata/string/fail_newline.carbon +++ b/explorer/testdata/string/fail_newline.carbon @@ -10,7 +10,7 @@ package ExplorerTest api; fn Main() -> i32 { // CHECK:STDERR: SYNTAX ERROR: {{.*}}/explorer/testdata/string/fail_newline.carbon:[[@LINE+2]]: missing closing quote in single-line string: "new - // CHECK-EMPTY: + // CHECK:STDERR: Print("new line"); return 0; diff --git a/explorer/testdata/string/fail_raw_more_hash_tags_on_left.carbon b/explorer/testdata/string/fail_raw_more_hash_tags_on_left.carbon index 3cc3837fea0f..1e0023af26b5 100644 --- a/explorer/testdata/string/fail_raw_more_hash_tags_on_left.carbon +++ b/explorer/testdata/string/fail_raw_more_hash_tags_on_left.carbon @@ -10,7 +10,7 @@ package ExplorerTest api; fn CompareStr(s: String) -> i32 { // CHECK:STDERR: SYNTAX ERROR: {{.*}}/explorer/testdata/string/fail_raw_more_hash_tags_on_left.carbon:[[@LINE+2]]: missing closing quote in single-line string: ##"str"#) { - // CHECK-EMPTY: + // CHECK:STDERR: if (s == ##"str"#) { return 0; } diff --git a/testing/file_test/BUILD b/testing/file_test/BUILD new file mode 100644 index 000000000000..c6ec62b9ee4b --- /dev/null +++ b/testing/file_test/BUILD @@ -0,0 +1,29 @@ +# Part of the Carbon Language project, under the Apache License v2.0 with LLVM +# Exceptions. See /LICENSE for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +load("rules.bzl", "file_test") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "file_test_base", + srcs = ["file_test_base.cpp"], + hdrs = ["file_test_base.h"], + deps = [ + "//common:check", + "@com_google_googletest//:gtest", + "@llvm-project//llvm:Support", + ], +) + +file_test( + name = "file_test", + srcs = ["file_test_base_test.cpp"], + tests = ["example.carbon"], + deps = [ + ":file_test_base", + "@com_google_googletest//:gtest", + "@llvm-project//llvm:Support", + ], +) diff --git a/testing/file_test/example.carbon b/testing/file_test/example.carbon new file mode 100644 index 000000000000..1f0301213f5b --- /dev/null +++ b/testing/file_test/example.carbon @@ -0,0 +1,10 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// CHECK:STDOUT: something +// CHECK:STDOUT: +// CHECK:STDOUT: [[@LINE+1]]: Line delta +// CHECK:STDOUT: [[@LINE-1]]: Negative line delta +// CHECK:STDOUT: +*[]{} +// CHECK:STDOUT: F{{.+}}z diff --git a/testing/file_test/fail_example.carbon b/testing/file_test/fail_example.carbon new file mode 100644 index 000000000000..2516b5d814cf --- /dev/null +++ b/testing/file_test/fail_example.carbon @@ -0,0 +1,5 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// CHECK:STDERR: Oops diff --git a/testing/file_test/file_test_base.cpp b/testing/file_test/file_test_base.cpp new file mode 100644 index 000000000000..4b9645f5fbf1 --- /dev/null +++ b/testing/file_test/file_test_base.cpp @@ -0,0 +1,223 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "testing/file_test/file_test_base.h" + +#include + +#include "common/check.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Support/InitLLVM.h" + +static std::string* subset_target = nullptr; + +namespace Carbon::Testing { + +using ::testing::Eq; + +void FileTestBase::RegisterTests( + const char* fixture_label, const std::vector& paths, + std::function factory) { + // Use RegisterTest instead of INSTANTIATE_TEST_CASE_P because of ordering + // issues between container initialization and test instantiation by + // InitGoogleTest. + for (auto path : paths) { + testing::RegisterTest(fixture_label, path.data(), nullptr, path.data(), + __FILE__, __LINE__, [=]() { return factory(path); }); + } +} + +// Splits outputs to string_view because gtest handles string_view by default. +static auto SplitOutput(llvm::StringRef output) + -> std::vector { + if (output.empty()) { + return {}; + } + llvm::SmallVector lines; + llvm::StringRef(output).split(lines, "\n"); + return std::vector(lines.begin(), lines.end()); +} + +// Runs a test and compares output. This keeps output split by line so that +// issues are a little easier to identify by the different line. +auto FileTestBase::TestBody() -> void { + llvm::errs() << "\nTo test this file alone, run:\n bazel test " + << *subset_target << " --test_arg=" << path() << "\n\n"; + + // Load expected output. + std::vector> expected_stdout; + std::vector> expected_stderr; + std::ifstream file_content(path_.str()); + int line_index = 0; + std::string line_str; + while (std::getline(file_content, line_str)) { + ++line_index; + llvm::StringRef line = line_str; + line = line.ltrim(); + if (!line.consume_front("// CHECK")) { + continue; + } + if (line.consume_front(":STDOUT:")) { + expected_stdout.push_back(TransformExpectation(line_index, line)); + } else if (line.consume_front(":STDERR:")) { + expected_stderr.push_back(TransformExpectation(line_index, line)); + } else { + FAIL() << "Unexpected CHECK in input: " << line_str; + } + } + + // Assume there is always a suffix `\n` in output. + if (!expected_stdout.empty()) { + expected_stdout.push_back(testing::StrEq("")); + } + if (!expected_stderr.empty()) { + expected_stderr.push_back(testing::StrEq("")); + } + + // Capture trace streaming, but only when in debug mode. + std::string stdout; + std::string stderr; + llvm::raw_string_ostream stdout_ostream(stdout); + llvm::raw_string_ostream stderr_ostream(stderr); + bool run_succeeded = RunOverFile(stdout_ostream, stderr_ostream); + if (HasFailure()) { + return; + } + EXPECT_THAT(!filename().starts_with("fail_"), Eq(run_succeeded)) + << "Tests should be prefixed with `fail_` if and only if running them " + "is expected to fail."; + + // Check results. + EXPECT_THAT(SplitOutput(stdout), ElementsAreArray(expected_stdout)); + EXPECT_THAT(SplitOutput(stderr), ElementsAreArray(expected_stderr)); +} + +auto FileTestBase::TransformExpectation(int line_index, llvm::StringRef in) + -> testing::Matcher { + if (in.empty()) { + return testing::StrEq(""); + } + CARBON_CHECK(in[0] == ' ') << "Malformated input: " << in; + std::string str = in.substr(1).str(); + for (int pos = 0; pos < static_cast(str.size());) { + switch (str[pos]) { + case '(': + case ')': + case ']': + case '}': + case '.': + case '^': + case '$': + case '*': + case '+': + case '?': + case '|': + case '\\': { + // Escape regex characters. + str.insert(pos, "\\"); + pos += 2; + break; + } + case '[': { + llvm::StringRef line_keyword_cursor = llvm::StringRef(str).substr(pos); + if (line_keyword_cursor.consume_front("[[")) { + static constexpr llvm::StringLiteral LineKeyword = "@LINE"; + if (line_keyword_cursor.consume_front(LineKeyword)) { + // Allow + or - here; consumeInteger handles -. + line_keyword_cursor.consume_front("+"); + int offset; + // consumeInteger returns true for errors, not false. + CARBON_CHECK(!line_keyword_cursor.consumeInteger(10, offset) && + line_keyword_cursor.consume_front("]]")) + << "Unexpected @LINE offset at `" + << line_keyword_cursor.substr(0, 5) << "` in: " << in; + std::string int_str = llvm::Twine(line_index + offset).str(); + int remove_len = (line_keyword_cursor.data() - str.data()) - pos; + str.replace(pos, remove_len, int_str); + pos += int_str.size(); + } else { + CARBON_FATAL() << "Unexpected [[, should be {{\\[\\[}} at `" + << line_keyword_cursor.substr(0, 5) + << "` in: " << in; + } + } else { + // Escape the `[`. + str.insert(pos, "\\"); + pos += 2; + } + break; + } + case '{': { + if (pos + 1 == static_cast(str.size()) || str[pos + 1] != '{') { + // Single `{`, escape it. + str.insert(pos, "\\"); + pos += 2; + } else { + // Replace the `{{...}}` regex syntax with standard `(...)` syntax. + str.replace(pos, 2, "("); + for (++pos; pos < static_cast(str.size() - 1); ++pos) { + if (str[pos] == '}' && str[pos + 1] == '}') { + str.replace(pos, 2, ")"); + ++pos; + break; + } + } + } + break; + } + default: { + ++pos; + } + } + } + + return testing::MatchesRegex(str); +} + +auto FileTestBase::filename() -> llvm::StringRef { + auto last_slash = path_.rfind("/"); + if (last_slash == llvm::StringRef::npos) { + return path_; + } else { + return path_.substr(last_slash + 1); + } +} + +} // namespace Carbon::Testing + +// Returns the name of the subset target. +static auto GetSubsetTarget() -> std::string { + char* name = getenv("TEST_TARGET"); + if (name == nullptr) { + return ""; + } + + if (llvm::StringRef(name).ends_with(".subset")) { + return name; + } else { + return std::string(name) + ".subset"; + } +} + +auto main(int argc, char** argv) -> int { + testing::InitGoogleTest(&argc, argv); + llvm::setBugReportMsg( + "Please report issues to " + "https://github.com/carbon-language/carbon-lang/issues and include the " + "crash backtrace.\n"); + llvm::InitLLVM init_llvm(argc, argv); + + if (argc < 2) { + llvm::errs() << "At least one test file must be provided.\n"; + return EXIT_FAILURE; + } + + std::string subset_target_storage = GetSubsetTarget(); + ::subset_target = &subset_target_storage; + + std::vector paths(argv + 1, argv + argc); + Carbon::Testing::RegisterFileTests(paths); + + return RUN_ALL_TESTS(); +} diff --git a/testing/file_test/file_test_base.h b/testing/file_test/file_test_base.h new file mode 100644 index 000000000000..7177504c7681 --- /dev/null +++ b/testing/file_test/file_test_base.h @@ -0,0 +1,73 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef CARBON_TESTING_FILE_TEST_FILE_TEST_BASE_H_ +#define CARBON_TESTING_FILE_TEST_FILE_TEST_BASE_H_ + +#include +#include + +#include +#include + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/raw_ostream.h" + +namespace Carbon::Testing { + +// A framework for testing files. Children implement `RegisterTestFiles` with +// calls to `RegisterTests` using a factory that constructs the child. +// `RunOverFile` must also be implemented and will be called as part of +// individual test executions. This framework includes a `main` implementation, +// so users must not provide one. +// +// Tests should have CHECK lines similar to `FileCheck` syntax: +// https://llvm.org/docs/CommandGuide/FileCheck.html +// +// Special nuances are that stdout and stderr will look like `// CHECK:STDOUT: +// ...` and `// CHECK:STDERR: ...` respectively. `[[@LINE+offset]` and +// `{{regex}}` syntaxes should also work. +// +// `lit_autoupdate.py` automatically constructs compatible lines. +class FileTestBase : public testing::Test { + public: + explicit FileTestBase(const llvm::StringRef path) : path_(path) {} + + // Used by children to register tests with gtest. + static void RegisterTests( + const char* fixture_label, const std::vector& paths, + std::function factory); + + // Implemented by children to run the test. Called by the TestBody + // implementation, which will validate stdout and stderr. The return value + // should be false when "fail_" is in the filename. + virtual auto RunOverFile(llvm::raw_ostream& stdout, llvm::raw_ostream& stderr) + -> bool = 0; + + // Runs a test and compares output. This keeps output split by line so that + // issues are a little easier to identify by the different line. + auto TestBody() -> void final; + + // Returns the filename of the file being tested. + auto filename() -> llvm::StringRef; + + // Returns the full path of the file being tested. + auto path() -> llvm::StringRef { return path_; }; + + private: + // Transforms an expectation on a given line from `FileCheck` syntax into a + // standard regex matcher. + static auto TransformExpectation(int line_index, llvm::StringRef in) + -> testing::Matcher; + + llvm::StringRef path_; +}; + +// Must be implemented by the individual file_test to initialize tests. +extern auto RegisterFileTests(const std::vector& paths) + -> void; + +} // namespace Carbon::Testing + +#endif // CARBON_TESTING_FILE_TEST_FILE_TEST_BASE_H_ diff --git a/testing/file_test/file_test_base_test.cpp b/testing/file_test/file_test_base_test.cpp new file mode 100644 index 000000000000..51ff16a1b834 --- /dev/null +++ b/testing/file_test/file_test_base_test.cpp @@ -0,0 +1,50 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "testing/file_test/file_test_base.h" + +#include +#include + +#include + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/raw_ostream.h" + +namespace Carbon::Testing { +namespace { + +class FileTestBaseTest : public FileTestBase { + public: + explicit FileTestBaseTest(llvm::StringRef path) : FileTestBase(path) {} + + auto RunOverFile(llvm::raw_ostream& stdout, llvm::raw_ostream& stderr) + -> bool override { + if (filename() == "example.carbon") { + stdout << "something\n" + "\n" + "8: Line delta\n" + "7: Negative line delta\n" + "+*[]{}\n" + "Foo baz\n"; + return true; + } else if (filename() == "fail_example.carbon") { + stderr << "Oops\n"; + return false; + } else { + ADD_FAILURE() << "Unexpected file: " << path().str(); + return false; + } + } +}; + +} // namespace + +auto RegisterFileTests(const std::vector& paths) -> void { + FileTestBaseTest::RegisterTests( + "FileTestBaseTest", paths, + [](llvm::StringRef path) { return new FileTestBaseTest(path); }); +} + +} // namespace Carbon::Testing diff --git a/testing/file_test/rules.bzl b/testing/file_test/rules.bzl new file mode 100644 index 000000000000..8cfa914a1a37 --- /dev/null +++ b/testing/file_test/rules.bzl @@ -0,0 +1,40 @@ +# Part of the Carbon Language project, under the Apache License v2.0 with LLVM +# Exceptions. See /LICENSE for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +"""Rules for building fuzz tests.""" + +load("@bazel_skylib//rules:native_binary.bzl", "native_test") + +def file_test(name, srcs, deps, tests, shard_count = 1): + """Generates tests using the file_test base. + + There will be one main test using `name` that can be sharded, and includes + all files. Additionally, per-file tests will be generated as + `name.file_path`; these per-file tests will be manual. + + Args: + name: The base name of the tests. + srcs: cc_test srcs. + deps: cc_test deps. + tests: The list of test files to use as data. + shard_count: The number of shards to use; defaults to 1. + """ + subset_name = "{0}.subset".format(name) + + native.cc_test( + name = name, + srcs = srcs, + deps = deps, + data = tests, + args = ["$(location {0})".format(x) for x in tests], + shard_count = shard_count, + ) + + native_test( + name = subset_name, + src = name, + out = subset_name, + data = tests, + tags = ["manual"], + ) diff --git a/toolchain/lowering/testdata/basics/zero.carbon b/toolchain/lowering/testdata/basics/zero.carbon index 26ad83f6bce5..186f64c6d6bf 100644 --- a/toolchain/lowering/testdata/basics/zero.carbon +++ b/toolchain/lowering/testdata/basics/zero.carbon @@ -6,7 +6,7 @@ // RUN: %{carbon-run-lowering} // CHECK:STDOUT: ; ModuleID = '{{.*}}/toolchain/lowering/testdata/basics/zero.carbon' // CHECK:STDOUT: source_filename = "{{.*}}/toolchain/lowering/testdata/basics/zero.carbon" -// CHECK-EMPTY: +// CHECK:STDOUT: // CHECK:STDOUT: define i32 @Main() { // CHECK:STDOUT: entry: // CHECK:STDOUT: ret i32 0 diff --git a/toolchain/lowering/testdata/function/definition/params_one.carbon b/toolchain/lowering/testdata/function/definition/params_one.carbon index dcab7e1f9495..106a70f7ed4c 100644 --- a/toolchain/lowering/testdata/function/definition/params_one.carbon +++ b/toolchain/lowering/testdata/function/definition/params_one.carbon @@ -6,7 +6,7 @@ // RUN: %{carbon-run-lowering} // CHECK:STDOUT: ; ModuleID = '{{.*}}/toolchain/lowering/testdata/function/definition/params_one.carbon' // CHECK:STDOUT: source_filename = "{{.*}}/toolchain/lowering/testdata/function/definition/params_one.carbon" -// CHECK-EMPTY: +// CHECK:STDOUT: // CHECK:STDOUT: define void @Foo(i32 %a) { // CHECK:STDOUT: entry: // CHECK:STDOUT: } diff --git a/toolchain/lowering/testdata/function/definition/params_two.carbon b/toolchain/lowering/testdata/function/definition/params_two.carbon index c090c0bc93db..3af7b7e1e430 100644 --- a/toolchain/lowering/testdata/function/definition/params_two.carbon +++ b/toolchain/lowering/testdata/function/definition/params_two.carbon @@ -6,7 +6,7 @@ // RUN: %{carbon-run-lowering} // CHECK:STDOUT: ; ModuleID = '{{.*}}/toolchain/lowering/testdata/function/definition/params_two.carbon' // CHECK:STDOUT: source_filename = "{{.*}}/toolchain/lowering/testdata/function/definition/params_two.carbon" -// CHECK-EMPTY: +// CHECK:STDOUT: // CHECK:STDOUT: define void @Foo(i32 %a, i32 %b) { // CHECK:STDOUT: entry: // CHECK:STDOUT: } diff --git a/toolchain/lowering/testdata/function/definition/params_zero.carbon b/toolchain/lowering/testdata/function/definition/params_zero.carbon index ff92d1e0aaa2..3de6972bac59 100644 --- a/toolchain/lowering/testdata/function/definition/params_zero.carbon +++ b/toolchain/lowering/testdata/function/definition/params_zero.carbon @@ -6,7 +6,7 @@ // RUN: %{carbon-run-lowering} // CHECK:STDOUT: ; ModuleID = '{{.*}}/toolchain/lowering/testdata/function/definition/params_zero.carbon' // CHECK:STDOUT: source_filename = "{{.*}}/toolchain/lowering/testdata/function/definition/params_zero.carbon" -// CHECK-EMPTY: +// CHECK:STDOUT: // CHECK:STDOUT: define void @Foo() { // CHECK:STDOUT: entry: // CHECK:STDOUT: } diff --git a/toolchain/lowering/testdata/return/no_value.carbon b/toolchain/lowering/testdata/return/no_value.carbon index ad7b4ff02cd6..fff9cb9e4e50 100644 --- a/toolchain/lowering/testdata/return/no_value.carbon +++ b/toolchain/lowering/testdata/return/no_value.carbon @@ -6,7 +6,7 @@ // RUN: %{carbon-run-lowering} // CHECK:STDOUT: ; ModuleID = '{{.*}}/toolchain/lowering/testdata/return/no_value.carbon' // CHECK:STDOUT: source_filename = "{{.*}}/toolchain/lowering/testdata/return/no_value.carbon" -// CHECK-EMPTY: +// CHECK:STDOUT: // CHECK:STDOUT: define void @Main() { // CHECK:STDOUT: entry: // CHECK:STDOUT: ret void diff --git a/toolchain/lowering/testdata/return/value.carbon b/toolchain/lowering/testdata/return/value.carbon index a0514dc6b97b..a2ce60b9c91a 100644 --- a/toolchain/lowering/testdata/return/value.carbon +++ b/toolchain/lowering/testdata/return/value.carbon @@ -6,7 +6,7 @@ // RUN: %{carbon-run-lowering} // CHECK:STDOUT: ; ModuleID = '{{.*}}/toolchain/lowering/testdata/return/value.carbon' // CHECK:STDOUT: source_filename = "{{.*}}/toolchain/lowering/testdata/return/value.carbon" -// CHECK-EMPTY: +// CHECK:STDOUT: // CHECK:STDOUT: define i32 @Main() { // CHECK:STDOUT: entry: // CHECK:STDOUT: ret i32 0