diff --git a/explorer/file_test.cpp b/explorer/file_test.cpp index 4704cdd46a76..90eea291fe4e 100644 --- a/explorer/file_test.cpp +++ b/explorer/file_test.cpp @@ -42,9 +42,13 @@ class ParseAndExecuteTestFile : public FileTestBase { } } - auto RunWithFiles(const llvm::SmallVector& test_files, + auto RunWithFiles(const llvm::SmallVector& test_args, + const llvm::SmallVector& test_files, llvm::raw_pwrite_stream& stdout, llvm::raw_pwrite_stream& stderr) -> bool override { + CARBON_CHECK(test_args.empty()) + << "ARGS are not currently used in explorer's file_test."; + if (test_files.size() != 1) { ADD_FAILURE() << "Only 1 file is supported: " << test_files.size() << " provided"; @@ -88,6 +92,10 @@ class ParseAndExecuteTestFile : public FileTestBase { return result.ok(); } + auto GetDefaultArgs() -> llvm::SmallVector override { + return {}; + } + private: bool trace_; bool is_trace_test = false; diff --git a/testing/file_test/file_test_base.cpp b/testing/file_test/file_test_base.cpp index ca7ae749639c..2a835f4777df 100644 --- a/testing/file_test/file_test_base.cpp +++ b/testing/file_test/file_test_base.cpp @@ -8,7 +8,9 @@ #include #include "common/check.h" +#include "llvm/ADT/StringExtras.h" #include "llvm/ADT/Twine.h" +#include "llvm/Support/FormatVariadic.h" #include "llvm/Support/InitLLVM.h" #include "testing/util/test_raw_ostream.h" @@ -20,6 +22,9 @@ static int base_dir_len = 0; static std::string* subset_target = nullptr; using ::testing::Eq; +using ::testing::Matcher; +using ::testing::MatchesRegex; +using ::testing::StrEq; void FileTestBase::RegisterTests( const char* fixture_label, @@ -70,18 +75,37 @@ auto FileTestBase::TestBody() -> void { std::string test_content = ReadFile(path()); // Load expected output. + llvm::SmallVector test_args; llvm::SmallVector test_files; - llvm::SmallVector> expected_stdout; - llvm::SmallVector> expected_stderr; - ProcessTestFile(test_content, test_files, expected_stdout, expected_stderr); + llvm::SmallVector> expected_stdout; + llvm::SmallVector> expected_stderr; + bool check_subset = false; + ProcessTestFile(test_content, test_args, test_files, expected_stdout, + expected_stderr, check_subset); if (HasFailure()) { return; } + // Process arguments. + if (test_args.empty()) { + test_args = GetDefaultArgs(); + } + DoArgReplacements(test_args, test_files); + if (HasFailure()) { + return; + } + + // Pass arguments as StringRef. + llvm::SmallVector test_args_ref; + test_args_ref.reserve(test_args.size()); + for (const auto& arg : test_args) { + test_args_ref.push_back(arg); + } + // Capture trace streaming, but only when in debug mode. TestRawOstream stdout; TestRawOstream stderr; - bool run_succeeded = RunWithFiles(test_files, stdout, stderr); + bool run_succeeded = RunWithFiles(test_args_ref, test_files, stdout, stderr); if (HasFailure()) { return; } @@ -91,14 +115,63 @@ auto FileTestBase::TestBody() -> void { "is expected to fail."; // Check results. - EXPECT_THAT(SplitOutput(stdout.TakeStr()), ElementsAreArray(expected_stdout)); - EXPECT_THAT(SplitOutput(stderr.TakeStr()), ElementsAreArray(expected_stderr)); + if (check_subset) { + EXPECT_THAT(SplitOutput(stdout.TakeStr()), IsSupersetOf(expected_stdout)); + EXPECT_THAT(SplitOutput(stderr.TakeStr()), IsSupersetOf(expected_stderr)); + + } else { + EXPECT_THAT(SplitOutput(stdout.TakeStr()), + ElementsAreArray(expected_stdout)); + EXPECT_THAT(SplitOutput(stderr.TakeStr()), + ElementsAreArray(expected_stderr)); + } +} + +auto FileTestBase::DoArgReplacements( + llvm::SmallVector& test_args, + const llvm::SmallVector& test_files) -> void { + for (auto* it = test_args.begin(); it != test_args.end(); ++it) { + auto percent = it->find("%"); + if (percent == std::string::npos) { + continue; + } + + if (percent + 1 >= it->size()) { + FAIL() << "% is not allowed on its own: " << *it; + } + char c = (*it)[percent + 1]; + switch (c) { + case 's': { + if (*it != "%s") { + FAIL() << "%s must be the full argument: " << *it; + } + it = test_args.erase(it); + for (const auto& file : test_files) { + it = test_args.insert(it, file.filename); + ++it; + } + // Back up once because the for loop will advance. + --it; + break; + } + case 't': { + char* temp = getenv("TEST_TMPDIR"); + CARBON_CHECK(temp != nullptr); + it->replace(percent, 2, llvm::formatv("{0}/temp_file", temp)); + break; + } + default: + FAIL() << "%" << c << " is not supported: " << *it; + } + } } auto FileTestBase::ProcessTestFile( - llvm::StringRef file_content, llvm::SmallVector& test_files, - llvm::SmallVector>& expected_stdout, - llvm::SmallVector>& expected_stderr) -> void { + llvm::StringRef file_content, llvm::SmallVector& test_args, + llvm::SmallVector& test_files, + llvm::SmallVector>& expected_stdout, + llvm::SmallVector>& expected_stderr, + bool& check_subset) -> void { llvm::StringRef cursor = file_content; bool found_content_pre_split = false; int line_index = 0; @@ -135,15 +208,34 @@ auto FileTestBase::ProcessTestFile( // Process expectations when found. auto line_trimmed = line.ltrim(); - if (!line_trimmed.consume_front("// CHECK")) { - continue; - } - if (line_trimmed.consume_front(":STDOUT:")) { - expected_stdout.push_back(TransformExpectation(line_index, line_trimmed)); - } else if (line_trimmed.consume_front(":STDERR:")) { - expected_stderr.push_back(TransformExpectation(line_index, line_trimmed)); - } else { - FAIL() << "Unexpected CHECK in input: " << line.str(); + if (line_trimmed.consume_front("// ARGS: ")) { + if (test_args.empty()) { + // Split the line into arguments. + std::pair cursor = + llvm::getToken(line_trimmed); + while (!cursor.first.empty()) { + test_args.push_back(std::string(cursor.first)); + cursor = llvm::getToken(cursor.second); + } + } else { + FAIL() << "ARGS was specified multiple times: " << line.str(); + } + } else if (line_trimmed == "// SET-CHECK-SUBSET") { + if (!check_subset) { + check_subset = true; + } else { + FAIL() << "SET-CHECK-SUBSET was specified multiple times"; + } + } else if (line_trimmed.consume_front("// CHECK")) { + if (line_trimmed.consume_front(":STDOUT:")) { + expected_stdout.push_back( + TransformExpectation(line_index, line_trimmed)); + } else if (line_trimmed.consume_front(":STDERR:")) { + expected_stderr.push_back( + TransformExpectation(line_index, line_trimmed)); + } else { + FAIL() << "Unexpected CHECK in input: " << line.str(); + } } } @@ -159,17 +251,17 @@ auto FileTestBase::ProcessTestFile( // Assume there is always a suffix `\n` in output. if (!expected_stdout.empty()) { - expected_stdout.push_back(testing::StrEq("")); + expected_stdout.push_back(StrEq("")); } if (!expected_stderr.empty()) { - expected_stderr.push_back(testing::StrEq("")); + expected_stderr.push_back(StrEq("")); } } auto FileTestBase::TransformExpectation(int line_index, llvm::StringRef in) - -> testing::Matcher { + -> Matcher { if (in.empty()) { - return testing::StrEq(""); + return StrEq(""); } CARBON_CHECK(in[0] == ' ') << "Malformated input: " << in; std::string str = in.substr(1).str(); @@ -245,7 +337,7 @@ auto FileTestBase::TransformExpectation(int line_index, llvm::StringRef in) } } - return testing::MatchesRegex(str); + return MatchesRegex(str); } } // namespace Carbon::Testing diff --git a/testing/file_test/file_test_base.h b/testing/file_test/file_test_base.h index 27f086d8fed6..77071fe3ee2c 100644 --- a/testing/file_test/file_test_base.h +++ b/testing/file_test/file_test_base.h @@ -25,14 +25,57 @@ namespace Carbon::Testing { // individual test executions. This framework includes a `main` implementation, // so users must not provide one. // -// Tests should have CHECK lines similar to `FileCheck` syntax: -// https://llvm.org/docs/CommandGuide/FileCheck.html +// Settings in files are provided in comments, similar to `FileCheck` syntax. +// `autoupdate_testdata.py` automatically constructs compatible CHECK:STDOUT: +// and CHECK:STDERR: lines. // -// Special nuances are that stdout and stderr will look like `// CHECK:STDOUT: -// ...` and `// CHECK:STDERR: ...` respectively. `[[@LINE+offset]` and -// `{{regex}}` syntaxes should also work. +// Supported comment markers are: // -// `autoupdate_testdata.py` automatically constructs compatible lines. +// - // ARGS: +// +// Provides a space-separated list of arguments, which will be passed to +// RunWithFiles as test_args. These are intended for use by the command as +// arguments. +// +// Supported replacements within arguments are: +// +// - %s +// +// Replaced with the list of files. Currently only allowed as a standalone +// argument, not a substring. +// +// - %t +// +// Replaced with `${TEST_TMPDIR}/temp_file`. +// +// ARGS can be specified at most once. If not provided, the FileTestBase child +// is responsible for providing default arguments. +// +// - // SET-CHECK-SUBSET +// +// By default, all lines of output must have a CHECK match. Adding this as a +// flag sets it so that non-matching lines are ignored. All provided +// CHECK:STDOUT: and CHECK:STDERR: lines must still have a match in output. +// +// SET-CHECK-SUBSET can be specified at most once. +// +// - // --- +// +// By default, all file content is provided to the test as a single file in +// test_files. Using this marker allows the file to be split into multiple +// files which will all be passed to test_files. +// +// Files are not created on disk; it's expected the child will create an +// InMemoryFilesystem if needed. +// +// - // CHECK:STDOUT: +// // CHECK:STDERR: +// +// These provides a match for output from the command. See SET-CHECK-SUBSET +// for how to change from full to subset matching of output. +// +// Output line matchers may contain `[[@LINE+offset]` and +// `{{regex}}` syntaxes, similar to `FileCheck`. class FileTestBase : public testing::Test { public: struct TestFile { @@ -72,10 +115,14 @@ class FileTestBase : public testing::Test { // Implemented by children to run the test. Called by the TestBody // implementation, which will validate stdout and stderr. The return value // should be false when "fail_" is in the filename. - virtual auto RunWithFiles(const llvm::SmallVector& test_files, + virtual auto RunWithFiles(const llvm::SmallVector& test_args, + const llvm::SmallVector& test_files, llvm::raw_pwrite_stream& stdout, llvm::raw_pwrite_stream& stderr) -> bool = 0; + // Returns default arguments. Only called when a file doesn't set ARGS. + virtual auto GetDefaultArgs() -> llvm::SmallVector = 0; + // Runs a test and compares output. This keeps output split by line so that // issues are a little easier to identify by the different line. auto TestBody() -> void final; @@ -84,12 +131,17 @@ class FileTestBase : public testing::Test { auto path() -> const std::filesystem::path& { return *path_; }; private: + // Does replacements in ARGS for %s and %t. + auto DoArgReplacements(llvm::SmallVector& test_args, + const llvm::SmallVector& test_files) -> void; + // Processes the test input, producing test files and expected output. auto ProcessTestFile( - llvm::StringRef file_content, llvm::SmallVector& test_files, + llvm::StringRef file_content, llvm::SmallVector& test_args, + llvm::SmallVector& test_files, llvm::SmallVector>& expected_stdout, - llvm::SmallVector>& expected_stderr) - -> void; + llvm::SmallVector>& expected_stderr, + bool& check_subset) -> void; // Transforms an expectation on a given line from `FileCheck` syntax into a // standard regex matcher. diff --git a/testing/file_test/file_test_base_test.cpp b/testing/file_test/file_test_base_test.cpp index 73826c05aa0c..95d4da86ce36 100644 --- a/testing/file_test/file_test_base_test.cpp +++ b/testing/file_test/file_test_base_test.cpp @@ -10,7 +10,7 @@ #include #include -#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringExtras.h" #include "llvm/Support/raw_ostream.h" namespace Carbon::Testing { @@ -20,30 +20,44 @@ using ::testing::AllOf; using ::testing::ElementsAre; using ::testing::Eq; using ::testing::Field; +using ::testing::Matcher; class FileTestBaseTest : public FileTestBase { public: explicit FileTestBaseTest(const std::filesystem::path& path) : FileTestBase(path) {} - static auto HasFilename(std::string filename) -> testing::Matcher { + static auto HasFilename(std::string filename) -> Matcher { return Field("filename", &TestFile::filename, Eq(filename)); } - static auto HasContent(std::string content) -> testing::Matcher { + static auto HasContent(std::string content) -> Matcher { return Field("content", &TestFile::content, Eq(content)); } - auto RunWithFiles(const llvm::SmallVector& test_files, + auto RunWithFiles(const llvm::SmallVector& test_args, + const llvm::SmallVector& test_files, llvm::raw_pwrite_stream& stdout, llvm::raw_pwrite_stream& stderr) -> bool override { + if (!test_args.empty()) { + llvm::ListSeparator sep; + stdout << test_args.size() << " args: "; + for (const auto& arg : test_args) { + stdout << sep << "`" << arg << "`"; + } + stdout << "\n"; + } + auto filename = path().filename(); - if (filename == "example.carbon") { + if (filename == "args.carbon") { + EXPECT_THAT(test_files, ElementsAre(HasFilename("args.carbon"))); + return true; + } else if (filename == "example.carbon") { EXPECT_THAT(test_files, ElementsAre(HasFilename("example.carbon"))); stdout << "something\n" "\n" - "8: Line delta\n" - "7: Negative line delta\n" + "9: Line delta\n" + "8: Negative line delta\n" "+*[]{}\n" "Foo baz\n"; return true; @@ -70,6 +84,10 @@ class FileTestBaseTest : public FileTestBase { return false; } } + + auto GetDefaultArgs() -> llvm::SmallVector override { + return {"default_args", "%s"}; + } }; } // namespace diff --git a/testing/file_test/testdata/args.carbon b/testing/file_test/testdata/args.carbon new file mode 100644 index 000000000000..696dd1ffb4a6 --- /dev/null +++ b/testing/file_test/testdata/args.carbon @@ -0,0 +1,6 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// ARGS: abc file=%t %s +// CHECK:STDOUT: 3 args: `abc`, `file={{.+}}/temp_file`, `args.carbon` diff --git a/testing/file_test/testdata/example.carbon b/testing/file_test/testdata/example.carbon index 1f0301213f5b..85b66ecf2ae4 100644 --- a/testing/file_test/testdata/example.carbon +++ b/testing/file_test/testdata/example.carbon @@ -2,6 +2,7 @@ // Exceptions. See /LICENSE for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// CHECK:STDOUT: 2 args: `default_args`, `example.carbon` // CHECK:STDOUT: something // CHECK:STDOUT: // CHECK:STDOUT: [[@LINE+1]]: Line delta diff --git a/testing/file_test/testdata/fail_example.carbon b/testing/file_test/testdata/fail_example.carbon index 2516b5d814cf..de4130b67915 100644 --- a/testing/file_test/testdata/fail_example.carbon +++ b/testing/file_test/testdata/fail_example.carbon @@ -2,4 +2,5 @@ // Exceptions. See /LICENSE for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// CHECK:STDOUT: 2 args: `default_args`, `fail_example.carbon` // CHECK:STDERR: Oops diff --git a/testing/file_test/testdata/two_files.carbon b/testing/file_test/testdata/two_files.carbon index 11757a8604f0..9d826c0f711f 100644 --- a/testing/file_test/testdata/two_files.carbon +++ b/testing/file_test/testdata/two_files.carbon @@ -2,6 +2,8 @@ // Exceptions. See /LICENSE for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// CHECK:STDOUT: 3 args: `default_args`, `a.carbon`, `b.carbon` + // --- a.carbon // CHECK:STDOUT: a.carbon: [[@LINE+0]] diff --git a/toolchain/codegen/BUILD b/toolchain/codegen/BUILD index 39dc612cc226..37472ee6c823 100644 --- a/toolchain/codegen/BUILD +++ b/toolchain/codegen/BUILD @@ -2,6 +2,8 @@ # Exceptions. See /LICENSE for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +load("//testing/file_test:rules.bzl", "file_test") + package(default_visibility = ["//visibility:public"]) cc_library( @@ -18,3 +20,13 @@ cc_library( "@llvm-project//llvm:TargetParser", ], ) + +file_test( + name = "codegen_file_test", + srcs = ["codegen_file_test.cpp"], + tests = glob(["testdata/**/*.carbon"]), + deps = [ + "//toolchain/driver:driver_file_test_base", + "@llvm-project//llvm:Support", + ], +) diff --git a/toolchain/codegen/codegen_file_test.cpp b/toolchain/codegen/codegen_file_test.cpp new file mode 100644 index 000000000000..bc6d0505c19d --- /dev/null +++ b/toolchain/codegen/codegen_file_test.cpp @@ -0,0 +1,31 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include +#include + +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "toolchain/driver/driver_file_test_base.h" + +namespace Carbon::Testing { +namespace { + +class CodeGenFileTest : public DriverFileTestBase { + public: + using DriverFileTestBase::DriverFileTestBase; + + auto GetDefaultArgs() -> llvm::SmallVector override { + CARBON_FATAL() << "ARGS is always set in these tests"; + } +}; + +} // namespace + +auto RegisterFileTests(const llvm::SmallVector& paths) + -> void { + CodeGenFileTest::RegisterTests("CodeGenFileTest", paths); +} + +} // namespace Carbon::Testing diff --git a/toolchain/driver/testdata/codegen_assembly.carbon b/toolchain/codegen/testdata/assembly/basic.carbon similarity index 70% rename from toolchain/driver/testdata/codegen_assembly.carbon rename to toolchain/codegen/testdata/assembly/basic.carbon index 845dffe2b2db..3337d0637379 100644 --- a/toolchain/driver/testdata/codegen_assembly.carbon +++ b/toolchain/codegen/testdata/assembly/basic.carbon @@ -2,7 +2,9 @@ // Exceptions. See /LICENSE for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// RUN: %{carbon} dump assembly --target_triple=x86_64-unknown-linux-gnu %s | %{FileCheck-allow-unmatched} +// ARGS: dump assembly --target_triple=x86_64-unknown-linux-gnu %s +// NOAUTOUPDATE +// SET-CHECK-SUBSET // CHECK:STDOUT: Main: fn Main() -> i32 { return 0; } diff --git a/toolchain/driver/testdata/error_codegen_objcode_target_triple.carbon b/toolchain/codegen/testdata/assembly/fail_target_triple.carbon similarity index 69% rename from toolchain/driver/testdata/error_codegen_objcode_target_triple.carbon rename to toolchain/codegen/testdata/assembly/fail_target_triple.carbon index 1207534b5e5c..57527c26b916 100644 --- a/toolchain/driver/testdata/error_codegen_objcode_target_triple.carbon +++ b/toolchain/codegen/testdata/assembly/fail_target_triple.carbon @@ -2,7 +2,9 @@ // Exceptions. See /LICENSE for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// RUN: %{not} %{carbon} dump objcode --target_triple=x86_684-unknown-linux-gnu --output_file=%t %s | %{FileCheck-strict} +// ARGS: dump assembly --target_triple=x86_687-unknown-linux-gnu %s +// TODO: Support autoupdate with ARGS. +// NOAUTOUPDATE // CHECK:STDERR: ERROR: Invalid -target_triple:{{.*}} fn Main() -> i32 { return 0; } diff --git a/toolchain/driver/testdata/success_codegen_objfile.carbon b/toolchain/codegen/testdata/objcode/basic.carbon similarity index 66% rename from toolchain/driver/testdata/success_codegen_objfile.carbon rename to toolchain/codegen/testdata/objcode/basic.carbon index 3480fc0dfe6f..28bcd2585486 100644 --- a/toolchain/driver/testdata/success_codegen_objfile.carbon +++ b/toolchain/codegen/testdata/objcode/basic.carbon @@ -2,7 +2,9 @@ // Exceptions. See /LICENSE for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// RUN: %{carbon} dump objcode --target_triple=x86_64-unknown-linux-gnu --output_file=%t %s | %{FileCheck-strict} +// ARGS: dump objcode --target_triple=x86_64-unknown-linux-gnu --output_file=%t %s +// TODO: Support autoupdate with ARGS. +// NOAUTOUPDATE // CHECK:STDOUT: Success: Object file is generated! fn Main() -> i32 { return 0; } diff --git a/toolchain/driver/testdata/error_codegen_objcode_no_input_file.carbon b/toolchain/codegen/testdata/objcode/fail_no_input_file.carbon similarity index 67% rename from toolchain/driver/testdata/error_codegen_objcode_no_input_file.carbon rename to toolchain/codegen/testdata/objcode/fail_no_input_file.carbon index 0e0396e6b986..73957f6a3c09 100644 --- a/toolchain/driver/testdata/error_codegen_objcode_no_input_file.carbon +++ b/toolchain/codegen/testdata/objcode/fail_no_input_file.carbon @@ -2,7 +2,9 @@ // Exceptions. See /LICENSE for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// RUN: %{not} %{carbon} dump objcode --output_file=%t --target_triple=x86_64-unknown-linux-gnu | %{FileCheck-strict} +// ARGS: dump objcode --output_file=%t --target_triple=x86_64-unknown-linux-gnu +// TODO: Support autoupdate with ARGS. +// NOAUTOUPDATE // CHECK:STDERR: ERROR: No input file specified. fn Main() -> i32 { return 0; } diff --git a/toolchain/driver/testdata/error_codegen_objcode_no_output_file.carbon b/toolchain/codegen/testdata/objcode/fail_no_output_file.carbon similarity index 69% rename from toolchain/driver/testdata/error_codegen_objcode_no_output_file.carbon rename to toolchain/codegen/testdata/objcode/fail_no_output_file.carbon index e358db4e60b4..95b6d8407edf 100644 --- a/toolchain/driver/testdata/error_codegen_objcode_no_output_file.carbon +++ b/toolchain/codegen/testdata/objcode/fail_no_output_file.carbon @@ -2,7 +2,9 @@ // Exceptions. See /LICENSE for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// RUN: %{not} %{carbon} dump objcode --target_triple=x86_64-unknown-linux-gnu %s | %{FileCheck-strict} +// ARGS: dump objcode --target_triple=x86_64-unknown-linux-gnu %s +// TODO: Support autoupdate with ARGS. +// NOAUTOUPDATE // CHECK:STDERR: ERROR: Must provide an output file. fn Main() -> i32 { return 0; } diff --git a/toolchain/driver/testdata/error_codegen_assembly_target_triple.carbon b/toolchain/codegen/testdata/objcode/fail_target_triple.carbon similarity index 66% rename from toolchain/driver/testdata/error_codegen_assembly_target_triple.carbon rename to toolchain/codegen/testdata/objcode/fail_target_triple.carbon index 78b93b39b8d2..df34fb3e99f7 100644 --- a/toolchain/driver/testdata/error_codegen_assembly_target_triple.carbon +++ b/toolchain/codegen/testdata/objcode/fail_target_triple.carbon @@ -2,7 +2,9 @@ // Exceptions. See /LICENSE for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// RUN: %{not} %{carbon} dump assembly --target_triple=x86_687-unknown-linux-gnu %s | %{FileCheck-strict} +// ARGS: dump objcode --target_triple=x86_684-unknown-linux-gnu --output_file=%t %s +// TODO: Support autoupdate with ARGS. +// NOAUTOUPDATE // CHECK:STDERR: ERROR: Invalid -target_triple:{{.*}} fn Main() -> i32 { return 0; } diff --git a/toolchain/driver/BUILD b/toolchain/driver/BUILD index 153cfc39ccbb..164481102be3 100644 --- a/toolchain/driver/BUILD +++ b/toolchain/driver/BUILD @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception load("//bazel/cc_toolchains:defs.bzl", "cc_env") +load("//testing/file_test:rules.bzl", "file_test") load("//testing/fuzzing:rules.bzl", "cc_fuzz_test") package(default_visibility = ["//visibility:public"]) @@ -27,6 +28,16 @@ cc_library( ], ) +file_test( + name = "driver_file_test", + srcs = ["driver_file_test.cpp"], + tests = glob(["testdata/**/*.carbon"]), + deps = [ + ":driver_file_test_base", + "@llvm-project//llvm:Support", + ], +) + cc_test( name = "driver_test", size = "small", diff --git a/toolchain/driver/driver_file_test.cpp b/toolchain/driver/driver_file_test.cpp new file mode 100644 index 000000000000..f80c421dafee --- /dev/null +++ b/toolchain/driver/driver_file_test.cpp @@ -0,0 +1,30 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include +#include + +#include "llvm/ADT/SmallVector.h" +#include "toolchain/driver/driver_file_test_base.h" + +namespace Carbon::Testing { +namespace { + +class DriverFileTest : public DriverFileTestBase { + public: + using DriverFileTestBase::DriverFileTestBase; + + auto GetDefaultArgs() -> llvm::SmallVector override { + CARBON_FATAL() << "ARGS is always set in these tests"; + } +}; + +} // namespace + +auto RegisterFileTests(const llvm::SmallVector& paths) + -> void { + DriverFileTest::RegisterTests("DriverFileTest", paths); +} + +} // namespace Carbon::Testing diff --git a/toolchain/driver/driver_file_test_base.h b/toolchain/driver/driver_file_test_base.h index e8a3e5dc376a..cc80d9dec17a 100644 --- a/toolchain/driver/driver_file_test_base.h +++ b/toolchain/driver/driver_file_test_base.h @@ -24,16 +24,13 @@ class DriverFileTestBase : public FileTestBase { public: using FileTestBase::FileTestBase; - auto RunWithFiles(const llvm::SmallVector& test_files, + auto RunWithFiles(const llvm::SmallVector& test_args, + const llvm::SmallVector& test_files, llvm::raw_pwrite_stream& stdout, llvm::raw_pwrite_stream& stderr) -> bool override { - // Prepare a list of filenames for MakeArgs. Also create the files - // in-memory. - llvm::SmallVector test_file_names; + // Create the files in-memory. llvm::vfs::InMemoryFileSystem fs; for (const auto& test_file : test_files) { - test_file_names.push_back(test_file.filename); - if (!fs.addFile(test_file.filename, /*ModificationTime=*/0, llvm::MemoryBuffer::getMemBuffer(test_file.content))) { ADD_FAILURE() << "File is repeated: " << test_file.filename; @@ -42,11 +39,8 @@ class DriverFileTestBase : public FileTestBase { } Driver driver(fs, stdout, stderr); - return driver.RunFullCommand(MakeArgs(test_file_names)); + return driver.RunFullCommand(test_args); } - - virtual auto MakeArgs(const llvm::SmallVector& test_files) - -> llvm::SmallVector = 0; }; } // namespace Carbon::Testing diff --git a/toolchain/driver/testdata/BUILD b/toolchain/driver/testdata/BUILD deleted file mode 100644 index 617ee1052063..000000000000 --- a/toolchain/driver/testdata/BUILD +++ /dev/null @@ -1,17 +0,0 @@ -# Part of the Carbon Language project, under the Apache License v2.0 with LLVM -# Exceptions. See /LICENSE for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -load("//testing/lit_test:rules.bzl", "glob_lit_tests") - -glob_lit_tests( - name = "all_lit_tests", - data = [ - "//testing/lit_test:merge_output", - "//toolchain/driver:carbon", - "@llvm-project//llvm:FileCheck", - "@llvm-project//llvm:not", - ], - driver = "lit.cfg.py", - test_file_exts = ["carbon"], -) diff --git a/toolchain/driver/testdata/errors_sorted_test.carbon b/toolchain/driver/testdata/errors_sorted_test.carbon deleted file mode 100644 index cea267ec115d..000000000000 --- a/toolchain/driver/testdata/errors_sorted_test.carbon +++ /dev/null @@ -1,19 +0,0 @@ -// Part of the Carbon Language project, under the Apache License v2.0 with LLVM -// Exceptions. See /LICENSE for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -// RUN: %{not} %{carbon} dump tokens %s | %{FileCheck-strict} -// CHECK:STDOUT: [ -// CHECK-COUNT-17:STDOUT: {{.*}} -// CHECK:STDOUT: ] - -// CHECK:STDERR: {{.*}}/errors_sorted_test.carbon:[[@LINE+3]]:24: Closing symbol does not match most recent opening symbol. -// CHECK:STDERR: fn run(String program) { -// CHECK:STDERR: ^ -fn run(String program) { - return True; - -// CHECK:STDERR: {{.*}}/errors_sorted_test.carbon:[[@LINE+3]]:10: Invalid digit 'a' in decimal numeric literal. -// CHECK:STDERR: var x = 3a; -// CHECK:STDERR: ^ -var x = 3a; diff --git a/toolchain/driver/testdata/errors_streamed_test.carbon b/toolchain/driver/testdata/errors_streamed_test.carbon deleted file mode 100644 index cf26950b99e0..000000000000 --- a/toolchain/driver/testdata/errors_streamed_test.carbon +++ /dev/null @@ -1,21 +0,0 @@ -// Part of the Carbon Language project, under the Apache License v2.0 with LLVM -// Exceptions. See /LICENSE for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -// RUN: %{not} %{carbon} --print-errors=streamed dump tokens %s | \ -// RUN: %{FileCheck-strict} -// CHECK:STDOUT: [ -// CHECK-COUNT-17:STDOUT: {{.*}} -// CHECK:STDOUT: ] - -fn run(String program) { - return True; - -var x = 3a; - -// CHECK:STDERR: {{.*}}/errors_streamed_test.carbon:[[@LINE-2]]:10: Invalid digit 'a' in decimal numeric literal. -// CHECK:STDERR: var x = 3a; -// CHECK:STDERR: ^ -// CHECK:STDERR: {{.*}}/errors_streamed_test.carbon:[[@LINE-8]]:24: Closing symbol does not match most recent opening symbol. -// CHECK:STDERR: fn run(String program) { -// CHECK:STDERR: ^ diff --git a/toolchain/driver/testdata/fail_errors_sorted.carbon b/toolchain/driver/testdata/fail_errors_sorted.carbon new file mode 100644 index 000000000000..4b54b287796c --- /dev/null +++ b/toolchain/driver/testdata/fail_errors_sorted.carbon @@ -0,0 +1,39 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// ARGS: dump tokens %s +// +// TODO: Support autoupdate with ARGS. +// NOAUTOUPDATE +// TODO: Disable token output, it's not interesting for these tests. +// CHECK:STDOUT: [ +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: ] + +// CHECK:STDERR: fail_errors_sorted.carbon:[[@LINE+3]]:24: Closing symbol does not match most recent opening symbol. +// CHECK:STDERR: fn run(String program) { +// CHECK:STDERR: ^ +fn run(String program) { + return True; + +// CHECK:STDERR: fail_errors_sorted.carbon:[[@LINE+3]]:10: Invalid digit 'a' in decimal numeric literal. +// CHECK:STDERR: var x = 3a; +// CHECK:STDERR: ^ +var x = 3a; diff --git a/toolchain/driver/testdata/fail_errors_streamed.carbon b/toolchain/driver/testdata/fail_errors_streamed.carbon new file mode 100644 index 000000000000..b2ea7e0baee7 --- /dev/null +++ b/toolchain/driver/testdata/fail_errors_streamed.carbon @@ -0,0 +1,40 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// ARGS: --print-errors=streamed dump tokens %s +// +// TODO: Support autoupdate with ARGS. +// NOAUTOUPDATE +// TODO: Disable token output, it's not interesting for these tests. +// CHECK:STDOUT: [ +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: {{.*}} +// CHECK:STDOUT: ] + +fn run(String program) { + return True; + +var x = 3a; + +// CHECK:STDERR: fail_errors_streamed.carbon:[[@LINE-2]]:10: Invalid digit 'a' in decimal numeric literal. +// CHECK:STDERR: var x = 3a; +// CHECK:STDERR: ^ +// CHECK:STDERR: fail_errors_streamed.carbon:[[@LINE-8]]:24: Closing symbol does not match most recent opening symbol. +// CHECK:STDERR: fn run(String program) { +// CHECK:STDERR: ^ diff --git a/toolchain/driver/testdata/lit.cfg.py b/toolchain/driver/testdata/lit.cfg.py deleted file mode 120000 index 27a7283e7bea..000000000000 --- a/toolchain/driver/testdata/lit.cfg.py +++ /dev/null @@ -1 +0,0 @@ -../../../testing/lit_test/lit.cfg.py \ No newline at end of file diff --git a/toolchain/lexer/lexer_file_test.cpp b/toolchain/lexer/lexer_file_test.cpp index b23e3e528da6..65cc3e3c9190 100644 --- a/toolchain/lexer/lexer_file_test.cpp +++ b/toolchain/lexer/lexer_file_test.cpp @@ -16,11 +16,8 @@ class LexerFileTest : public DriverFileTestBase { public: using DriverFileTestBase::DriverFileTestBase; - auto MakeArgs(const llvm::SmallVector& test_files) - -> llvm::SmallVector override { - llvm::SmallVector args({"dump", "tokens"}); - args.insert(args.end(), test_files.begin(), test_files.end()); - return args; + auto GetDefaultArgs() -> llvm::SmallVector override { + return {"dump", "tokens", "%s"}; } }; diff --git a/toolchain/lowering/lowering_file_test.cpp b/toolchain/lowering/lowering_file_test.cpp index 18065a338656..9da23f3dc027 100644 --- a/toolchain/lowering/lowering_file_test.cpp +++ b/toolchain/lowering/lowering_file_test.cpp @@ -16,11 +16,8 @@ class LoweringFileTest : public DriverFileTestBase { public: using DriverFileTestBase::DriverFileTestBase; - auto MakeArgs(const llvm::SmallVector& test_files) - -> llvm::SmallVector override { - llvm::SmallVector args({"dump", "llvm-ir"}); - args.insert(args.end(), test_files.begin(), test_files.end()); - return args; + auto GetDefaultArgs() -> llvm::SmallVector override { + return {"dump", "llvm-ir", "%s"}; } }; diff --git a/toolchain/parser/parse_tree_file_test.cpp b/toolchain/parser/parse_tree_file_test.cpp index 64213b842b9f..d89578ba8118 100644 --- a/toolchain/parser/parse_tree_file_test.cpp +++ b/toolchain/parser/parse_tree_file_test.cpp @@ -16,11 +16,8 @@ class ParseTreeFileTest : public DriverFileTestBase { public: using DriverFileTestBase::DriverFileTestBase; - auto MakeArgs(const llvm::SmallVector& test_files) - -> llvm::SmallVector override { - llvm::SmallVector args({"dump", "parse-tree"}); - args.insert(args.end(), test_files.begin(), test_files.end()); - return args; + auto GetDefaultArgs() -> llvm::SmallVector override { + return {"dump", "parse-tree", "%s"}; } }; diff --git a/toolchain/semantics/semantics_file_test.cpp b/toolchain/semantics/semantics_file_test.cpp index e20cef09c9d8..09f54022b93d 100644 --- a/toolchain/semantics/semantics_file_test.cpp +++ b/toolchain/semantics/semantics_file_test.cpp @@ -16,11 +16,8 @@ class SemanticsFileTest : public DriverFileTestBase { public: using DriverFileTestBase::DriverFileTestBase; - auto MakeArgs(const llvm::SmallVector& test_files) - -> llvm::SmallVector override { - llvm::SmallVector args({"dump", "semantics-ir"}); - args.insert(args.end(), test_files.begin(), test_files.end()); - return args; + auto GetDefaultArgs() -> llvm::SmallVector override { + return {"dump", "semantics-ir", "%s"}; } }; diff --git a/toolchain/driver/testdata/semantics_builtin_nodes.carbon b/toolchain/semantics/testdata/basics/builtin_nodes.carbon similarity index 92% rename from toolchain/driver/testdata/semantics_builtin_nodes.carbon rename to toolchain/semantics/testdata/basics/builtin_nodes.carbon index 55c26edc46ee..b233cafa5a40 100644 --- a/toolchain/driver/testdata/semantics_builtin_nodes.carbon +++ b/toolchain/semantics/testdata/basics/builtin_nodes.carbon @@ -2,8 +2,10 @@ // Exceptions. See /LICENSE for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// RUN: %{carbon} dump semantics-ir --include_builtins %s | \ -// RUN: %{FileCheck-strict} +// ARGS: dump semantics-ir --include_builtins %s +// +// TODO: Support autoupdate with ARGS +// NOAUTOUPDATE // CHECK:STDOUT: cross_reference_irs_size: 1 // CHECK:STDOUT: functions: [ // CHECK:STDOUT: ] diff --git a/toolchain/driver/testdata/semantics_verbose.carbon b/toolchain/semantics/testdata/basics/verbose.carbon similarity index 86% rename from toolchain/driver/testdata/semantics_verbose.carbon rename to toolchain/semantics/testdata/basics/verbose.carbon index b5f9d1fea60f..eabd1d2df7d2 100644 --- a/toolchain/driver/testdata/semantics_verbose.carbon +++ b/toolchain/semantics/testdata/basics/verbose.carbon @@ -2,9 +2,11 @@ // Exceptions. See /LICENSE for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// RUN: %{carbon} -v dump semantics-ir %s | %{FileCheck-allow-unmatched} +// ARGS: -v dump semantics-ir %s // // Only checks a couple statements in order to minimize manual update churn. +// NOAUTOUPDATE +// SET-CHECK-SUBSET // CHECK:STDERR: Node Push 0: FunctionIntroducer -> // CHECK:STDERR: AddNode block{{[0-9]+}}: {kind: FunctionDeclaration, arg0: function{{[0-9]+}}}