mirror of
https://github.com/carbon-language/carbon-lang.git
synced 2026-09-24 22:02:23 +01:00
Jon Ross-Perkins
d0b2b7bc41
Trying to more proactively catch when autoupdate is missed. Most of the execution time of these tests should be in running the program under test, not processing output, so this should have marginal overhead in order to produce a useful reminder.
558 lines
20 KiB
C++
558 lines
20 KiB
C++
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
|
|
// Exceptions. See /LICENSE for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
|
|
#include "testing/file_test/file_test_base.h"
|
|
|
|
#include <filesystem>
|
|
#include <fstream>
|
|
#include <optional>
|
|
#include <string>
|
|
#include <utility>
|
|
|
|
#include "absl/flags/flag.h"
|
|
#include "absl/flags/parse.h"
|
|
#include "common/check.h"
|
|
#include "llvm/ADT/StringExtras.h"
|
|
#include "llvm/ADT/Twine.h"
|
|
#include "llvm/Support/FormatVariadic.h"
|
|
#include "llvm/Support/InitLLVM.h"
|
|
#include "llvm/Support/MemoryBuffer.h"
|
|
#include "llvm/Support/PrettyStackTrace.h"
|
|
#include "testing/file_test/autoupdate.h"
|
|
|
|
ABSL_FLAG(std::vector<std::string>, file_tests, {},
|
|
"A comma-separated list of repo-relative names of test files. "
|
|
"Overrides test_targets_file.");
|
|
ABSL_FLAG(std::string, test_targets_file, "",
|
|
"A path to a file containing repo-relative names of test files.");
|
|
ABSL_FLAG(bool, autoupdate, false,
|
|
"Instead of verifying files match test output, autoupdate files "
|
|
"based on test output.");
|
|
|
|
namespace Carbon::Testing {
|
|
|
|
using ::testing::Eq;
|
|
using ::testing::Matcher;
|
|
using ::testing::MatchesRegex;
|
|
using ::testing::StrEq;
|
|
|
|
// Reads a file to string.
|
|
static auto ReadFile(std::string_view path) -> std::string {
|
|
std::ifstream proto_file(path);
|
|
std::stringstream buffer;
|
|
buffer << proto_file.rdbuf();
|
|
proto_file.close();
|
|
return buffer.str();
|
|
}
|
|
|
|
// Splits outputs to string_view because gtest handles string_view by default.
|
|
static auto SplitOutput(llvm::StringRef output)
|
|
-> llvm::SmallVector<std::string_view> {
|
|
if (output.empty()) {
|
|
return {};
|
|
}
|
|
llvm::SmallVector<llvm::StringRef> lines;
|
|
llvm::StringRef(output).split(lines, "\n");
|
|
return llvm::SmallVector<std::string_view>(lines.begin(), lines.end());
|
|
}
|
|
|
|
// Runs a test and compares output. This keeps output split by line so that
|
|
// issues are a little easier to identify by the different line.
|
|
auto FileTestBase::TestBody() -> void {
|
|
const char* target = getenv("TEST_TARGET");
|
|
CARBON_CHECK(target);
|
|
// This advice overrides the --file_tests flag provided by the file_test rule.
|
|
llvm::errs() << "\nTo test this file alone, run:\n bazel test " << target
|
|
<< " --test_arg=--file_tests=" << test_name_ << "\n\n";
|
|
|
|
// Add a crash trace entry with a command that runs this test in isolation.
|
|
llvm::PrettyStackTraceFormat stack_trace_entry(
|
|
"bazel test %s --test_arg=--file_tests=%s", target, test_name_);
|
|
|
|
TestContext context;
|
|
auto run_result = ProcessTestFileAndRun(context);
|
|
ASSERT_TRUE(run_result.ok()) << run_result.error();
|
|
ValidateRun();
|
|
auto test_filename = std::filesystem::path(test_name_.str()).filename();
|
|
EXPECT_THAT(!llvm::StringRef(test_filename).starts_with("fail_"),
|
|
Eq(context.exit_with_success))
|
|
<< "Tests should be prefixed with `fail_` if and only if running them "
|
|
"is expected to fail.";
|
|
|
|
// Check results. Include a reminder of the autoupdate command for any
|
|
// stdout/stderr differences.
|
|
std::string update_message;
|
|
if (context.autoupdate_line_number) {
|
|
update_message = llvm::formatv(
|
|
"If these differences are expected, try the autoupdater:\n"
|
|
"\tbazel run {0} -- --autoupdate --file_tests={1}",
|
|
target, test_name_);
|
|
} else {
|
|
update_message =
|
|
"If these differences are expected, content must be updated manually.";
|
|
}
|
|
SCOPED_TRACE(update_message);
|
|
if (context.check_subset) {
|
|
EXPECT_THAT(SplitOutput(context.stdout),
|
|
IsSupersetOf(context.expected_stdout));
|
|
EXPECT_THAT(SplitOutput(context.stderr),
|
|
IsSupersetOf(context.expected_stderr));
|
|
|
|
} else {
|
|
EXPECT_THAT(SplitOutput(context.stdout),
|
|
ElementsAreArray(context.expected_stdout));
|
|
EXPECT_THAT(SplitOutput(context.stderr),
|
|
ElementsAreArray(context.expected_stderr));
|
|
}
|
|
|
|
// If there are no other test failures, check if autoupdate would make
|
|
// changes. We don't do this when there _are_ failures because the
|
|
// SCOPED_TRACE already contains the autoupdate reminder.
|
|
if (!HasFailure() && RunAutoupdater(context, /*dry_run=*/true)) {
|
|
ADD_FAILURE() << "Autoupdate would make changes to the file content.";
|
|
}
|
|
}
|
|
|
|
auto FileTestBase::RunAutoupdater(const TestContext& context, bool dry_run)
|
|
-> bool {
|
|
if (!context.autoupdate_line_number) {
|
|
return false;
|
|
}
|
|
|
|
llvm::SmallVector<llvm::StringRef> filenames;
|
|
filenames.reserve(context.non_check_lines.size());
|
|
if (context.has_splits) {
|
|
// There are splits, so we provide an empty name for the first file.
|
|
filenames.push_back({});
|
|
}
|
|
for (const auto& file : context.test_files) {
|
|
filenames.push_back(file.filename);
|
|
}
|
|
|
|
llvm::ArrayRef expected_filenames = filenames;
|
|
if (filenames.size() > 1) {
|
|
expected_filenames = expected_filenames.drop_front();
|
|
}
|
|
|
|
return FileTestAutoupdater(
|
|
std::filesystem::absolute(test_name_.str()), context.input_content,
|
|
filenames, *context.autoupdate_line_number,
|
|
context.non_check_lines, context.stdout, context.stderr,
|
|
GetDefaultFileRE(expected_filenames),
|
|
GetLineNumberReplacements(expected_filenames),
|
|
[&](std::string& line) { DoExtraCheckReplacements(line); })
|
|
.Run(dry_run);
|
|
}
|
|
|
|
auto FileTestBase::Autoupdate() -> ErrorOr<bool> {
|
|
// Add a crash trace entry mentioning which file we're updating.
|
|
llvm::PrettyStackTraceFormat stack_trace_entry("performing autoupdate for %s",
|
|
test_name_);
|
|
|
|
TestContext context;
|
|
auto run_result = ProcessTestFileAndRun(context);
|
|
if (!run_result.ok()) {
|
|
return ErrorBuilder() << "Error updating " << test_name_ << ": "
|
|
<< run_result.error();
|
|
}
|
|
return RunAutoupdater(context, /*dry_run=*/false);
|
|
}
|
|
|
|
auto FileTestBase::GetLineNumberReplacements(
|
|
llvm::ArrayRef<llvm::StringRef> filenames)
|
|
-> llvm::SmallVector<LineNumberReplacement> {
|
|
return {{.has_file = true,
|
|
.re = std::make_shared<RE2>(
|
|
llvm::formatv(R"(({0}):(\d+))", llvm::join(filenames, "|"))),
|
|
.line_formatv = R"({0})"}};
|
|
}
|
|
|
|
auto FileTestBase::ProcessTestFileAndRun(TestContext& context)
|
|
-> ErrorOr<Success> {
|
|
// Store the file so that test_files can use references to content.
|
|
context.input_content = ReadFile(test_name_);
|
|
|
|
// Load expected output.
|
|
CARBON_RETURN_IF_ERROR(ProcessTestFile(context));
|
|
|
|
// Process arguments.
|
|
if (context.test_args.empty()) {
|
|
context.test_args = GetDefaultArgs();
|
|
}
|
|
CARBON_RETURN_IF_ERROR(
|
|
DoArgReplacements(context.test_args, context.test_files));
|
|
|
|
// Create the files in-memory.
|
|
llvm::vfs::InMemoryFileSystem fs;
|
|
for (const auto& test_file : context.test_files) {
|
|
if (!fs.addFile(test_file.filename, /*ModificationTime=*/0,
|
|
llvm::MemoryBuffer::getMemBuffer(
|
|
test_file.content, test_file.filename,
|
|
/*RequiresNullTerminator=*/false))) {
|
|
return ErrorBuilder() << "File is repeated: " << test_file.filename;
|
|
}
|
|
}
|
|
|
|
// Convert the arguments to StringRef and const char* to match the
|
|
// expectations of PrettyStackTraceProgram and Run.
|
|
llvm::SmallVector<llvm::StringRef> test_args_ref;
|
|
llvm::SmallVector<const char*> test_argv_for_stack_trace;
|
|
test_args_ref.reserve(context.test_args.size());
|
|
test_argv_for_stack_trace.reserve(context.test_args.size() + 1);
|
|
for (const auto& arg : context.test_args) {
|
|
test_args_ref.push_back(arg);
|
|
test_argv_for_stack_trace.push_back(arg.c_str());
|
|
}
|
|
// Add a trailing null so that this is a proper argv.
|
|
test_argv_for_stack_trace.push_back(nullptr);
|
|
|
|
// Add a stack trace entry for the test invocation.
|
|
llvm::PrettyStackTraceProgram stack_trace_entry(
|
|
test_argv_for_stack_trace.size(), test_argv_for_stack_trace.data());
|
|
|
|
// Capture trace streaming, but only when in debug mode.
|
|
llvm::raw_svector_ostream stdout(context.stdout);
|
|
llvm::raw_svector_ostream stderr(context.stderr);
|
|
CARBON_ASSIGN_OR_RETURN(context.exit_with_success,
|
|
Run(test_args_ref, fs, stdout, stderr));
|
|
return Success();
|
|
}
|
|
|
|
auto FileTestBase::DoArgReplacements(
|
|
llvm::SmallVector<std::string>& test_args,
|
|
const llvm::SmallVector<TestFile>& test_files) -> ErrorOr<Success> {
|
|
for (auto* it = test_args.begin(); it != test_args.end(); ++it) {
|
|
auto percent = it->find("%");
|
|
if (percent == std::string::npos) {
|
|
continue;
|
|
}
|
|
|
|
if (percent + 1 >= it->size()) {
|
|
return ErrorBuilder() << "% is not allowed on its own: " << *it;
|
|
}
|
|
char c = (*it)[percent + 1];
|
|
switch (c) {
|
|
case 's': {
|
|
if (*it != "%s") {
|
|
return ErrorBuilder() << "%s must be the full argument: " << *it;
|
|
}
|
|
it = test_args.erase(it);
|
|
for (const auto& file : test_files) {
|
|
it = test_args.insert(it, file.filename);
|
|
++it;
|
|
}
|
|
// Back up once because the for loop will advance.
|
|
--it;
|
|
break;
|
|
}
|
|
case 't': {
|
|
char* tmpdir = getenv("TEST_TMPDIR");
|
|
CARBON_CHECK(tmpdir != nullptr);
|
|
it->replace(percent, 2, llvm::formatv("{0}/temp_file", tmpdir));
|
|
break;
|
|
}
|
|
default:
|
|
return ErrorBuilder() << "%" << c << " is not supported: " << *it;
|
|
}
|
|
}
|
|
return Success();
|
|
}
|
|
|
|
auto FileTestBase::ProcessTestFile(TestContext& context) -> ErrorOr<Success> {
|
|
// Original file content, and a cursor for walking through it.
|
|
llvm::StringRef file_content = context.input_content;
|
|
llvm::StringRef cursor = file_content;
|
|
|
|
// Whether content has been found, only updated before a file split is found
|
|
// (which may be never).
|
|
bool found_content_pre_split = false;
|
|
|
|
// Whether either AUTOUDPATE or NOAUTOUPDATE was found.
|
|
bool found_autoupdate = false;
|
|
|
|
// The index in the current test file. Will be reset on splits.
|
|
int line_index = 0;
|
|
|
|
// The current file name, considering splits. Not set for the default file.
|
|
llvm::StringRef current_file_name;
|
|
|
|
// The current file's start.
|
|
const char* current_file_start = nullptr;
|
|
|
|
int file_number = 0;
|
|
while (!cursor.empty()) {
|
|
auto [line, next_cursor] = cursor.split("\n");
|
|
cursor = next_cursor;
|
|
auto line_trimmed = line.ltrim();
|
|
|
|
static constexpr llvm::StringLiteral SplitPrefix = "// ---";
|
|
if (line_trimmed.consume_front(SplitPrefix)) {
|
|
if (!found_autoupdate) {
|
|
// If there's a split, all output is appended at the end of each file
|
|
// before AUTOUPDATE. We may want to change that, but it's not necessary
|
|
// to handle right now.
|
|
return ErrorBuilder()
|
|
<< "AUTOUPDATE/NOAUTOUPDATE setting must be in the first file.";
|
|
}
|
|
|
|
context.has_splits = true;
|
|
++file_number;
|
|
context.non_check_lines.push_back(FileTestLine(file_number, 0, line));
|
|
// On a file split, add the previous file, then start a new one.
|
|
if (current_file_start) {
|
|
context.test_files.push_back(TestFile(
|
|
current_file_name.str(),
|
|
llvm::StringRef(current_file_start, line_trimmed.begin() -
|
|
current_file_start -
|
|
SplitPrefix.size())));
|
|
} else if (found_content_pre_split) {
|
|
// For the first split, we make sure there was no content prior.
|
|
return ErrorBuilder()
|
|
<< "When using split files, there must be no content before the "
|
|
"first split file.";
|
|
}
|
|
current_file_name = line_trimmed.trim();
|
|
current_file_start = cursor.begin();
|
|
line_index = 0;
|
|
continue;
|
|
} else if (!current_file_start && !line_trimmed.starts_with("//") &&
|
|
!line_trimmed.empty()) {
|
|
found_content_pre_split = true;
|
|
}
|
|
++line_index;
|
|
|
|
// Process expectations when found.
|
|
if (line_trimmed.consume_front("// CHECK")) {
|
|
// Don't build expectations when doing an autoupdate. We don't want to
|
|
// break the autoupdate on an invalid CHECK line.
|
|
if (!absl::GetFlag(FLAGS_autoupdate)) {
|
|
llvm::SmallVector<Matcher<std::string>>* expected = nullptr;
|
|
if (line_trimmed.consume_front(":STDOUT:")) {
|
|
expected = &context.expected_stdout;
|
|
} else if (line_trimmed.consume_front(":STDERR:")) {
|
|
expected = &context.expected_stderr;
|
|
} else {
|
|
return ErrorBuilder() << "Unexpected CHECK in input: " << line.str();
|
|
}
|
|
CARBON_ASSIGN_OR_RETURN(Matcher<std::string> check_matcher,
|
|
TransformExpectation(line_index, line_trimmed));
|
|
expected->push_back(check_matcher);
|
|
}
|
|
} else {
|
|
context.non_check_lines.push_back(
|
|
FileTestLine(file_number, line_index, line));
|
|
if (line_trimmed.consume_front("// ARGS: ")) {
|
|
if (context.test_args.empty()) {
|
|
// Split the line into arguments.
|
|
std::pair<llvm::StringRef, llvm::StringRef> cursor =
|
|
llvm::getToken(line_trimmed);
|
|
while (!cursor.first.empty()) {
|
|
context.test_args.push_back(std::string(cursor.first));
|
|
cursor = llvm::getToken(cursor.second);
|
|
}
|
|
} else {
|
|
return ErrorBuilder()
|
|
<< "ARGS was specified multiple times: " << line.str();
|
|
}
|
|
} else if (line_trimmed == "// AUTOUPDATE" ||
|
|
line_trimmed == "// NOAUTOUPDATE") {
|
|
if (found_autoupdate) {
|
|
return ErrorBuilder()
|
|
<< "Multiple AUTOUPDATE/NOAUTOUPDATE settings found";
|
|
}
|
|
found_autoupdate = true;
|
|
if (line_trimmed == "// AUTOUPDATE") {
|
|
context.autoupdate_line_number = line_index;
|
|
}
|
|
} else if (line_trimmed == "// SET-CHECK-SUBSET") {
|
|
if (!context.check_subset) {
|
|
context.check_subset = true;
|
|
} else {
|
|
return ErrorBuilder()
|
|
<< "SET-CHECK-SUBSET was specified multiple times";
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!found_autoupdate) {
|
|
return ErrorBuilder() << "Missing AUTOUPDATE/NOAUTOUPDATE setting";
|
|
}
|
|
|
|
if (current_file_start) {
|
|
context.test_files.push_back(
|
|
TestFile(current_file_name.str(),
|
|
llvm::StringRef(current_file_start,
|
|
file_content.end() - current_file_start)));
|
|
} else {
|
|
// If no file splitting happened, use the main file as the test file.
|
|
// There will always be a `/` unless tests are in the repo root.
|
|
context.test_files.push_back(TestFile(
|
|
test_name_.drop_front(test_name_.rfind("/") + 1).str(), file_content));
|
|
}
|
|
|
|
// Assume there is always a suffix `\n` in output.
|
|
if (!context.expected_stdout.empty()) {
|
|
context.expected_stdout.push_back(StrEq(""));
|
|
}
|
|
if (!context.expected_stderr.empty()) {
|
|
context.expected_stderr.push_back(StrEq(""));
|
|
}
|
|
|
|
return Success();
|
|
}
|
|
|
|
auto FileTestBase::TransformExpectation(int line_index, llvm::StringRef in)
|
|
-> ErrorOr<Matcher<std::string>> {
|
|
if (in.empty()) {
|
|
return Matcher<std::string>{StrEq("")};
|
|
}
|
|
if (in[0] != ' ') {
|
|
return ErrorBuilder() << "Malformated CHECK line: " << in;
|
|
}
|
|
std::string str = in.substr(1).str();
|
|
for (int pos = 0; pos < static_cast<int>(str.size());) {
|
|
switch (str[pos]) {
|
|
case '(':
|
|
case ')':
|
|
case ']':
|
|
case '}':
|
|
case '.':
|
|
case '^':
|
|
case '$':
|
|
case '*':
|
|
case '+':
|
|
case '?':
|
|
case '|':
|
|
case '\\': {
|
|
// Escape regex characters.
|
|
str.insert(pos, "\\");
|
|
pos += 2;
|
|
break;
|
|
}
|
|
case '[': {
|
|
llvm::StringRef line_keyword_cursor = llvm::StringRef(str).substr(pos);
|
|
if (line_keyword_cursor.consume_front("[[")) {
|
|
static constexpr llvm::StringLiteral LineKeyword = "@LINE";
|
|
if (line_keyword_cursor.consume_front(LineKeyword)) {
|
|
// Allow + or - here; consumeInteger handles -.
|
|
line_keyword_cursor.consume_front("+");
|
|
int offset;
|
|
// consumeInteger returns true for errors, not false.
|
|
if (line_keyword_cursor.consumeInteger(10, offset) ||
|
|
!line_keyword_cursor.consume_front("]]")) {
|
|
return ErrorBuilder()
|
|
<< "Unexpected @LINE offset at `"
|
|
<< line_keyword_cursor.substr(0, 5) << "` in: " << in;
|
|
}
|
|
std::string int_str = llvm::Twine(line_index + offset).str();
|
|
int remove_len = (line_keyword_cursor.data() - str.data()) - pos;
|
|
str.replace(pos, remove_len, int_str);
|
|
pos += int_str.size();
|
|
} else {
|
|
return ErrorBuilder()
|
|
<< "Unexpected [[, should be {{\\[\\[}} at `"
|
|
<< line_keyword_cursor.substr(0, 5) << "` in: " << in;
|
|
}
|
|
} else {
|
|
// Escape the `[`.
|
|
str.insert(pos, "\\");
|
|
pos += 2;
|
|
}
|
|
break;
|
|
}
|
|
case '{': {
|
|
if (pos + 1 == static_cast<int>(str.size()) || str[pos + 1] != '{') {
|
|
// Single `{`, escape it.
|
|
str.insert(pos, "\\");
|
|
pos += 2;
|
|
} else {
|
|
// Replace the `{{...}}` regex syntax with standard `(...)` syntax.
|
|
str.replace(pos, 2, "(");
|
|
for (++pos; pos < static_cast<int>(str.size() - 1); ++pos) {
|
|
if (str[pos] == '}' && str[pos + 1] == '}') {
|
|
str.replace(pos, 2, ")");
|
|
++pos;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
default: {
|
|
++pos;
|
|
}
|
|
}
|
|
}
|
|
|
|
return Matcher<std::string>{MatchesRegex(str)};
|
|
}
|
|
|
|
// Returns the tests to run.
|
|
static auto GetTests() -> llvm::SmallVector<std::string> {
|
|
// Prefer a user-specified list if present.
|
|
auto specific_tests = absl::GetFlag(FLAGS_file_tests);
|
|
if (!specific_tests.empty()) {
|
|
return llvm::SmallVector<std::string>(specific_tests.begin(),
|
|
specific_tests.end());
|
|
}
|
|
|
|
// Extracts tests from the target file.
|
|
CARBON_CHECK(!absl::GetFlag(FLAGS_test_targets_file).empty())
|
|
<< "Missing --test_targets_file.";
|
|
auto content = ReadFile(absl::GetFlag(FLAGS_test_targets_file));
|
|
llvm::SmallVector<std::string> all_tests;
|
|
for (llvm::StringRef file_ref : llvm::split(content, "\n")) {
|
|
if (file_ref.empty()) {
|
|
continue;
|
|
}
|
|
all_tests.push_back(file_ref.str());
|
|
}
|
|
return all_tests;
|
|
}
|
|
|
|
// Implements main() within the Carbon::Testing namespace for convenience.
|
|
static auto Main(int argc, char** argv) -> int {
|
|
absl::ParseCommandLine(argc, argv);
|
|
testing::InitGoogleTest(&argc, argv);
|
|
llvm::setBugReportMsg(
|
|
"Please report issues to "
|
|
"https://github.com/carbon-language/carbon-lang/issues and include the "
|
|
"crash backtrace.\n");
|
|
llvm::InitLLVM init_llvm(argc, argv);
|
|
|
|
if (argc > 1) {
|
|
llvm::errs() << "Unexpected arguments starting at: " << argv[1] << "\n";
|
|
return EXIT_FAILURE;
|
|
}
|
|
|
|
llvm::SmallVector<std::string> tests = GetTests();
|
|
auto test_factory = GetFileTestFactory();
|
|
if (absl::GetFlag(FLAGS_autoupdate)) {
|
|
for (const auto& test_name : tests) {
|
|
std::unique_ptr<FileTestBase> test(test_factory.factory_fn(test_name));
|
|
auto result = test->Autoupdate();
|
|
llvm::errs() << (result.ok() ? (*result ? "!" : ".")
|
|
: result.error().message());
|
|
}
|
|
llvm::errs() << "\nDone!\n";
|
|
return EXIT_SUCCESS;
|
|
} else {
|
|
for (llvm::StringRef test_name : tests) {
|
|
testing::RegisterTest(test_factory.name, test_name.data(), nullptr,
|
|
test_name.data(), __FILE__, __LINE__,
|
|
[&test_factory, test_name = test_name]() {
|
|
return test_factory.factory_fn(test_name);
|
|
});
|
|
}
|
|
return RUN_ALL_TESTS();
|
|
}
|
|
}
|
|
|
|
} // namespace Carbon::Testing
|
|
|
|
auto main(int argc, char** argv) -> int {
|
|
return Carbon::Testing::Main(argc, argv);
|
|
}
|