diff --git a/bazel/testing/BUILD b/bazel/testing/BUILD index 8261f52b45ad..79f78bf76308 100644 --- a/bazel/testing/BUILD +++ b/bazel/testing/BUILD @@ -2,4 +2,4 @@ # Exceptions. See /LICENSE for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -exports_files(["golden_test.sh"]) +exports_files(["golden_test.py"]) diff --git a/bazel/testing/golden_test.bzl b/bazel/testing/golden_test.bzl index 84577f0e29e2..3f4c11d32829 100644 --- a/bazel/testing/golden_test.bzl +++ b/bazel/testing/golden_test.bzl @@ -4,22 +4,27 @@ """Rule for a golden test.""" -def golden_test(name, golden, subject, **kwargs): +def golden_test(name, golden, cmd, data, **kwargs): """Compares two files. Passes if they are identical. Args: name: Name of the build rule. - subject: The generated file to be compared. - golden: The golden file to be compared. - **kwargs: Any additional parameters for the generated sh_test. + cmd: The command whose output is being tested. + golden: The golden file to be compared against the command output. + **kwargs: Any additional parameters for the generated py_test. """ - native.sh_test( + native.py_test( name = name, - srcs = ["//bazel/testing:golden_test.sh"], + srcs = ["//bazel/testing:golden_test.py"], + main = "//bazel/testing:golden_test.py", args = [ "$(location %s)" % golden, - "$(location %s)" % subject, + cmd, ], - data = [golden, subject], + data = [golden] + data, + env = { + # TODO(#580): Remove this when leaks are fixed. + "ASAN_OPTIONS": "detect_leaks=0", + }, **kwargs ) diff --git a/bazel/testing/golden_test.py b/bazel/testing/golden_test.py new file mode 100644 index 000000000000..4dc3cb817215 --- /dev/null +++ b/bazel/testing/golden_test.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 + +"""Compare a command's output against an expected "golden" output file. + +Usage: + +golden_test.py [--update] + + is the path to the golden file, and is +the command to run, including any arguments. If --update is specified, +the command will be run and its output stored in the golden file. +Otherwise, the command will be run and its output compared against +the contents of the golden file. + +For these purposes, the command's output consists of the interleaved +contents of stdout and stderr, as well as the command's exit code. Thus, +golden tests can provide coverage of cases where the command is expected +to fail, as well as cases where it's expected to succeed. + +This script is designed to be run by a `golden_test` Bazel rule, +and may not work when run outside that context. +""" + +__copyright__ = """ +Part of the Carbon Language project, under the Apache License v2.0 with LLVM +Exceptions. See /LICENSE for license information. +SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +""" + +import os +import subprocess +import sys + +golden_path = sys.argv[1] +subject_cmd_args = sys.argv[2].split() + +subject_cmd = subprocess.run( + args=subject_cmd_args, + stdout=subprocess.PIPE, # Capture stdout as a string + stderr=subprocess.STDOUT, # Send stderr to the same place as stdout + universal_newlines=True, +) + +subject = subject_cmd.stdout +if subject_cmd.returncode != 0: + subject += "EXIT CODE: {0}\n".format(subject_cmd.returncode) + +if len(sys.argv) == 4 and sys.argv[3] == "--update": + with open(golden_path, "w") as golden: + golden.write(subject) + sys.exit(0) + +# TODO: consider using difflib instead of a subprocess +diff_cmd = subprocess.run( + args=["diff", "-u", golden_path, "-"], + input=subject, + universal_newlines=True, +) +if diff_cmd.returncode == 0: + print("PASS") + sys.exit(0) + +error_output = """When running under: + {dir} +the golden contents of: + {golden_path} +do not match generated output of: + {subject_cmd_args} + +To update the golden file, run the following: + + bazel run ${test_target} -- --update +""".format( + dir=os.getenv("TEST_SRCDIR"), + golden_path=golden_path, + subject_cmd_args=sys.argv[2], + test_target=os.getenv("TEST_TARGET"), +) + +print(error_output) +sys.exit(1) diff --git a/bazel/testing/golden_test.sh b/bazel/testing/golden_test.sh deleted file mode 100755 index e13591f9dace..000000000000 --- a/bazel/testing/golden_test.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# Part of the Carbon Language project, under the Apache License v2.0 with LLVM -# Exceptions. See /LICENSE for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -set -e -u -o pipefail - -GOLDEN=$1 -SUBJECT=$2 - -if [[ $# == 3 && $3 == "--update" ]]; then - cp "${SUBJECT}" "${GOLDEN}" - exit $? -fi - -CMD=("diff" "-u" "${GOLDEN}" "${SUBJECT}") - -if "${CMD[@]}"; then - echo "PASS" - exit 0 -fi - -cat <GetFieldAccess().aggregate, types, values, - nullptr, TCContext::ValueContext); + auto res = TypeCheckExp(e->GetIndex().aggregate, types, values, nullptr, + TCContext::ValueContext); auto t = res.type; switch (t->tag) { case ValKind::TupleV: { diff --git a/executable_semantics/syntax/BUILD b/executable_semantics/syntax/BUILD index 64e52689dc70..31fda0c45bfa 100644 --- a/executable_semantics/syntax/BUILD +++ b/executable_semantics/syntax/BUILD @@ -47,7 +47,7 @@ cc_test( name = "paren_contents_test", srcs = ["paren_contents_test.cpp"], env = { - # FIXME: Remove this when leaks are fixed. + # TODO(#580): Remove this when leaks are fixed. "ASAN_OPTIONS": "detect_leaks=0", }, deps = [