From 165651c75c7a0a132ebe503662f481eb5d83566e Mon Sep 17 00:00:00 2001 From: Jon Meow <46229924+jonmeow@users.noreply.github.com> Date: Mon, 26 Jul 2021 14:57:00 -0700 Subject: [PATCH] Add script to automate mass test updates (#674) Trying to simplify the process of updating golden output, and ensuring that we aren't missing any tests. With this script, it should be roughly "Add the .carbon file, then run tests.py --update_all" Co-authored-by: Geoff Romer --- executable_semantics/BUILD | 99 +++--------------- executable_semantics/test_list.bzl | 87 ++++++++++++++++ executable_semantics/tests.py | 155 +++++++++++++++++++++++++++++ 3 files changed, 257 insertions(+), 84 deletions(-) create mode 100644 executable_semantics/test_list.bzl create mode 100755 executable_semantics/tests.py diff --git a/executable_semantics/BUILD b/executable_semantics/BUILD index 26b7d3196d3f..d0f8af7d0475 100644 --- a/executable_semantics/BUILD +++ b/executable_semantics/BUILD @@ -7,6 +7,7 @@ load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library") load("//bazel/testing:golden_test.bzl", "golden_test") +load("test_list.bzl", "TEST_LIST") cc_binary( name = "executable_semantics", @@ -18,88 +19,6 @@ cc_binary( ], ) -EXAMPLES = [ - "assignment_copy1", - "assignment_copy2", - "block1", - "block2", - "break1", - "choice1", - "continue1", - "fun_named_params", - "fun_named_params2", - "fun_recur", - "fun1", - "fun2", - "fun3", - "fun4", - "fun5", - "fun6_fail_type", - "funptr1", - "generic_function1", - "generic_function2", - "generic_function3", - "generic_function_apply", - "generic_function_fail1", - "generic_function_fail2", - "generic_function_fail3", - "generic_function_swap", - "generic_function_tuple_map", - "global_variable1", - "global_variable2", - "global_variable3", - "global_variable4", - "global_variable5", - "global_variable6", - "global_variable7", - "global_variable8", - "ignored_parameter", - "if1", - "if2", - "if3", - "invalid_char", - "match_any_int", - "match_int_default", - "match_int", - "match_placeholder", - "match_type", - "next", - "pattern_init", - "pattern_variable_fail", - "placeholder_variable", - "record1", - "star", - "struct1", - "struct2", - "struct3", - "tuple_assign", - "tuple_equality", - "tuple_equality2", - "tuple_equality3", - "tuple_match", - "tuple_match2", - "tuple_match3", - "tuple1", - "tuple2", - "tuple3", - "tuple4", - "tuple5", - "type_compute", - "type_compute2", - "type_compute3", - "while1", - "zero", - "experimental_continuation1", - "experimental_continuation2", - "experimental_continuation3", - "experimental_continuation4", - "experimental_continuation5", - "experimental_continuation6", - "experimental_continuation7", - "experimental_continuation8", - "experimental_continuation9", -] - [golden_test( name = "%s_test" % e, cmd = "'$(location executable_semantics) $(location testdata/%s.carbon)'" % e, @@ -112,7 +31,13 @@ EXAMPLES = [ "ASAN_OPTIONS": "detect_leaks=0", }, golden = "testdata/%s.golden" % e, -) for e in EXAMPLES] +) for e in TEST_LIST] + +# Convenience suite for running golden tests. +test_suite( + name = "golden_tests", + tests = [":%s_test" % e for e in TEST_LIST], +) # Test --trace by expecting golden output to be a *subset* of trace output. Note # the normal test must be used to update golden files. @@ -130,4 +55,10 @@ EXAMPLES = [ }, golden = "testdata/%s.golden" % e, golden_is_subset = True, -) for e in EXAMPLES] +) for e in TEST_LIST] + +# Convenience suite for running trace tests. +test_suite( + name = "trace_tests", + tests = [":%s_trace_test" % e for e in TEST_LIST], +) diff --git a/executable_semantics/test_list.bzl b/executable_semantics/test_list.bzl new file mode 100644 index 000000000000..84136a9def3b --- /dev/null +++ b/executable_semantics/test_list.bzl @@ -0,0 +1,87 @@ +# Part of the Carbon Language project, under the Apache License v2.0 with LLVM +# Exceptions. See /LICENSE for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +"""Auto-generated list of tests. Run `./tests.py --update_list` to update.""" + +TEST_LIST = [ + "assignment_copy1", + "assignment_copy2", + "block1", + "block2", + "break1", + "choice1", + "continue1", + "experimental_continuation1", + "experimental_continuation2", + "experimental_continuation3", + "experimental_continuation4", + "experimental_continuation5", + "experimental_continuation6", + "experimental_continuation7", + "experimental_continuation8", + "experimental_continuation9", + "fun1", + "fun2", + "fun3", + "fun4", + "fun5", + "fun6_fail_type", + "fun_named_params", + "fun_named_params2", + "fun_recur", + "funptr1", + "generic_function1", + "generic_function2", + "generic_function3", + "generic_function_apply", + "generic_function_fail1", + "generic_function_fail2", + "generic_function_fail3", + "generic_function_swap", + "generic_function_tuple_map", + "global_variable1", + "global_variable2", + "global_variable3", + "global_variable4", + "global_variable5", + "global_variable6", + "global_variable7", + "global_variable8", + "if1", + "if2", + "if3", + "ignored_parameter", + "invalid_char", + "match_any_int", + "match_int", + "match_int_default", + "match_placeholder", + "match_type", + "next", + "pattern_init", + "pattern_variable_fail", + "placeholder_variable", + "record1", + "star", + "struct1", + "struct2", + "struct3", + "tuple1", + "tuple2", + "tuple3", + "tuple4", + "tuple5", + "tuple_assign", + "tuple_equality", + "tuple_equality2", + "tuple_equality3", + "tuple_match", + "tuple_match2", + "tuple_match3", + "type_compute", + "type_compute2", + "type_compute3", + "while1", + "zero", +] diff --git a/executable_semantics/tests.py b/executable_semantics/tests.py new file mode 100755 index 000000000000..374594b40a32 --- /dev/null +++ b/executable_semantics/tests.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 + +"""Helps manage tests.""" + +__copyright__ = """ +Part of the Carbon Language project, under the Apache License v2.0 with LLVM +Exceptions. See /LICENSE for license information. +SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +""" + +import argparse +from concurrent import futures +import os +import re +import subprocess +import sys + +_TESTDATA = "./testdata" +_TEST_LIST_BZL = "./test_list.bzl" + +_TEST_LIST_HEADER = """ +# Part of the Carbon Language project, under the Apache License v2.0 with LLVM +# Exceptions. See /LICENSE for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +\"""Auto-generated list of tests. Run `./tests.py --update_list` to update.\""" + +TEST_LIST = [ +""" + +_TEST_LIST_FOOTER = """ +] +""" + + +def _parse_args(args=None): + """Parses command-line arguments and flags.""" + parser = argparse.ArgumentParser(description=__doc__) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument( + "--update_all", + action="store_true", + help="Runs all updates.", + ) + group.add_argument( + "--update_goldens", + action="store_true", + help="Updates golden files by running executable_semantics.", + ) + group.add_argument( + "--update_list", action="store_true", help="Updates test_list.bzl." + ) + parsed_args = parser.parse_args(args=args) + return parsed_args + + +def _update_list(): + """Updates test_list.bzl.""" + # Get the list of tests and goldens from the filesystem. + tests = set() + goldens = set() + for f in os.listdir(_TESTDATA): + basename, ext = os.path.splitext(f) + if ext == ".carbon": + tests.add(basename) + elif ext == ".golden": + goldens.add(basename) + else: + sys.exit("Unrecognized file type in testdata: %s" % f) + + # Update test_list.bzl if needed, creating any missing golden files too. + test_list = _TEST_LIST_HEADER.lstrip("\n") + for test in sorted(tests): + test_list += ' "%s",\n' % test + if test not in goldens: + print("Creating empty golden '%s.golden' for test." % test) + open(os.path.join(_TESTDATA, "%s.golden" % test), "w").close() + test_list += _TEST_LIST_FOOTER.lstrip("\n") + bzl_content = open(_TEST_LIST_BZL).read() + if bzl_content != test_list: + print("Updating test_list.bzl") + with open(_TEST_LIST_BZL, "w") as bzl: + bzl.write(test_list) + else: + print("test_list.bzl is up-to-date") + + # Garbage collect unnecessary golden files. + for golden in sorted(goldens): + if golden not in tests: + print( + "Removing golden '%s.golden' because it has no test." % golden + ) + os.unlink(os.path.join(_TESTDATA, golden)) + + +def _update_golden(test): + """Updates the golden file for `test` by running executable_semantics.""" + # TODO(#580): Remove this when leaks are fixed. + env = os.environ.copy() + env["ASAN_OPTIONS"] = "detect_leaks=0" + # Invoke the test update directly in order to allow parallel execution + # (`bazel run` will serialize). + p = subprocess.run( + [ + "../bazel-bin/executable_semantics/%s_test" % test, + "./testdata/%s.golden" % test, + "../bazel-bin/executable_semantics/executable_semantics " + + "./testdata/%s.carbon" % test, + "--update", + ], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + if p.returncode != 0: + out = p.stdout.decode("utf-8") + print(out, file=sys.stderr, end="") + sys.exit("ERROR: Updating test '%s' failed" % test) + print(".", end="", flush=True) + + +def _update_goldens(): + """Runs bazel to update golden files.""" + # Load tests from the bzl file. This isn't done through os.listdir because + # building new tests requires --update_list. + bzl_content = open(_TEST_LIST_BZL).read() + tests = re.findall(r'"(\w+)",', bzl_content) + + # Build all tests at once in order to allow parallel updates. + print("Building tests...") + subprocess.check_call( + ["bazel", "build", "//executable_semantics:golden_tests"] + ) + + print("Updating %d goldens..." % len(tests)) + with futures.ThreadPoolExecutor() as exec: + results = [exec.submit(lambda: _update_golden(test)) for test in tests] + # Propagate exceptions. + for result in results: + result.result() + # Each golden indicates progress with a dot without a newline, so put a + # newline to wrap. + print("\nUpdated goldens.") + + +def main(): + parsed_args = _parse_args() + if parsed_args.update_all or parsed_args.update_list: + _update_list() + if parsed_args.update_all or parsed_args.update_goldens: + _update_goldens() + + +if __name__ == "__main__": + main()