diff --git a/executable_semantics/BUILD b/executable_semantics/BUILD index 26b7d3196d3f..d0f8af7d0475 100644 --- a/executable_semantics/BUILD +++ b/executable_semantics/BUILD @@ -7,6 +7,7 @@ load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library") load("//bazel/testing:golden_test.bzl", "golden_test") +load("test_list.bzl", "TEST_LIST") cc_binary( name = "executable_semantics", @@ -18,88 +19,6 @@ cc_binary( ], ) -EXAMPLES = [ - "assignment_copy1", - "assignment_copy2", - "block1", - "block2", - "break1", - "choice1", - "continue1", - "fun_named_params", - "fun_named_params2", - "fun_recur", - "fun1", - "fun2", - "fun3", - "fun4", - "fun5", - "fun6_fail_type", - "funptr1", - "generic_function1", - "generic_function2", - "generic_function3", - "generic_function_apply", - "generic_function_fail1", - "generic_function_fail2", - "generic_function_fail3", - "generic_function_swap", - "generic_function_tuple_map", - "global_variable1", - "global_variable2", - "global_variable3", - "global_variable4", - "global_variable5", - "global_variable6", - "global_variable7", - "global_variable8", - "ignored_parameter", - "if1", - "if2", - "if3", - "invalid_char", - "match_any_int", - "match_int_default", - "match_int", - "match_placeholder", - "match_type", - "next", - "pattern_init", - "pattern_variable_fail", - "placeholder_variable", - "record1", - "star", - "struct1", - "struct2", - "struct3", - "tuple_assign", - "tuple_equality", - "tuple_equality2", - "tuple_equality3", - "tuple_match", - "tuple_match2", - "tuple_match3", - "tuple1", - "tuple2", - "tuple3", - "tuple4", - "tuple5", - "type_compute", - "type_compute2", - "type_compute3", - "while1", - "zero", - "experimental_continuation1", - "experimental_continuation2", - "experimental_continuation3", - "experimental_continuation4", - "experimental_continuation5", - "experimental_continuation6", - "experimental_continuation7", - "experimental_continuation8", - "experimental_continuation9", -] - [golden_test( name = "%s_test" % e, cmd = "'$(location executable_semantics) $(location testdata/%s.carbon)'" % e, @@ -112,7 +31,13 @@ EXAMPLES = [ "ASAN_OPTIONS": "detect_leaks=0", }, golden = "testdata/%s.golden" % e, -) for e in EXAMPLES] +) for e in TEST_LIST] + +# Convenience suite for running golden tests. +test_suite( + name = "golden_tests", + tests = [":%s_test" % e for e in TEST_LIST], +) # Test --trace by expecting golden output to be a *subset* of trace output. Note # the normal test must be used to update golden files. @@ -130,4 +55,10 @@ EXAMPLES = [ }, golden = "testdata/%s.golden" % e, golden_is_subset = True, -) for e in EXAMPLES] +) for e in TEST_LIST] + +# Convenience suite for running trace tests. +test_suite( + name = "trace_tests", + tests = [":%s_trace_test" % e for e in TEST_LIST], +) diff --git a/executable_semantics/test_list.bzl b/executable_semantics/test_list.bzl new file mode 100644 index 000000000000..84136a9def3b --- /dev/null +++ b/executable_semantics/test_list.bzl @@ -0,0 +1,87 @@ +# Part of the Carbon Language project, under the Apache License v2.0 with LLVM +# Exceptions. See /LICENSE for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +"""Auto-generated list of tests. Run `./tests.py --update_list` to update.""" + +TEST_LIST = [ + "assignment_copy1", + "assignment_copy2", + "block1", + "block2", + "break1", + "choice1", + "continue1", + "experimental_continuation1", + "experimental_continuation2", + "experimental_continuation3", + "experimental_continuation4", + "experimental_continuation5", + "experimental_continuation6", + "experimental_continuation7", + "experimental_continuation8", + "experimental_continuation9", + "fun1", + "fun2", + "fun3", + "fun4", + "fun5", + "fun6_fail_type", + "fun_named_params", + "fun_named_params2", + "fun_recur", + "funptr1", + "generic_function1", + "generic_function2", + "generic_function3", + "generic_function_apply", + "generic_function_fail1", + "generic_function_fail2", + "generic_function_fail3", + "generic_function_swap", + "generic_function_tuple_map", + "global_variable1", + "global_variable2", + "global_variable3", + "global_variable4", + "global_variable5", + "global_variable6", + "global_variable7", + "global_variable8", + "if1", + "if2", + "if3", + "ignored_parameter", + "invalid_char", + "match_any_int", + "match_int", + "match_int_default", + "match_placeholder", + "match_type", + "next", + "pattern_init", + "pattern_variable_fail", + "placeholder_variable", + "record1", + "star", + "struct1", + "struct2", + "struct3", + "tuple1", + "tuple2", + "tuple3", + "tuple4", + "tuple5", + "tuple_assign", + "tuple_equality", + "tuple_equality2", + "tuple_equality3", + "tuple_match", + "tuple_match2", + "tuple_match3", + "type_compute", + "type_compute2", + "type_compute3", + "while1", + "zero", +] diff --git a/executable_semantics/tests.py b/executable_semantics/tests.py new file mode 100755 index 000000000000..374594b40a32 --- /dev/null +++ b/executable_semantics/tests.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 + +"""Helps manage tests.""" + +__copyright__ = """ +Part of the Carbon Language project, under the Apache License v2.0 with LLVM +Exceptions. See /LICENSE for license information. +SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +""" + +import argparse +from concurrent import futures +import os +import re +import subprocess +import sys + +_TESTDATA = "./testdata" +_TEST_LIST_BZL = "./test_list.bzl" + +_TEST_LIST_HEADER = """ +# Part of the Carbon Language project, under the Apache License v2.0 with LLVM +# Exceptions. See /LICENSE for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +\"""Auto-generated list of tests. Run `./tests.py --update_list` to update.\""" + +TEST_LIST = [ +""" + +_TEST_LIST_FOOTER = """ +] +""" + + +def _parse_args(args=None): + """Parses command-line arguments and flags.""" + parser = argparse.ArgumentParser(description=__doc__) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument( + "--update_all", + action="store_true", + help="Runs all updates.", + ) + group.add_argument( + "--update_goldens", + action="store_true", + help="Updates golden files by running executable_semantics.", + ) + group.add_argument( + "--update_list", action="store_true", help="Updates test_list.bzl." + ) + parsed_args = parser.parse_args(args=args) + return parsed_args + + +def _update_list(): + """Updates test_list.bzl.""" + # Get the list of tests and goldens from the filesystem. + tests = set() + goldens = set() + for f in os.listdir(_TESTDATA): + basename, ext = os.path.splitext(f) + if ext == ".carbon": + tests.add(basename) + elif ext == ".golden": + goldens.add(basename) + else: + sys.exit("Unrecognized file type in testdata: %s" % f) + + # Update test_list.bzl if needed, creating any missing golden files too. + test_list = _TEST_LIST_HEADER.lstrip("\n") + for test in sorted(tests): + test_list += ' "%s",\n' % test + if test not in goldens: + print("Creating empty golden '%s.golden' for test." % test) + open(os.path.join(_TESTDATA, "%s.golden" % test), "w").close() + test_list += _TEST_LIST_FOOTER.lstrip("\n") + bzl_content = open(_TEST_LIST_BZL).read() + if bzl_content != test_list: + print("Updating test_list.bzl") + with open(_TEST_LIST_BZL, "w") as bzl: + bzl.write(test_list) + else: + print("test_list.bzl is up-to-date") + + # Garbage collect unnecessary golden files. + for golden in sorted(goldens): + if golden not in tests: + print( + "Removing golden '%s.golden' because it has no test." % golden + ) + os.unlink(os.path.join(_TESTDATA, golden)) + + +def _update_golden(test): + """Updates the golden file for `test` by running executable_semantics.""" + # TODO(#580): Remove this when leaks are fixed. + env = os.environ.copy() + env["ASAN_OPTIONS"] = "detect_leaks=0" + # Invoke the test update directly in order to allow parallel execution + # (`bazel run` will serialize). + p = subprocess.run( + [ + "../bazel-bin/executable_semantics/%s_test" % test, + "./testdata/%s.golden" % test, + "../bazel-bin/executable_semantics/executable_semantics " + + "./testdata/%s.carbon" % test, + "--update", + ], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + if p.returncode != 0: + out = p.stdout.decode("utf-8") + print(out, file=sys.stderr, end="") + sys.exit("ERROR: Updating test '%s' failed" % test) + print(".", end="", flush=True) + + +def _update_goldens(): + """Runs bazel to update golden files.""" + # Load tests from the bzl file. This isn't done through os.listdir because + # building new tests requires --update_list. + bzl_content = open(_TEST_LIST_BZL).read() + tests = re.findall(r'"(\w+)",', bzl_content) + + # Build all tests at once in order to allow parallel updates. + print("Building tests...") + subprocess.check_call( + ["bazel", "build", "//executable_semantics:golden_tests"] + ) + + print("Updating %d goldens..." % len(tests)) + with futures.ThreadPoolExecutor() as exec: + results = [exec.submit(lambda: _update_golden(test)) for test in tests] + # Propagate exceptions. + for result in results: + result.result() + # Each golden indicates progress with a dot without a newline, so put a + # newline to wrap. + print("\nUpdated goldens.") + + +def main(): + parsed_args = _parse_args() + if parsed_args.update_all or parsed_args.update_list: + _update_list() + if parsed_args.update_all or parsed_args.update_goldens: + _update_goldens() + + +if __name__ == "__main__": + main()