carbon-language/carbon-lang
1
0
Fork 0
mirror of https://github.com/carbon-language/carbon-lang.git synced 2026-09-24 20:40:11 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Refactor update_checks into a more generic lit_autoupdate (#2277)

Browse Source 
I've refactored the script in order to make it work in more contexts, which is why the delta is lost. I've actually refactored a significant amount with the intent of making the logic easier to understand, because I was also adjusting bits of it.

Some key notes:

- Removes the multi-pass update that was dealing with unfixed line numbers in explorer (I think the current script should work in one pass)
  - Fixed explorer to handle multiple line numbers on the same line (turns out we can rely on local format for line numbers).
- Using execv instead of imports because making Python imports work in a setup like this feels like it's not worth it; only a nuisance.
- Adding __init__.py to satisfy mypy, which otherwise considers the lit_autoupdate.py scripts to be issues.
- Using py because I was thinking sh would be more platform-dependent. py should port better to Windows.
- Getting rid of [[ID#]] capture groups in the semantics-ir tests because with the autoupdate it's kind of moot (also, hard to autogenerate the pairs without relying on the %### value).

Note this does mean tests switch to more of a "make a change, see which tests change" setup. I don't know that that's a _bad_ thing though -- it's pretty much how tests are being written right now, which is why I went down this rabbit hole. It's a nuisance to make a change then _manually_ have to update a bunch of code.

My intent is to use this for to convert parse-tree tests to lit, but I wanted to do this with _existing_ tests first as a proof of concept and to make sure there's agreement.
This commit is contained in:
Jon Ross-Perkins
2022-10-12 15:01:38 -07:00
committed by GitHub
parent 61552a788b
commit 55e124a667
30 changed files with 602 additions and 338 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bazel/testing/__init__.py
View File
bazel/testing/lit_autoupdate_base.py
Executable
+416
View File
@@ -0,0 +1,416 @@
#!/usr/bin/env python3
"""Updates the CHECK: lines in lit tests based on the AUTOUPDATE line."""
__copyright__ = """
Part of the Carbon Language project, under the Apache License v2.0 with LLVM
Exceptions. See /LICENSE for license information.
SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
"""
from abc import ABC, abstractmethod
import argparse
from concurrent import futures
import os
from pathlib import Path
import re
import subprocess
from typing import Any, Dict, List, NamedTuple, Optional, Pattern, Set, Tuple
# A prefix followed by a command to run for autoupdating checked output.
AUTOUPDATE_MARKER = "// AUTOUPDATE: "
# Indicates no autoupdate is requested.
NOAUTOUPDATE_MARKER = "// NOAUTOUPDATE"
class ParsedArgs(NamedTuple):
build_mode: str
build_target: str
cmd_replace: Tuple[str, str]
extra_check_replacements: List[Tuple[Pattern, Pattern, str]]
line_number_format: str
line_number_pattern: Pattern
testdata: str
tests: List[Path]
def parse_args() -> ParsedArgs:
"""Parses command-line arguments and flags."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("tests", nargs="*")
parser.add_argument(
"--build_mode",
metavar="MODE",
default="opt",
help="The build mode to use. Defaults to opt for faster execution.",
)
parser.add_argument(
"--build_target",
metavar="TARGET",
required=True,
help="The target to build.",
)
parser.add_argument(
"--cmd_replace",
nargs=2,
metavar=("BEFORE", "AFTER"),
required=True,
help="Adds a command replacement of `BEFORE` with `AFTER`. Typically "
"`BEFORE` will look like `%{name}`",
)
parser.add_argument(
"--extra_check_replacement",
nargs=3,
metavar=("MATCHING", "BEFORE", "AFTER"),
default=[],
action="append",
help="On a CHECK line with MATCHING, does a regex replacement of "
"BEFORE with AFTER.",
)
parser.add_argument(
"--line_number_format",
metavar="FORMAT",
default="[[@LINE%(delta)s]]",
help="An optional format string for line number delta replacements.",
)
parser.add_argument(
"--line_number_pattern",
metavar="PATTERN",
required=True,
help="A regular expression which matches line numbers to update as its "
"only group.",
)
parser.add_argument(
"--testdata",
metavar="PATH",
required=True,
help="The path to the testdata to update, relative to the workspace "
"root.",
)
parsed_args = parser.parse_args()
extra_check_replacements = [
(re.compile(line_matcher), re.compile(before), after)
for line_matcher, before, after in parsed_args.extra_check_replacement
]
return ParsedArgs(
build_mode=parsed_args.build_mode,
build_target=parsed_args.build_target,
cmd_replace=parsed_args.cmd_replace,
extra_check_replacements=extra_check_replacements,
line_number_format=parsed_args.line_number_format,
line_number_pattern=re.compile(parsed_args.line_number_pattern),
testdata=parsed_args.testdata,
tests=[Path(test).resolve() for test in parsed_args.tests],
)
def get_tests(testdata: str) -> Set[Path]:
"""Get the list of tests from the filesystem."""
tests = set()
for root, _, files in os.walk(testdata):
for f in files:
if f in {"lit.cfg.py", "BUILD"}:
# Ignore the lit config.
continue
if os.path.splitext(f)[1] == ".carbon":
tests.add(Path(root).joinpath(f))
else:
exit(f"Unrecognized file type in testdata: {f}")
return tests
class Line(ABC):
"""A line that may appear in the resulting test file."""
@abstractmethod
def format(
self, *, output_line_number: int, line_number_remap: Dict[int, int]
) -> str:
raise NotImplementedError
class OriginalLine(Line):
"""A line that was copied from the original test file."""
def __init__(self, line_number: int, text: str) -> None:
self.line_number = line_number
self.text = text
def format(self, **kwargs: Any) -> str:
return self.text
class CheckLine(Line):
"""A `// CHECK:` line generated from the test output.
If there's a line number, it'll be fixed up after we've figured out which
lines to include in the resulting test file and in what order, because
their contents depend on where an original input line appears in the output.
"""
def __init__(
self,
out_line: str,
line_number_format: str,
line_number_pattern: Pattern,
) -> None:
super().__init__()
self.indent = ""
self.out_line = out_line.rstrip()
self.line_number_format = line_number_format
self.line_number_pattern = line_number_pattern
self.line_numbers = [
int(n) - 1 for n in line_number_pattern.findall(self.out_line)
]
def format(
self, *, output_line_number: int, line_number_remap: Dict[int, int]
) -> str:
if not self.out_line:
return f"{self.indent}// CHECK-EMPTY:\n"
result = self.out_line
for line_number in self.line_numbers:
delta = line_number_remap[line_number] - output_line_number
# We use `:+d` here to produce `LINE-n` or `LINE+n` as appropriate.
result = self.line_number_pattern.sub(
self.line_number_format % {"delta": f"{delta:+d}"},
result,
count=1,
)
return f"{self.indent}// CHECK:{result}\n"
class Autoupdate(NamedTuple):
line_number: int
cmd: str
def find_autoupdate(test: str, orig_lines: List[str]) -> Optional[Autoupdate]:
"""Figures out whether autoupdate should occur.
For AUTOUPDATE, returns the line and command. For NOAUTOUPDATE, returns
None.
"""
found = 0
result = None
for line_number, line in enumerate(orig_lines):
if line.startswith(AUTOUPDATE_MARKER):
found += 1
result = Autoupdate(line_number, line[len(AUTOUPDATE_MARKER) :])
elif line.startswith(NOAUTOUPDATE_MARKER):
found += 1
if found == 0:
raise ValueError(
f"{test} must have either '{AUTOUPDATE_MARKER}' or "
f"'{NOAUTOUPDATE_MARKER}'"
)
elif found > 1:
raise ValueError(
f"{test} must have only one of '{AUTOUPDATE_MARKER}' or "
f"'{NOAUTOUPDATE_MARKER}'"
)
return result
def replace_all(s: str, replacements: List[Tuple[str, str]]) -> str:
"""Runs multiple replacements on a string."""
for before, after in replacements:
s = s.replace(before, after)
return s
def get_matchable_test_output(
parsed_args: ParsedArgs,
test: str,
autoupdate_cmd: str,
extra_check_replacements: List[Tuple[Pattern, Pattern, str]],
) -> List[str]:
"""Runs the autoupdate command and returns the output lines."""
# Mirror lit.cfg.py substitutions; bazel runs don't need --prelude.
# Also replaces `%s` with the test file.
autoupdate_cmd = replace_all(
autoupdate_cmd, [parsed_args.cmd_replace, ("%s", test)]
)
# Run the autoupdate command to generate output.
# (`bazel run` would serialize)
p = subprocess.run(
autoupdate_cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
out = p.stdout.decode("utf-8")
# `lit` uses full paths to the test file, so use a regex to ignore paths
# when used.
out = replace_all(
out,
[
("{{", "{{[{][{]}}"),
("[[", "{{[[][[]}}"),
# TODO: Maybe revisit and see if lit can be convinced to give a
# root-relative path.
(test, f"{{{{.*}}}}/{test}"),
],
)
out_lines = out.splitlines()
for i, line in enumerate(out_lines):
for line_matcher, before, after in extra_check_replacements:
if line_matcher.match(line):
out_lines[i] = before.sub(after, line)
return out_lines
def merge_lines(
line_number_format: str,
line_number_pattern: Pattern,
autoupdate_line_number: int,
raw_orig_lines: List[str],
out_lines: List[str],
) -> List[Line]:
"""Merges the original output and new lines."""
orig_lines = [
OriginalLine(i, line)
for i, line in enumerate(raw_orig_lines)
# Remove CHECK lines in the original output.
if not line.lstrip().startswith("// CHECK")
]
check_lines = [
CheckLine(out_line, line_number_format, line_number_pattern)
for out_line in out_lines
]
result_lines: List[Line] = []
# CHECK lines must go after AUTOUPDATE.
while orig_lines and orig_lines[0].line_number <= autoupdate_line_number:
result_lines.append(orig_lines.pop(0))
# Interleave the original lines and the CHECK: lines.
while orig_lines and check_lines:
# Original lines go first when the CHECK line is known and later.
if (
check_lines[0].line_numbers
and check_lines[0].line_numbers[0] > orig_lines[0].line_number
):
result_lines.append(orig_lines.pop(0))
else:
check_line = check_lines.pop(0)
# Indent to match the next original line.
check_line.indent = re.findall("^ *", orig_lines[0].text)[0]
result_lines.append(check_line)
# One list is non-empty; append remaining lines from both to catch it.
result_lines.extend(orig_lines)
result_lines.extend(check_lines)
return result_lines
def update_check(parsed_args: ParsedArgs, test: Path) -> bool:
"""Updates the CHECK: lines for `test` by running explorer.
Returns true if a change was made.
"""
with test.open() as f:
orig_lines = f.readlines()
# Make sure we're supposed to autoupdate.
autoupdate = find_autoupdate(str(test), orig_lines)
if autoupdate is None:
return False
# Determine the merged output lines.
out_lines = get_matchable_test_output(
parsed_args,
str(test),
autoupdate.cmd,
parsed_args.extra_check_replacements,
)
result_lines = merge_lines(
parsed_args.line_number_format,
parsed_args.line_number_pattern,
autoupdate.line_number,
orig_lines,
out_lines,
)
# Calculate the remap for original lines.
line_number_remap = dict(
[
(line.line_number, i)
for i, line in enumerate(result_lines)
if isinstance(line, OriginalLine)
]
)
# If the last line of the original output was a CHECK, replace it with an
# empty line.
if orig_lines[-1].lstrip().startswith("// CHECK"):
line_number_remap[len(orig_lines) - 1] = len(result_lines) - 1
# Generate contents for any lines that depend on line numbers.
formatted_result_lines = [
line.format(output_line_number=i, line_number_remap=line_number_remap)
for i, line in enumerate(result_lines)
]
# If nothing's changed, we're done.
if formatted_result_lines == orig_lines:
return False
# Interleave the new CHECK: lines with the tested content.
with test.open("w") as f:
f.writelines(formatted_result_lines)
return True
def update_checks(parsed_args: ParsedArgs, tests: Set[Path]) -> None:
"""Updates CHECK: lines in lit tests."""
def map_helper(test: Path) -> bool:
updated = update_check(parsed_args, test)
print(".", end="", flush=True)
return updated
print(f"Updating {len(tests)} lit test(s)...")
with futures.ThreadPoolExecutor() as exec:
# list() iterates in order to immediately propagate exceptions.
results = list(exec.map(map_helper, tests))
# Each update call indicates progress with a dot without a newline, so put a
# newline to wrap.
print(f"\nUpdated {results.count(True)} lit test(s).")
def main() -> None:
# Parse arguments relative to the working directory.
parsed_args = parse_args()
# Remaining script logic should be relative to the repository root.
os.chdir(Path(__file__).parent.parent.parent)
if parsed_args.tests:
tests = set(parsed_args.tests)
else:
print("HINT: run `update_checks.py f1 f2 ...` to update specific tests")
tests = get_tests(parsed_args.testdata)
# Build inputs.
print("Building explorer...")
subprocess.check_call(
[
"bazel",
"build",
"-c",
parsed_args.build_mode,
parsed_args.build_target,
]
)
# Run updates.
update_checks(parsed_args, tests)
if __name__ == "__main__":
main()
explorer/README.md
+2 -2
View File
@@ -99,7 +99,7 @@ To explain this boilerplate:
[`lit` substitution](https://llvm.org/docs/CommandGuide/lit.html#substitutions)
for the path to the given test file.
- The `AUTOUPDATE` line indicates that `CHECK` lines will be automatically
inserted immediately below by the `./update_checks.py` script.
inserted immediately below by the `./lit_autoupdate.py` script.
- The `CHECK` lines indicate expected output, verified by `FileCheck`.
- Where a `CHECK` line contains text like `{{.*}}`, the double curly
braces indicate a contained regular expression.
@@ -107,7 +107,7 @@ To explain this boilerplate:
### Useful commands
- `./update_checks.py` -- Updates expected output.
- `./lit_autodupate.py` -- Updates expected output.
- `bazel test ... --test_output=errors` -- Runs tests and prints any errors.
### Updating fuzzer logic after making AST changes
explorer/lit_autoupdate.py
Executable
+39
View File
@@ -0,0 +1,39 @@
#!/usr/bin/env python3
"""Updates the CHECK: lines in lit tests based on the AUTOUPDATE line."""
__copyright__ = """
Part of the Carbon Language project, under the Apache License v2.0 with LLVM
Exceptions. See /LICENSE for license information.
SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
"""
import os
import sys
from pathlib import Path
def main() -> None:
# Calls the main script with explorer settings. This uses execv in order to
# avoid Python import behaviors.
actual_py = Path(__file__).parent.parent.joinpath(
"bazel", "testing", "lit_autoupdate_base.py"
)
args = [
sys.argv[0],
# Flags to configure for explorer testing.
"--build_target",
"//explorer",
"--cmd_replace",
"%{explorer}",
"./bazel-bin/explorer/explorer",
"--testdata",
"explorer/testdata",
"--line_number_pattern",
r"(?<=\.carbon:)(\d+)(?=(?:\D|$))",
] + sys.argv[1:]
os.execv(actual_py, args)
if __name__ == "__main__":
main()
explorer/testdata/assoc_const/fail_redefined.carbon
Vendored
+1 -1
View File
@@ -10,7 +10,7 @@ package ExplorerTest api;
interface Iface {
let T:! Type;
// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/assoc_const/fail_redefined.carbon:[[@LINE+1]]: Duplicate name `T` also found at {{.*}}/explorer/testdata/assoc_const/fail_redefined.carbon:12
// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/assoc_const/fail_redefined.carbon:[[@LINE+1]]: Duplicate name `T` also found at {{.*}}/explorer/testdata/assoc_const/fail_redefined.carbon:[[@LINE-1]]
let T:! Type;
}
explorer/testdata/destructor/fail_multiple_destructor_declaration.carbon
Vendored
+1 -1
View File
@@ -14,7 +14,7 @@ class A{
}
destructor[me: Self]{
Print("DESTRUCTOR A2 {0}",me.n);
// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/destructor/fail_multiple_destructor_declaration.carbon:[[@LINE+1]]: Duplicate name `destructor` also found at {{.*}}/explorer/testdata/destructor/fail_multiple_destructor_declaration.carbon:14
// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/destructor/fail_multiple_destructor_declaration.carbon:[[@LINE+1]]: Duplicate name `destructor` also found at {{.*}}/explorer/testdata/destructor/fail_multiple_destructor_declaration.carbon:[[@LINE-3]]
}
var n: i32;
}
explorer/testdata/mixin/fail_field_member_name_clash.carbon
Vendored
+1 -1
View File
@@ -17,7 +17,7 @@ __mixin M {
class C {
let F: i32 = 0;
__mix M;
// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/mixin/fail_field_member_name_clash.carbon:[[@LINE+1]]: Member named F (declared at {{.*}}/explorer/testdata/mixin/fail_field_member_name_clash.carbon:14) cannot be mixed into C because it clashes with an existing member with the same name (declared at {{.*}}/explorer/testdata/mixin/fail_field_member_name_clash.carbon:18)
// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/mixin/fail_field_member_name_clash.carbon:[[@LINE+1]]: Member named F (declared at {{.*}}/explorer/testdata/mixin/fail_field_member_name_clash.carbon:[[@LINE-6]]) cannot be mixed into C because it clashes with an existing member with the same name (declared at {{.*}}/explorer/testdata/mixin/fail_field_member_name_clash.carbon:[[@LINE-2]])
}
fn Main() -> i32 {
explorer/testdata/mixin/fail_method_member_name_clash.carbon
Vendored
+1 -1
View File
@@ -19,7 +19,7 @@ class C {
return x;
}
__mix M;
// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/mixin/fail_method_member_name_clash.carbon:[[@LINE+1]]: Member named F (declared at {{.*}}/explorer/testdata/mixin/fail_method_member_name_clash.carbon:14) cannot be mixed into C because it clashes with an existing member with the same name (declared at {{.*}}/explorer/testdata/mixin/fail_method_member_name_clash.carbon:20)
// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/mixin/fail_method_member_name_clash.carbon:[[@LINE+1]]: Member named F (declared at {{.*}}/explorer/testdata/mixin/fail_method_member_name_clash.carbon:[[@LINE-8]]) cannot be mixed into C because it clashes with an existing member with the same name (declared at {{.*}}/explorer/testdata/mixin/fail_method_member_name_clash.carbon:[[@LINE-2]])
}
fn Main() -> i32 {
explorer/testdata/mixin/fail_mix_diamond_clash.carbon
Vendored
+1 -1
View File
@@ -28,7 +28,7 @@ __mixin M3 {
class C {
__mix M2;
__mix M3;
// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/mixin/fail_mix_diamond_clash.carbon:[[@LINE+1]]: Member named F1 (declared at {{.*}}/explorer/testdata/mixin/fail_mix_diamond_clash.carbon:13) is being mixed multiple times into C
// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/mixin/fail_mix_diamond_clash.carbon:[[@LINE+1]]: Member named F1 (declared at {{.*}}/explorer/testdata/mixin/fail_mix_diamond_clash.carbon:[[@LINE-18]]) is being mixed multiple times into C
}
fn Main() -> i32 {
explorer/testdata/mixin/fail_mix_members_clash.carbon
Vendored
+1 -1
View File
@@ -21,7 +21,7 @@ __mixin M2 {
class C {
__mix M1;
__mix M2;
// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/mixin/fail_mix_members_clash.carbon:[[@LINE+1]]: Member named F1 (declared at {{.*}}/explorer/testdata/mixin/fail_mix_members_clash.carbon:18) cannot be mixed into C because it clashes with an existing member with the same name (declared at {{.*}}/explorer/testdata/mixin/fail_mix_members_clash.carbon:13)
// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/mixin/fail_mix_members_clash.carbon:[[@LINE+1]]: Member named F1 (declared at {{.*}}/explorer/testdata/mixin/fail_mix_members_clash.carbon:[[@LINE-6]]) cannot be mixed into C because it clashes with an existing member with the same name (declared at {{.*}}/explorer/testdata/mixin/fail_mix_members_clash.carbon:[[@LINE-11]])
}
fn Main() -> i32 {
explorer/testdata/mixin/fail_recursive_mixing.carbon
Vendored
+1 -1
View File
@@ -12,7 +12,7 @@ __mixin M1 {
return x;
}
__mix M1;
// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/mixin/fail_recursive_mixing.carbon:[[@LINE+1]]: Member named F1 (declared at {{.*}}/explorer/testdata/mixin/fail_recursive_mixing.carbon:13) is being mixed multiple times into M1
// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/mixin/fail_recursive_mixing.carbon:[[@LINE+1]]: Member named F1 (declared at {{.*}}/explorer/testdata/mixin/fail_recursive_mixing.carbon:[[@LINE-2]]) is being mixed multiple times into M1
}
class C {
explorer/testdata/name_lookup/fail_block_duplicate.carbon
Vendored
+1 -1
View File
@@ -10,7 +10,7 @@ package ExplorerTest api;
fn Main() -> i32 {
var x: i32 = 0;
// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/name_lookup/fail_block_duplicate.carbon:[[@LINE+1]]: Duplicate name `x` also found at {{.*}}/explorer/testdata/name_lookup/fail_block_duplicate.carbon:12
// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/name_lookup/fail_block_duplicate.carbon:[[@LINE+1]]: Duplicate name `x` also found at {{.*}}/explorer/testdata/name_lookup/fail_block_duplicate.carbon:[[@LINE-1]]
var x: i32 = 0;
return 0;
}
explorer/testdata/name_lookup/fail_class_duplicate.carbon
Vendored
+1 -1
View File
@@ -10,7 +10,7 @@ package ExplorerTest api;
class Foo {
var x: i32;
// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/name_lookup/fail_class_duplicate.carbon:[[@LINE+1]]: Duplicate name `x` also found at {{.*}}/explorer/testdata/name_lookup/fail_class_duplicate.carbon:12
// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/name_lookup/fail_class_duplicate.carbon:[[@LINE+1]]: Duplicate name `x` also found at {{.*}}/explorer/testdata/name_lookup/fail_class_duplicate.carbon:[[@LINE-1]]
var x: i32;
}
explorer/testdata/name_lookup/fail_global_duplicate.carbon
Vendored
+1 -1
View File
@@ -9,7 +9,7 @@
package ExplorerTest api;
var x: i32 = 0;
// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/name_lookup/fail_global_duplicate.carbon:[[@LINE+1]]: Duplicate name `x` also found at {{.*}}/explorer/testdata/name_lookup/fail_global_duplicate.carbon:11
// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/name_lookup/fail_global_duplicate.carbon:[[@LINE+1]]: Duplicate name `x` also found at {{.*}}/explorer/testdata/name_lookup/fail_global_duplicate.carbon:[[@LINE-1]]
var x: i32 = 0;
fn Main() -> i32 {
explorer/testdata/returned_var/fail_duplicate_return_var.carbon
Vendored
+1 -1
View File
@@ -11,7 +11,7 @@ package ExplorerTest api;
fn AddInt(a: i32, b: i32) -> i32 {
returned var ret: i32 = a + b;
if (a == b) {
// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/returned_var/fail_duplicate_return_var.carbon:[[@LINE+1]]: Duplicate definition of returned var also found at {{.*}}/explorer/testdata/returned_var/fail_duplicate_return_var.carbon:12
// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/returned_var/fail_duplicate_return_var.carbon:[[@LINE+1]]: Duplicate definition of returned var also found at {{.*}}/explorer/testdata/returned_var/fail_duplicate_return_var.carbon:[[@LINE-2]]
returned var ret2: i32 = a + b;
}
return var;
explorer/testdata/returned_var/fail_missing_return.carbon
Vendored
+1 -1
View File
@@ -10,7 +10,7 @@ package ExplorerTest api;
fn AddInt(a: i32, b: i32) -> i32 {
returned var ret: i32 = a + b;
// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/returned_var/fail_missing_return.carbon:[[@LINE+1]]: `return <expression>` is not allowed with a returned var defined in scope: {{.*}}/explorer/testdata/returned_var/fail_missing_return.carbon:12
// CHECK:COMPILATION ERROR: {{.*}}/explorer/testdata/returned_var/fail_missing_return.carbon:[[@LINE+1]]: `return <expression>` is not allowed with a returned var defined in scope: {{.*}}/explorer/testdata/returned_var/fail_missing_return.carbon:[[@LINE-1]]
return ret;
}
explorer/update_checks.py
-303
View File
@@ -1,303 +0,0 @@
#!/usr/bin/env python3
"""Updates the CHECK: lines in lit tests based on the AUTOUPDATE line."""
__copyright__ = """
Part of the Carbon Language project, under the Apache License v2.0 with LLVM
Exceptions. See /LICENSE for license information.
SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
"""
import argparse
from concurrent import futures
import os
import re
import subprocess
import sys
from abc import ABC, abstractmethod
from typing import Any, Dict, List, Optional, Set
_BIN = "./bazel-bin/explorer/explorer"
_TESTDATA = "explorer/testdata"
# A prefix followed by a command to run for autoupdating checked output.
_AUTOUPDATE_MARKER = "// AUTOUPDATE: "
# Indicates no autoupdate is requested.
_NOAUTOUPDATE_MARKER = "// NOAUTOUPDATE"
# A regexp matching lines that contain line number references.
_LINE_NUMBER_RE = (
r"((?:SYNTAX|COMPILATION|RUNTIME) ERROR: [^:]*:)([1-9][0-9]*)(:.*)"
)
def _get_tests() -> Set[str]:
"""Get the list of tests from the filesystem."""
tests = set()
for root, _, files in os.walk(_TESTDATA):
for f in files:
if f in {"lit.cfg.py", "BUILD"}:
# Ignore the lit config.
continue
if os.path.splitext(f)[1] == ".carbon":
tests.add(os.path.join(root, f))
else:
sys.exit("Unrecognized file type in testdata: %s" % f)
return tests
class Line(ABC):
"""A line that may appear in the resulting test file."""
@abstractmethod
def format(
self, *, output_line_number: int, line_number_remap: Dict[int, int]
) -> str:
raise NotImplementedError
class OriginalLine(Line):
"""A line that was copied from the original test file."""
def __init__(self, line_number: int, text: str) -> None:
self.line_number = line_number
self.text = text
def format(self, **kwargs: Any) -> str:
return self.text
class CheckLine(Line):
"""A `// CHECK:` line generated from the test output."""
def __init__(self) -> None:
self.indent = ""
@staticmethod
def escape(s: str) -> str:
"""Escape any FileCheck special characters in `s`."""
return s.replace("{{", "{{[{][{]}}").replace("[[", "{{[[][[]}}")
def print_before_line(self, line: int) -> bool:
"""Determine if we'd prefer to print this CHECK before line `line`."""
return True
class SimpleCheckLine(CheckLine):
"""A `// CHECK:` line that checks for an exact string."""
def __init__(self, expected: str) -> None:
super().__init__()
self.expected = expected
def format(self, **kwargs: Any) -> str:
if self.expected:
return f"{self.indent}// CHECK:{self.expected}\n"
else:
return f"{self.indent}// CHECK-EMPTY:\n"
class CheckLineWithLineNumber(CheckLine):
"""A `// CHECK:` line where the expected output includes a line number.
Such result lines need to be fixed up after we've figured out which lines
to include in the resulting test file and in what order, because their
contents depend on where an original input line appears in the output.
"""
def __init__(self, before: str, line_number: int, after: str) -> None:
super().__init__()
self.before = before
self.line_number = line_number
self.after = after
def format(
self, *, output_line_number: int, line_number_remap: Dict[int, int]
) -> str:
delta = line_number_remap[self.line_number] - output_line_number
# We use `:+d` here to produce `LINE-n` or `LINE+n` as appropriate.
return (
f"{self.indent}// CHECK:{self.before}[[@LINE{delta:+d}]]"
+ f"{self.after}\n"
)
def print_before_line(self, line: int) -> bool:
return line >= self.line_number
def _make_check_line(out_line: str) -> CheckLine:
"""Given a line of output, determine what CHECK line to produce."""
out_line = out_line.rstrip()
match = re.match(_LINE_NUMBER_RE, out_line)
if match:
# Convert from 1-based line numbers to 0-based indexes.
diagnostic_line_number = int(match[2]) - 1
return CheckLineWithLineNumber(
match[1], diagnostic_line_number, match[3]
)
else:
return SimpleCheckLine(out_line)
def _should_produce_check_line(
check_line: CheckLine,
orig_line: Optional[OriginalLine],
autoupdate_index: int,
) -> bool:
"""Determine whether it's time to produce a given CHECK line."""
if not orig_line:
# If there's no original line, we have no choice.
return True
if orig_line.line_number <= autoupdate_index:
# Don't put any CHECK lines before the AUTOUPDATE line.
return False
return check_line.print_before_line(orig_line.line_number)
def _update_check_once(test: str) -> bool:
"""Updates the CHECK: lines for `test` by running explorer.
Returns True if the number of lines changes.
"""
with open(test) as f:
orig_lines = f.readlines()
# Remove old OUT.
autoupdate_index = None
noautoupdate_index = None
for line_index, line in enumerate(orig_lines):
if line.startswith(_AUTOUPDATE_MARKER):
autoupdate_index = line_index
autoupdate_cmd = line[len(_AUTOUPDATE_MARKER) :]
if line.startswith(_NOAUTOUPDATE_MARKER):
noautoupdate_index = line_index
if autoupdate_index is None:
if noautoupdate_index is None:
raise ValueError(
"%s must have either '%s' or '%s'"
% (test, _AUTOUPDATE_MARKER, _NOAUTOUPDATE_MARKER)
)
else:
return False
elif noautoupdate_index is not None:
raise ValueError(
"%s has both '%s' and '%s', must have only one"
% (test, _AUTOUPDATE_MARKER, _NOAUTOUPDATE_MARKER)
)
# Mirror lit.cfg.py substitutions; bazel runs don't need --prelude.
autoupdate_cmd = autoupdate_cmd.replace("%{explorer}", _BIN)
# Run the autoupdate command to generate output.
# (`bazel run` would serialize)
p = subprocess.run(
autoupdate_cmd % test,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
out = p.stdout.decode("utf-8")
# `lit` uses full paths to the test file, so use a regex to ignore paths
# when used.
# TODO: Maybe revisit and see if lit can be convinced to give a
# root-relative path.
out = CheckLine.escape(out).replace(test, "{{.*}}/%s" % test)
out_lines = out.splitlines()
orig_line_iter = iter(
OriginalLine(i, line) for i, line in enumerate(orig_lines)
)
check_line_iter = iter(_make_check_line(out_line) for out_line in out_lines)
next_orig_line: Optional[OriginalLine] = next(orig_line_iter, None)
next_check_line: Optional[CheckLine] = next(check_line_iter, None)
# Interleave the original lines and the CHECK: lines into a list of
# `result_lines`.
result_lines: List[Line] = []
# Mapping from `orig_lines` indexes to `result_lines` indexes.
line_number_remap: Dict[int, int] = {}
while next_orig_line or next_check_line:
if next_check_line and _should_produce_check_line(
next_check_line, next_orig_line, autoupdate_index
):
# Indent the CHECK: line to match the next original line.
if next_orig_line:
match = re.match(" *", next_orig_line.text)
if match:
next_check_line.indent = match[0]
result_lines.append(next_check_line)
next_check_line = next(check_line_iter, None)
else:
assert next_orig_line, "no lines left"
# Include this original line if it isn't a CHECK: line.
if not re.match(" *// CHECK", next_orig_line.text):
line_number_remap[next_orig_line.line_number] = len(
result_lines
)
result_lines.append(next_orig_line)
next_orig_line = next(orig_line_iter, None)
# Generate contents for any lines that depend on line numbers.
formatted_result_lines = [
line.format(output_line_number=i, line_number_remap=line_number_remap)
for i, line in enumerate(result_lines)
]
# If nothing's changed, we're done.
if formatted_result_lines == orig_lines:
return False
# Interleave the new CHECK: lines with the tested content.
with open(test, "w") as f:
f.writelines(formatted_result_lines)
return True
def _update_check(test: str) -> None:
"""Wraps CHECK: updates for test files."""
# If the number of output lines changes, run again because output can be
# line-specific. However, output should stabilize quickly.
if (
_update_check_once(test)
and _update_check_once(test)
and _update_check_once(test)
):
raise ValueError("The output of %s kept changing" % test)
print(".", end="", flush=True)
def _update_checks(tests: Set[str]) -> None:
"""Runs bazel to update CHECK: lines in lit tests."""
# Build all tests at once in order to allow parallel updates.
print("Building explorer...")
subprocess.check_call(["bazel", "build", "//explorer"])
print("Updating %d lit test(s)..." % len(tests))
with futures.ThreadPoolExecutor() as exec:
# list() iterates to propagate exceptions.
list(exec.map(_update_check, tests))
# Each update call indicates progress with a dot without a newline, so put a
# newline to wrap.
print("\nUpdated lit tests.")
def main() -> None:
# Go to the repository root so that paths will match bazel's view.
os.chdir(os.path.join(os.path.dirname(__file__), ".."))
parser = argparse.ArgumentParser()
parser.add_argument("tests", nargs="*")
args = parser.parse_args()
if args.tests:
tests = set(args.tests)
else:
print("HINT: run `update_checks.py f1 f2 ...` to update specific tests")
tests = _get_tests()
_update_checks(tests)
if __name__ == "__main__":
main()
toolchain/lexer/__init__.py
View File
toolchain/lexer/lit_autoupdate.py
Executable
+48
View File
@@ -0,0 +1,48 @@
#!/usr/bin/env python3
"""Updates the CHECK: lines in lit tests based on the AUTOUPDATE line."""
__copyright__ = """
Part of the Carbon Language project, under the Apache License v2.0 with LLVM
Exceptions. See /LICENSE for license information.
SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
"""
import os
import sys
from pathlib import Path
def main() -> None:
# Calls the main script with explorer settings. This uses execv in order to
# avoid Python import behaviors.
actual_py = Path(__file__).parent.parent.parent.joinpath(
"bazel", "testing", "lit_autoupdate_base.py"
)
args = [
sys.argv[0],
# Flags to configure for explorer testing.
"--build_target",
"//toolchain/driver:carbon",
"--cmd_replace",
"%{carbon}",
"./bazel-bin/toolchain/driver/carbon",
# Ignore the resulting column of EndOfFile because it's typically the
# end of the CHECK comment.
"--extra_check_replacement",
".*'EndOfFile'",
r"column: (?:\d+)",
"column: {{[0-9]+}}",
# Ignore spaces that are used to columnize lines.
"--line_number_format",
"{{ *}}[[@LINE%(delta)s]]",
"--line_number_pattern",
r"(?<= line: )( *\d+)(?=,)",
"--testdata",
"toolchain/lexer/testdata",
] + sys.argv[1:]
os.execv(actual_py, args)
if __name__ == "__main__":
main()
toolchain/lexer/testdata/BUILD
Vendored
+15
View File
@@ -0,0 +1,15 @@
# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
# Exceptions. See /LICENSE for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
load("//bazel/testing:lit.bzl", "glob_lit_tests")
glob_lit_tests(
data = [
"//toolchain/driver:carbon",
"@llvm-project//llvm:FileCheck",
"@llvm-project//llvm:not",
],
driver = "lit.cfg.py",
test_file_exts = ["carbon"],
)
toolchain/driver/testdata/carbon_test.carbon → toolchain/lexer/testdata/carbon_test.carbon
Vendored
+2 -1
View File
@@ -3,6 +3,7 @@
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// RUN: %{carbon} dump tokens %s 2>&1 | %{FileCheck-strict} %s
// AUTOUPDATE: %{carbon} dump tokens %s
// CHECK:token: { index: 0, kind: 'Fn', line: {{ *}}[[@LINE+7]], column: 1, indent: 1, spelling: 'fn', has_trailing_space: true }
// CHECK:token: { index: 1, kind: 'Identifier', line: {{ *}}[[@LINE+6]], column: 4, indent: 1, spelling: 'run', identifier: 0 }
@@ -18,4 +19,4 @@ fn run(String program) {
return True;
// CHECK:token: { index: 10, kind: 'CloseCurlyBrace', line: {{ *}}[[@LINE+1]], column: 1, indent: 1, spelling: '}', opening_token: 6, has_trailing_space: true }
}
// CHECK:token: { index: 11, kind: 'EndOfFile', line: {{ *}}[[@LINE]], column: {{[0-9]+}}, indent: 1, spelling: '' }
// CHECK:token: { index: 11, kind: 'EndOfFile', line: {{ *}}[[@LINE+0]], column: {{[0-9]+}}, indent: 1, spelling: '' }
toolchain/lexer/testdata/lit.cfg.py
Vendored Symlink
+1
View File
@@ -0,0 +1 @@
../../../bazel/testing/lit.cfg.py
toolchain/semantics/__init__.py
View File
toolchain/semantics/lit_autoupdate.py
Executable
+41
View File
@@ -0,0 +1,41 @@
#!/usr/bin/env python3
"""Updates the CHECK: lines in lit tests based on the AUTOUPDATE line."""
__copyright__ = """
Part of the Carbon Language project, under the Apache License v2.0 with LLVM
Exceptions. See /LICENSE for license information.
SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
"""
import os
import sys
from pathlib import Path
def main() -> None:
# Calls the main script with explorer settings. This uses execv in order to
# avoid Python import behaviors.
actual_py = Path(__file__).parent.parent.parent.joinpath(
"bazel", "testing", "lit_autoupdate_base.py"
)
args = [
sys.argv[0],
# Flags to configure for explorer testing.
"--build_target",
"//toolchain/driver:carbon",
"--cmd_replace",
"%{carbon}",
"./bazel-bin/toolchain/driver/carbon",
# TODO: This should eventually have lines in output, but it doesn't
# right now.
"--line_number_pattern",
"UNUSED",
"--testdata",
"toolchain/semantics/testdata",
] + sys.argv[1:]
os.execv(actual_py, args)
if __name__ == "__main__":
main()
toolchain/semantics/testdata/empty.carbon
Vendored
+1
View File
@@ -3,5 +3,6 @@
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// RUN: %{carbon} dump semantics-ir %s 2>&1 | %{FileCheck-strict} %s
// AUTOUPDATE: %{carbon} dump semantics-ir %s
// CHECK:{
// CHECK:}
toolchain/semantics/testdata/function/basic.carbon
Vendored
+3 -2
View File
@@ -3,12 +3,13 @@
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// RUN: %{carbon} dump semantics-ir %s 2>&1 | %{FileCheck-strict} %s
// AUTOUPDATE: %{carbon} dump semantics-ir %s
// CHECK:{
// CHECK: Function(
// CHECK: %[[ID0:[0-9]+]],
// CHECK: %0,
// CHECK: {
// CHECK: }),
// CHECK: SetName(`Foo`, %[[ID0]]),
// CHECK: SetName(`Foo`, %0),
// CHECK:}
fn Foo() {}
toolchain/semantics/testdata/function/order.carbon
Vendored
+7 -6
View File
@@ -3,22 +3,23 @@
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// RUN: %{carbon} dump semantics-ir %s 2>&1 | %{FileCheck-strict} %s
// AUTOUPDATE: %{carbon} dump semantics-ir %s
// CHECK:{
// CHECK: Function(
// CHECK: %[[ID0:[0-9]+]],
// CHECK: %2,
// CHECK: {
// CHECK: }),
// CHECK: SetName(`Foo`, %[[ID0]]),
// CHECK: SetName(`Foo`, %2),
// CHECK: Function(
// CHECK: %[[ID1:[0-9]+]],
// CHECK: %1,
// CHECK: {
// CHECK: }),
// CHECK: SetName(`Bar`, %[[ID1]]),
// CHECK: SetName(`Bar`, %1),
// CHECK: Function(
// CHECK: %[[ID2:[0-9]+]],
// CHECK: %0,
// CHECK: {
// CHECK: }),
// CHECK: SetName(`Baz`, %[[ID2]]),
// CHECK: SetName(`Baz`, %0),
// CHECK:}
fn Foo() {}
toolchain/semantics/testdata/return/binary_op.carbon
Vendored
+7 -6
View File
@@ -3,16 +3,17 @@
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// RUN: %{carbon} dump semantics-ir %s 2>&1 | %{FileCheck-strict} %s
// AUTOUPDATE: %{carbon} dump semantics-ir %s
// CHECK:{
// CHECK: Function(
// CHECK: %[[ID0:[0-9]+]],
// CHECK: %0,
// CHECK: {
// CHECK: IntegerLiteral(%[[ID1:[0-9]+]], 12),
// CHECK: IntegerLiteral(%[[ID2:[0-9]+]], 34),
// CHECK: BinaryOperator(%[[ID3:[0-9]+]], +, %[[ID1]], %[[ID2]]),
// CHECK: Return(%[[ID3]]),
// CHECK: IntegerLiteral(%3, 12),
// CHECK: IntegerLiteral(%2, 34),
// CHECK: BinaryOperator(%1, +, %3, %2),
// CHECK: Return(%1),
// CHECK: }),
// CHECK: SetName(`Main`, %[[ID0]]),
// CHECK: SetName(`Main`, %0),
// CHECK:}
fn Main() {
toolchain/semantics/testdata/return/literal.carbon
Vendored
+5 -4
View File
@@ -3,14 +3,15 @@
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// RUN: %{carbon} dump semantics-ir %s 2>&1 | %{FileCheck-strict} %s
// AUTOUPDATE: %{carbon} dump semantics-ir %s
// CHECK:{
// CHECK: Function(
// CHECK: %[[ID0:[0-9]+]],
// CHECK: %0,
// CHECK: {
// CHECK: IntegerLiteral(%[[ID1:[0-9]+]], 0),
// CHECK: Return(%[[ID1]]),
// CHECK: IntegerLiteral(%1, 0),
// CHECK: Return(%1),
// CHECK: }),
// CHECK: SetName(`Main`, %[[ID0]]),
// CHECK: SetName(`Main`, %0),
// CHECK:}
fn Main() {
toolchain/semantics/testdata/return/trivial.carbon
Vendored
+3 -2
View File
@@ -3,13 +3,14 @@
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// RUN: %{carbon} dump semantics-ir %s 2>&1 | %{FileCheck-strict} %s
// AUTOUPDATE: %{carbon} dump semantics-ir %s
// CHECK:{
// CHECK: Function(
// CHECK: %[[ID0:[0-9]+]],
// CHECK: %0,
// CHECK: {
// CHECK: Return(None),
// CHECK: }),
// CHECK: SetName(`Main`, %[[ID0]]),
// CHECK: SetName(`Main`, %0),
// CHECK:}
fn Main() {