Introduce git-based target selection. (#3106)

This adds access to the `target-determinator` tool that computes the possibly impacted set of targets between the current checkout and a specific `git` commit. The tool is downloaded (and cached) with a Python script that wraps it and allows us to easily run it in our CI environment. And finally, switches the CI for pull requests and the merge queue to use this script and run a minimal set of impacted tests rather than all of them. In order to make this as simple as possible, this also merges the previously separate build and test steps of our CI. Note that the CI testing post-submit (push events) continues to use a blanket target pattern so that we have a good backstop in case something outside of what is tracked here changes. The important paths, especially the ones that might be in an interactive critical path, are the PR and merge queue. This has been tested on the `action-test` branch to try to make sure it works. Some example runs for folks to inspect: - `push` event with blanket test: https://github.com/carbon-language/carbon-lang/actions/runs/6070060958/job/16465422569 - A "large" PR that impacts a bunch of toolchain tests: - ubuntu-opt: https://github.com/carbon-language/carbon-lang/actions/runs/6070063050/job/16465428107?pr=3188 - macos-opt: https://github.com/carbon-language/carbon-lang/actions/runs/6070063050/job/16465428420?pr=3188 - The merge queue for this PR: - ubuntu-opt: https://github.com/carbon-language/carbon-lang/actions/runs/6070202665/job/16465825387 - macos-opt: https://github.com/carbon-language/carbon-lang/actions/runs/6070202665/job/16465825740 - A "small" PR that impacts one test: - ubuntu-opt: https://github.com/carbon-language/carbon-lang/actions/runs/6070659668/job/16467147641?pr=3189 - macos-opt: https://github.com/carbon-language/carbon-lang/actions/runs/6070659668/job/16467147933?pr=3189 - The merge queue for the small PR: - ubuntu-opt: https://github.com/carbon-language/carbon-lang/actions/runs/6070754584/job/16467427268 - macos-opt: https://github.com/carbon-language/carbon-lang/actions/runs/6070754584/job/16467427736 I have observed one failure while testing with this PR, but I've not yet been able to reproduce it. Every other failure (including empty lists, etc.) I've tried to address. However, we may have to keep an eye on actions after this to make sure there isn't some scenario I've not been able to work through in a test branch. --------- Co-authored-by: Jon Ross-Perkins <jperkins@google.com> Co-authored-by: Geoff Romer <gromer@google.com>
2026-10-05 21:41:27 +01:00 · 2023-09-08 00:18:50 +00:00
parent bc63e6ae0a
commit 2751f02258
6 changed files with 269 additions and 45 deletions
@@ -14,10 +14,12 @@ from pathlib import Path
 import platform
 import shutil
 import time
-from typing import Optional
+from typing import Dict, Optional
 import urllib.request

-_URL = "https://github.com/bazelbuild/buildtools/releases/download/v6.3.3/"
+_BAZEL_TOOLS_URL = (
+    "https://github.com/bazelbuild/buildtools/releases/download/v6.3.3/"
+)

 """Version SHAs.

@@ -32,7 +34,7 @@ Gather shas with:
        echo },
    done
 """
-_VERSION_SHAS = {
+_BAZEL_TOOLS_VERSION_SHAS = {
    "buildozer": {
        "darwin-amd64": "9b0bbecb3745250e5ad5a9c36da456699cb55e52999451c3c74047d2b1f0085f",  # noqa: E501
        "darwin-arm64": "085928dd4deffa1a7fd38c66c4475e37326b2d4942408e8e3d993953ae4c626c",  # noqa: E501
@@ -49,6 +51,25 @@ _VERSION_SHAS = {
    },
 }

+_TARGET_DETERMINATOR_URL = "https://github.com/bazel-contrib/target-determinator/releases/download/v0.23.0/"  # noqa: E501
+
+"""Version SHAs.
+
+Gather shas with:
+    for v in darwin.amd64 darwin.arm64 linux.amd64 linux.arm64 \
+        windows.amd64.exe
+    do
+        echo "\"$v\": \"$(wget -q -O - https://github.com/bazel-contrib/target-determinator/releases/download/v0.23.0/target-determinator.$v | sha256sum | cut -d ' ' -f1)\", # noqa: E501"
+    done
+"""
+_TARGET_DETERMINATOR_SHAS = {
+    "darwin.amd64": "aba6dce8a978d2174b37dd1355eecba86db93be1ff77742d0753d8efd6a8a316",  # noqa: E501
+    "darwin.arm64": "6c3c308dcfc651408ed5490245ea3e0180fc49d4cc9b762ab84a4b979bcb07b8",  # noqa: E501
+    "linux.amd64": "5200dbca0dd4980690d5060cf8e04abac927efaca143567c51fe24cf973364d2",  # noqa: E501
+    "linux.arm64": "3c04f8bb2742219eb3415c6d675dcfe9175745eb7b1d6c3706085a9987f9f719",  # noqa: E501
+    "windows.amd64.exe": "3aea5bd52fdf29bfe6995ffcacc2b2c2299af02dc58f1039022ff758b58214c3",  # noqa: E501
+}
+

 class Release(Enum):
    BUILDOZER = "buildozer"
@@ -85,44 +106,22 @@ def _download(url: str, local_path: Path) -> Optional[int]:
    return None


-def get_release(release: Release) -> str:
-    """Install a file to carbon-lang's cache.
-
-    release: The release to cache.
-    """
+def _get_cached_binary(name: str, url: str, want_hash: str) -> str:
    cache_dir = Path.home().joinpath(".cache", "carbon-lang-scripts")
    cache_dir.mkdir(parents=True, exist_ok=True)

-    # Translate platform information into Bazel's release form.
-    machine = platform.machine()
-    if machine == "x86_64":
-        machine = "amd64"
-    version = f"{platform.system().lower()}-{machine}"
-
-    # Get ready to add .exe for Windows.
-    ext = ""
-    if platform.system() == "Windows":
-        ext = ".exe"
-
-    # Ensure the platform is supported, and grab its hash.
-    if version not in _VERSION_SHAS[release.value]:
-        # If this because a platform support issue, we may need to print errors.
-        exit(f"No {release.value} release available for platform: {version}")
-    want_hash = _VERSION_SHAS[release.value][version]
-
    # Hold a lock while checksumming and downloading the path. Otherwise,
    # parallel runs by pre-commit may conflict with one another with
    # simultaneous downloads.
-    with open(cache_dir.joinpath(f"{release.value}.lock"), "w") as lock_file:
+    with open(cache_dir.joinpath(f"{name}.lock"), "w") as lock_file:
        fcntl.lockf(lock_file.fileno(), fcntl.LOCK_EX)

        # Check if there's a cached file that can be used.
-        local_path = cache_dir.joinpath(f"{release.value}{ext}")
+        local_path = cache_dir.joinpath(name)
        if local_path.is_file() and want_hash == _get_hash(local_path):
            return str(local_path)

        # Download the file.
-        url = f"{_URL}/{release.value}-{version}{ext}"
        retries = 5
        while True:
            err = _download(url, local_path)
@@ -130,9 +129,7 @@ def get_release(release: Release) -> str:
                break
            retries -= 1
            if retries == 0:
-                exit(
-                    f"Failed to download {release.value}-{version}: HTTP {err}."
-                )
+                exit(f"Failed to download {url}: HTTP {err}.")
            time.sleep(1)
        local_path.chmod(0o755)

@@ -140,7 +137,7 @@ def get_release(release: Release) -> str:
        found_hash = _get_hash(local_path)
        if want_hash != found_hash:
            exit(
-                f"Downloaded {release.value}-{version} but found sha256 "
+                f"Downloaded {url} but found sha256 "
                f"{found_hash} ({local_path.stat().st_size} bytes), wanted "
                f"{want_hash}"
            )
@@ -148,6 +145,53 @@ def get_release(release: Release) -> str:
    return str(local_path)


+def _get_machine() -> str:
+    machine = platform.machine()
+    if machine == "x86_64":
+        machine = "amd64"
+    return machine
+
+
+def _get_platform_ext() -> str:
+    if platform.system() == "Windows":
+        return ".exe"
+    else:
+        return ""
+
+
+def _select_hash(hashes: Dict[str, str], version: str) -> str:
+    # Ensure the platform version is supported and has a hash.
+    if version not in hashes:
+        # If this because a platform support issue, we may need to print errors.
+        exit(f"No release available for platform: {version}")
+    return hashes[version]
+
+
+def get_target_determinator() -> str:
+    """Install the Bazel target-determinator tool to carbon-lang's cache."""
+    # Translate platform information into this tool's release binary form.
+    version = f"{platform.system().lower()}.{_get_machine()}"
+    ext = _get_platform_ext()
+    url = f"{_TARGET_DETERMINATOR_URL}/target-determinator.{version}{ext}"
+    want_hash = _select_hash(_TARGET_DETERMINATOR_SHAS, version)
+
+    return _get_cached_binary(f"target-determinator{ext}", url, want_hash)
+
+
+def get_release(release: Release) -> str:
+    """Install a Bazel-released tool to carbon-lang's cache.
+
+    release: The release to cache.
+    """
+    # Translate platform information into Bazel's release form.
+    version = f"{platform.system().lower()}-{_get_machine()}"
+    ext = _get_platform_ext()
+    url = f"{_BAZEL_TOOLS_URL}/{release.value}-{version}{ext}"
+    want_hash = _select_hash(_BAZEL_TOOLS_VERSION_SHAS[release.value], version)
+
+    return _get_cached_binary(f"{release.value}{ext}", url, want_hash)
+
+
 def locate_bazel() -> str:
    """Returns the bazel command.