mirror of
https://github.com/carbon-language/carbon-lang.git
synced 2026-11-22 15:02:38 +00:00
This adds GitHub `actionlint` to validate our actions much more thoroughly ahead of time. This would have caught the recent issue of benchmarking workflow. To make this work, the PR adds a `run_actionlint.py` and extends our script utilities to support a release in an archive where we extract the binary. The `prek` check combines `actionlint` with `shellcheck-py` to also check the shell within the actions. It runs on all the workflows and action files. And we fix various existing issues: - Adds missing `description` metadata and missing `id: pages`. - Quotes shell variables across several files. - Document the intentional word-splitting in `clangd_tidy.yaml`. Assisted-by: Antigravity with Gemini
343 lines
13 KiB
Python
343 lines
13 KiB
Python
"""Utilities for scripts."""
|
|
|
|
__copyright__ = """
|
|
Part of the Carbon Language project, under the Apache License v2.0 with LLVM
|
|
Exceptions. See /LICENSE for license information.
|
|
SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
"""
|
|
|
|
import fcntl
|
|
import hashlib
|
|
import os
|
|
import platform
|
|
import shutil
|
|
import tarfile
|
|
import tempfile
|
|
import time
|
|
import urllib.request
|
|
import zipfile
|
|
from enum import Enum
|
|
from pathlib import Path
|
|
from typing import NamedTuple, Optional
|
|
|
|
|
|
# The tools we track releases for.
|
|
class Release(Enum):
|
|
ACTIONLINT = "actionlint"
|
|
BAZELISK = "bazelisk"
|
|
BUILDIFIER = "buildifier"
|
|
BUILDOZER = "buildozer"
|
|
TARGET_DETERMINATOR = "target-determinator"
|
|
|
|
|
|
class ReleaseInfo(NamedTuple):
|
|
# The base URL for downloads. Should include the version.
|
|
url: str
|
|
# The separator in a binary's name, `-`, `.`, or `_`.
|
|
separator: str
|
|
# Optional version string embedded in archived release asset names.
|
|
version_in_name: Optional[str] = None
|
|
|
|
|
|
_ACTIONLINT_VERSION = "1.7.12"
|
|
_BAZEL_TOOLS_URL = (
|
|
"https://github.com/bazelbuild/buildtools/releases/download/v8.5.1/"
|
|
)
|
|
|
|
# Structured information per release tool.
|
|
_RELEASES = {
|
|
Release.ACTIONLINT: ReleaseInfo(
|
|
f"https://github.com/rhysd/actionlint/releases/download/v{_ACTIONLINT_VERSION}/", # noqa: E501
|
|
"_",
|
|
version_in_name=_ACTIONLINT_VERSION,
|
|
),
|
|
Release.BAZELISK: ReleaseInfo(
|
|
"https://github.com/bazelbuild/bazelisk/releases/download/v1.28.1/", "-"
|
|
),
|
|
Release.BUILDIFIER: ReleaseInfo(_BAZEL_TOOLS_URL, "-"),
|
|
Release.BUILDOZER: ReleaseInfo(_BAZEL_TOOLS_URL, "-"),
|
|
Release.TARGET_DETERMINATOR: ReleaseInfo(
|
|
"https://github.com/bazel-contrib/target-determinator/releases/download/v0.32.0/", # noqa: E501
|
|
".",
|
|
),
|
|
}
|
|
|
|
|
|
# Shas for the tools.
|
|
#
|
|
# To update, change the version in a tool's URL and use
|
|
# `calculate_release_shas.py`. This is maintained separate from _RELEASES just
|
|
# to make copy-paste updates simpler.
|
|
_RELEASE_SHAS = {
|
|
Release.ACTIONLINT: {
|
|
"darwin_amd64.tar.gz": "5b44c3bc2255115c9b69e30efc0fecdf498fdb63c5d58e17084fd5f16324c644", # noqa: E501
|
|
"darwin_arm64.tar.gz": "aba9ced2dee8d27fecca3dc7feb1a7f9a52caefa1eb46f3271ea66b6e0e6953f", # noqa: E501
|
|
"linux_amd64.tar.gz": "8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8", # noqa: E501
|
|
"linux_arm64.tar.gz": "325e971b6ba9bfa504672e29be93c24981eeb1c07576d730e9f7c8805afff0c6", # noqa: E501
|
|
"windows_amd64.zip": "6e7241b51e6817ea6a047693d8e6fed13b31819c9a0dd6c5a726e1592d22f6e9", # noqa: E501
|
|
},
|
|
Release.BAZELISK: {
|
|
"darwin-amd64": "023225736cea5dc88f2b0807d5b1af4eb0f69a4ed45e3994b2c18c263bc80e48", # noqa: E501
|
|
"darwin-arm64": "dea3f3f5de2dbc5e269e0132cdd369d5efe738f7b973d5d4eb2b4f7055a97b39", # noqa: E501
|
|
"linux-amd64": "22e7d3a188699982f661cf4687137ee52d1f24fec1ec893d91a6c4d791a75de8", # noqa: E501
|
|
"linux-arm64": "8ded44b58a0d9425a4178af26cf17693feac3b87bdcfef0a2a0898fcd1afc9f2", # noqa: E501
|
|
"windows-amd64.exe": "b9d65a1f7c2d7af885a96a4fd5aa36b40fb41816d30944390569eef908bdc954", # noqa: E501
|
|
},
|
|
Release.BUILDIFIER: {
|
|
"darwin-amd64": "31de189e1a3fe53aa9e8c8f74a0309c325274ad19793393919e1ca65163ca1a4", # noqa: E501
|
|
"darwin-arm64": "62836a9667fa0db309b0d91e840f0a3f2813a9c8ea3e44b9cd58187c90bc88ba", # noqa: E501
|
|
"linux-amd64": "887377fc64d23a850f4d18a077b5db05b19913f4b99b270d193f3c7334b5a9a7", # noqa: E501
|
|
"linux-arm64": "947bf6700d708026b2057b09bea09abbc3cafc15d9ecea35bb3885c4b09ccd04", # noqa: E501
|
|
"windows-amd64.exe": "f4ecb9c73de2bc38b845d4ee27668f6248c4813a6647db4b4931a7556052e4e1", # noqa: E501
|
|
},
|
|
Release.BUILDOZER: {
|
|
"darwin-amd64": "b85b9ad59c1543999a5d8bc8bee6e42b9f025be3ff520bc2d090213698850b43", # noqa: E501
|
|
"darwin-arm64": "d0cf2f6e11031d62bfd4584e46eb6bb708a883ff948be76538b34b83de833262", # noqa: E501
|
|
"linux-amd64": "2b745ca2ad41f1e01673fb59ac50af6b45ca26105c1d20fad64c3d05a95522f5", # noqa: E501
|
|
"linux-arm64": "87ee1d2d81d08ccae8f9147fc58503967c85878279e892f2990912412feef1a1", # noqa: E501
|
|
"windows-amd64.exe": "e177155c2c8ef41569791de34f13077cefe3e5623f9f02e099347232bc028901", # noqa: E501
|
|
},
|
|
Release.TARGET_DETERMINATOR: {
|
|
"darwin.amd64": "289c61f8f4553a29d6ad2fbf1779a83180e7504c278196851becfd3f4163f6f4", # noqa: E501
|
|
"darwin.arm64": "6e688292b43f99f7b76d0af0fc32ac1eec8c110571a323c7e30bcfc9ac41275c", # noqa: E501
|
|
"linux.amd64": "792ed4c6f53aad60e8255686c272f28d3c039eaaef03518dbbde33519713802a", # noqa: E501
|
|
"linux.arm64": "339d0b7d3c72734f435d67e9b66936261c58c02be4ce64bdb76c85f1683e8084", # noqa: E501
|
|
"windows.amd64.exe": "457aba640e737edaf06d3922fa33914109dbe6382af951dfe6a99dbdf50c714c", # noqa: E501
|
|
},
|
|
}
|
|
|
|
|
|
def chdir_repo_root() -> None:
|
|
"""Change the working directory to the repository root.
|
|
|
|
This is done so that scripts run from a consistent directory.
|
|
"""
|
|
os.chdir(Path(__file__).parents[1])
|
|
|
|
|
|
def _get_hash(file: Path) -> str:
|
|
"""Returns the sha256 of a file."""
|
|
digest = hashlib.sha256()
|
|
with file.open("rb") as f:
|
|
while True:
|
|
chunk = f.read(1024 * 64)
|
|
if not chunk:
|
|
break
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _download(url: str, local_path: Path) -> Optional[int]:
|
|
"""Downloads the URL to the path. Returns an HTTP error code on failure."""
|
|
with urllib.request.urlopen(url) as response:
|
|
if response.code != 200:
|
|
return int(response.code)
|
|
with local_path.open("wb") as f:
|
|
shutil.copyfileobj(response, f)
|
|
return None
|
|
|
|
|
|
def _download_with_retries(url: str, dest: Path) -> None:
|
|
"""Downloads `url` to `dest`, retrying transient HTTP errors."""
|
|
retries = 5
|
|
while True:
|
|
err = _download(url, dest)
|
|
if err is None:
|
|
return
|
|
retries -= 1
|
|
if retries == 0:
|
|
exit(f"Failed to download {url}: HTTP {err}.")
|
|
time.sleep(1)
|
|
|
|
|
|
def _extract_binary_from_archive(
|
|
archive_path: Path, archive_url: str, binary_name: str, dest: Path
|
|
) -> None:
|
|
"""Extracts `binary_name` from a `.tar.gz` or `.zip` archive into `dest`."""
|
|
if archive_url.endswith(".tar.gz"):
|
|
with tarfile.open(archive_path, mode="r:gz") as tf:
|
|
member = tf.getmember(binary_name)
|
|
extracted = tf.extractfile(member)
|
|
if extracted is None:
|
|
exit(f"Entry {binary_name} in {archive_url} is not a file.")
|
|
with extracted, dest.open("wb") as out:
|
|
shutil.copyfileobj(extracted, out)
|
|
elif archive_url.endswith(".zip"):
|
|
with zipfile.ZipFile(archive_path) as zf:
|
|
with zf.open(binary_name) as extracted, dest.open("wb") as out:
|
|
shutil.copyfileobj(extracted, out)
|
|
else:
|
|
exit(f"Unsupported archive format: {archive_url}")
|
|
|
|
|
|
def _get_cached_binary(name: str, url: str, want_hash: str) -> str:
|
|
"""Returns the path to the cached binary.
|
|
|
|
If the matching version is already cached, returns it. Otherwise, downloads
|
|
from the URL and verifies the hash matches.
|
|
"""
|
|
cache_dir = Path.home().joinpath(".cache", "carbon-lang-scripts")
|
|
cache_dir.mkdir(parents=True, exist_ok=True)
|
|
is_archive = url.endswith((".tar.gz", ".zip"))
|
|
|
|
# Hold a lock while checksumming and downloading the path. Otherwise,
|
|
# parallel runs by prek may conflict with one another with
|
|
# simultaneous downloads.
|
|
with open(cache_dir.joinpath(f"{name}.lock"), "w") as lock_file:
|
|
fcntl.lockf(lock_file.fileno(), fcntl.LOCK_EX)
|
|
|
|
local_path = cache_dir.joinpath(name)
|
|
hash_sidecar = cache_dir.joinpath(f"{name}.sha256")
|
|
if is_archive:
|
|
if local_path.is_file() and hash_sidecar.is_file():
|
|
archive_sha, sep, bin_sha = (
|
|
hash_sidecar.read_text(encoding="utf-8")
|
|
.strip()
|
|
.partition(":")
|
|
)
|
|
if (
|
|
sep
|
|
and archive_sha == want_hash
|
|
and bin_sha == _get_hash(local_path)
|
|
):
|
|
return str(local_path)
|
|
|
|
with tempfile.TemporaryDirectory(dir=cache_dir) as tmp_dir:
|
|
archive_path = Path(tmp_dir) / "archive"
|
|
_download_with_retries(url, archive_path)
|
|
found_hash = _get_hash(archive_path)
|
|
if want_hash != found_hash:
|
|
exit(
|
|
f"Downloaded {url} but found sha256 "
|
|
f"{found_hash} ({archive_path.stat().st_size} bytes), "
|
|
f"wanted {want_hash}"
|
|
)
|
|
extracted_tmp = Path(tmp_dir) / name
|
|
_extract_binary_from_archive(
|
|
archive_path, url, name, extracted_tmp
|
|
)
|
|
extracted_tmp.chmod(0o755)
|
|
bin_hash = _get_hash(extracted_tmp)
|
|
extracted_tmp.replace(local_path)
|
|
hash_sidecar.write_text(
|
|
f"{want_hash}:{bin_hash}\n", encoding="utf-8"
|
|
)
|
|
return str(local_path)
|
|
|
|
# Check if there's a cached file that can be used.
|
|
if local_path.is_file() and want_hash == _get_hash(local_path):
|
|
return str(local_path)
|
|
|
|
_download_with_retries(url, local_path)
|
|
local_path.chmod(0o755)
|
|
|
|
# Verify the downloaded hash.
|
|
found_hash = _get_hash(local_path)
|
|
if want_hash != found_hash:
|
|
exit(
|
|
f"Downloaded {url} but found sha256 "
|
|
f"{found_hash} ({local_path.stat().st_size} bytes), wanted "
|
|
f"{want_hash}"
|
|
)
|
|
|
|
return str(local_path)
|
|
|
|
|
|
def _get_machine() -> str:
|
|
machine = platform.machine()
|
|
if machine == "x86_64":
|
|
machine = "amd64"
|
|
elif machine == "aarch64":
|
|
machine = "arm64"
|
|
return machine
|
|
|
|
|
|
def _get_platform_ext() -> str:
|
|
if platform.system() == "Windows":
|
|
return ".exe"
|
|
else:
|
|
return ""
|
|
|
|
|
|
def _select_hash(hashes: dict[str, str], version: str) -> str:
|
|
# Ensure the platform version is supported and has a hash.
|
|
if version not in hashes:
|
|
# If this because a platform support issue, we may need to print errors.
|
|
exit(f"No release available for platform: {version}")
|
|
return hashes[version]
|
|
|
|
|
|
def _release_asset_url(
|
|
release: Release, info: ReleaseInfo, platform_label: str
|
|
) -> str:
|
|
base = info.url.rstrip("/")
|
|
if info.version_in_name is not None:
|
|
return (
|
|
f"{base}/{release.value}{info.separator}"
|
|
f"{info.version_in_name}{info.separator}{platform_label}"
|
|
)
|
|
return f"{base}/{release.value}{info.separator}{platform_label}"
|
|
|
|
|
|
def get_release(release: Release) -> str:
|
|
"""Install a tool to carbon-lang's cache and return its path.
|
|
|
|
release: The release to cache.
|
|
"""
|
|
info = _RELEASES[release]
|
|
shas = _RELEASE_SHAS[release]
|
|
|
|
# Translate platform information into the release asset's naming scheme.
|
|
ext = _get_platform_ext()
|
|
if info.version_in_name is not None:
|
|
asset_ext = ".zip" if platform.system() == "Windows" else ".tar.gz"
|
|
else:
|
|
asset_ext = ext
|
|
platform_label = (
|
|
f"{platform.system().lower()}{info.separator}"
|
|
f"{_get_machine()}{asset_ext}"
|
|
)
|
|
url = _release_asset_url(release, info, platform_label)
|
|
want_hash = _select_hash(shas, platform_label)
|
|
|
|
return _get_cached_binary(f"{release.value}{ext}", url, want_hash)
|
|
|
|
|
|
def calculate_release_shas() -> None:
|
|
"""Prints sha information for tracked tool releases."""
|
|
print("_RELEASE_SHAS = {")
|
|
for release, info in _RELEASES.items():
|
|
shas = _RELEASE_SHAS[release]
|
|
|
|
print(f" {release}: {{")
|
|
for platform_label in shas.keys():
|
|
url = _release_asset_url(release, info, platform_label)
|
|
with tempfile.NamedTemporaryFile() as f:
|
|
path = Path(f.name)
|
|
_download(url, path)
|
|
hash = _get_hash(path)
|
|
print(f' "{platform_label}": "{hash}", # noqa: E501')
|
|
print(" },")
|
|
print("}")
|
|
|
|
|
|
def locate_bazel() -> str:
|
|
"""Returns the bazel command.
|
|
|
|
In order, try:
|
|
1. The `BAZEL` environment variable.
|
|
2. `bazelisk`
|
|
3. `bazel`
|
|
4. `run_bazelisk.py`
|
|
"""
|
|
bazel = os.environ.get("BAZEL")
|
|
if bazel:
|
|
return bazel
|
|
|
|
for cmd in ("bazelisk", "bazel"):
|
|
target = shutil.which(cmd)
|
|
if target:
|
|
return target
|
|
|
|
return str(Path(__file__).parent / "run_bazelisk.py")
|