mirror of
https://github.com/carbon-language/carbon-lang.git
synced 2026-11-22 14:22:37 +00:00
Add actionlint to our prek checks and fix issues (#8157)
This adds GitHub `actionlint` to validate our actions much more thoroughly ahead of time. This would have caught the recent issue of benchmarking workflow. To make this work, the PR adds a `run_actionlint.py` and extends our script utilities to support a release in an archive where we extract the binary. The `prek` check combines `actionlint` with `shellcheck-py` to also check the shell within the actions. It runs on all the workflows and action files. And we fix various existing issues: - Adds missing `description` metadata and missing `id: pages`. - Quotes shell variables across several files. - Document the intentional word-splitting in `clangd_tidy.yaml`. Assisted-by: Antigravity with Gemini
This commit is contained in:
1 parent
cc7e707321
commit
534197a3cd
12 files changed
+166
-24
No files matched your search
@@ -2,7 +2,8 @@
|
||||
# Exceptions. See /LICENSE for license information.
|
||||
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
|
||||
name: Setup build environment (macOS)
|
||||
name: Setup build environment
|
||||
description: Sets up uv, OS build dependencies, and Bazel configuration.
|
||||
inputs:
|
||||
matrix_runner:
|
||||
required: true
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
|
||||
name: Setup build environment (macOS)
|
||||
description: Frees disk space and installs macOS build dependencies.
|
||||
inputs:
|
||||
matrix_runner:
|
||||
required: true
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
|
||||
name: Setup build environment (Ubuntu)
|
||||
description: Frees disk space and installs Ubuntu build dependencies.
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
|
||||
name: Test setup
|
||||
description:
|
||||
Filters changed paths, sets up the build, and computes impacted Bazel targets.
|
||||
|
||||
inputs:
|
||||
matrix_runner:
|
||||
|
||||
@@ -92,4 +92,6 @@ jobs:
|
||||
env:
|
||||
FILTER_FILES: ${{ steps.filter.outputs.has_cpp_files }}
|
||||
run: |
|
||||
# FILTER_FILES is a space-separated file list from paths-filter.
|
||||
# shellcheck disable=SC2086
|
||||
uvx --with clangd-tidy==1.1.0.post2 clangd-tidy -p . -j 10 $FILTER_FILES
|
||||
@@ -45,6 +45,7 @@ jobs:
|
||||
- name: Prebuild actions
|
||||
run: ./website/prebuild.py
|
||||
- name: Setup Pages
|
||||
id: pages
|
||||
uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0
|
||||
- name: Setup Ruby
|
||||
uses: ruby/setup-ruby@b3aaceaef7e0ee231b499e7b5f49c03fc1d32199 # v1.338.0
|
||||
|
||||
@@ -82,9 +82,9 @@ jobs:
|
||||
env:
|
||||
REMOTE_CACHE_KEY: ${{ secrets.CARBON_BUILDS_GITHUB }}
|
||||
run: |
|
||||
echo "$REMOTE_CACHE_KEY" | base64 -d > $HOME/remote_cache_key.json
|
||||
echo "$REMOTE_CACHE_KEY" | base64 -d > "$HOME/remote_cache_key.json"
|
||||
echo "remote_cache_upload=--google_credentials=$HOME/remote_cache_key.json" \
|
||||
>> $GITHUB_ENV
|
||||
>> "$GITHUB_ENV"
|
||||
|
||||
- uses: ./.github/actions/build-setup-common
|
||||
with:
|
||||
@@ -93,14 +93,14 @@ jobs:
|
||||
|
||||
- name: Get nightly date
|
||||
run: |
|
||||
echo "nightly_date=$(date '+%Y.%m.%d')" >> $GITHUB_ENV
|
||||
echo "nightly_date=$(date '+%Y.%m.%d')" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build release
|
||||
run: |
|
||||
./scripts/run_bazel.py \
|
||||
--attempts=5 --jobs-on-last-attempt=4 \
|
||||
test -c opt --stamp --remote_download_toplevel \
|
||||
--pre_release=nightly --nightly_date=${nightly_date} \
|
||||
--pre_release=nightly --nightly_date="${nightly_date}" \
|
||||
//toolchain \
|
||||
//toolchain/install:carbon_toolchain_tar_gz \
|
||||
//toolchain/install:carbon_toolchain_tar_gz_test
|
||||
@@ -114,7 +114,7 @@ jobs:
|
||||
VERSION=$( \
|
||||
./bazel-bin/toolchain/carbon version \
|
||||
| cut -d' ' -f5 | cut -d'+' -f1)
|
||||
echo "release_version=$VERSION" >> $GITHUB_ENV
|
||||
echo "release_version=$VERSION" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Create the release
|
||||
env:
|
||||
|
||||
@@ -78,7 +78,7 @@ jobs:
|
||||
run: |
|
||||
mkdir -p prek-output
|
||||
git diff > prek-output/diff
|
||||
cp $GITHUB_EVENT_PATH prek-output/event
|
||||
cp "$GITHUB_EVENT_PATH" prek-output/event
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
if: failure()
|
||||
with:
|
||||
|
||||
@@ -122,7 +122,7 @@ jobs:
|
||||
./scripts/run_bazel.py \
|
||||
--attempts=5 --jobs-on-last-attempt=4 \
|
||||
test ${{ matrix.config.flags }} \
|
||||
--target_pattern_file=$TARGETS_FILE
|
||||
--target_pattern_file="$TARGETS_FILE"
|
||||
|
||||
# See "Disk space before build" in `test-setup`.
|
||||
- name: Disk space after build
|
||||
|
||||
@@ -183,6 +183,15 @@ repos:
|
||||
# Run linters last, as formatters and other checks may fix issues.
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: actionlint
|
||||
name: actionlint
|
||||
entry: scripts/run_actionlint.py
|
||||
args: ['-pyflakes=']
|
||||
language: python
|
||||
additional_dependencies:
|
||||
- shellcheck-py==0.11.0.1
|
||||
files: ^\.github/(workflows|actions)/.*\.(ya?ml)$
|
||||
pass_filenames: false
|
||||
- id: forbid-llvm-googletest
|
||||
name: Checks for deps on LLVM's version of GoogleTest
|
||||
entry: scripts/forbid_llvm_googletest.py
|
||||
|
||||
Executable
+28
@@ -0,0 +1,28 @@
|
||||
#!/usr/bin/env -S uv run --script
|
||||
|
||||
# /// script
|
||||
# requires-python = ">=3.12"
|
||||
# ///
|
||||
|
||||
|
||||
"""Runs actionlint on GitHub workflow files, mainly for prek."""
|
||||
|
||||
__copyright__ = """
|
||||
Part of the Carbon Language project, under the Apache License v2.0 with LLVM
|
||||
Exceptions. See /LICENSE for license information.
|
||||
SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
import scripts_utils
|
||||
|
||||
|
||||
def main() -> None:
|
||||
actionlint = scripts_utils.get_release(scripts_utils.Release.ACTIONLINT)
|
||||
os.execv(actionlint, [actionlint] + sys.argv[1:])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+113
-16
@@ -11,9 +11,11 @@ import hashlib
|
||||
import os
|
||||
import platform
|
||||
import shutil
|
||||
import tarfile
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import NamedTuple, Optional
|
||||
@@ -21,6 +23,7 @@ from typing import NamedTuple, Optional
|
||||
|
||||
# The tools we track releases for.
|
||||
class Release(Enum):
|
||||
ACTIONLINT = "actionlint"
|
||||
BAZELISK = "bazelisk"
|
||||
BUILDIFIER = "buildifier"
|
||||
BUILDOZER = "buildozer"
|
||||
@@ -30,16 +33,24 @@ class Release(Enum):
|
||||
class ReleaseInfo(NamedTuple):
|
||||
# The base URL for downloads. Should include the version.
|
||||
url: str
|
||||
# The separator in a binary's name, either `-` or `.`.
|
||||
# The separator in a binary's name, `-`, `.`, or `_`.
|
||||
separator: str
|
||||
# Optional version string embedded in archived release asset names.
|
||||
version_in_name: Optional[str] = None
|
||||
|
||||
|
||||
_ACTIONLINT_VERSION = "1.7.12"
|
||||
_BAZEL_TOOLS_URL = (
|
||||
"https://github.com/bazelbuild/buildtools/releases/download/v8.5.1/"
|
||||
)
|
||||
|
||||
# Structured information per release tool.
|
||||
_RELEASES = {
|
||||
Release.ACTIONLINT: ReleaseInfo(
|
||||
f"https://github.com/rhysd/actionlint/releases/download/v{_ACTIONLINT_VERSION}/", # noqa: E501
|
||||
"_",
|
||||
version_in_name=_ACTIONLINT_VERSION,
|
||||
),
|
||||
Release.BAZELISK: ReleaseInfo(
|
||||
"https://github.com/bazelbuild/bazelisk/releases/download/v1.28.1/", "-"
|
||||
),
|
||||
@@ -58,6 +69,13 @@ _RELEASES = {
|
||||
# `calculate_release_shas.py`. This is maintained separate from _RELEASES just
|
||||
# to make copy-paste updates simpler.
|
||||
_RELEASE_SHAS = {
|
||||
Release.ACTIONLINT: {
|
||||
"darwin_amd64.tar.gz": "5b44c3bc2255115c9b69e30efc0fecdf498fdb63c5d58e17084fd5f16324c644", # noqa: E501
|
||||
"darwin_arm64.tar.gz": "aba9ced2dee8d27fecca3dc7feb1a7f9a52caefa1eb46f3271ea66b6e0e6953f", # noqa: E501
|
||||
"linux_amd64.tar.gz": "8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8", # noqa: E501
|
||||
"linux_arm64.tar.gz": "325e971b6ba9bfa504672e29be93c24981eeb1c07576d730e9f7c8805afff0c6", # noqa: E501
|
||||
"windows_amd64.zip": "6e7241b51e6817ea6a047693d8e6fed13b31819c9a0dd6c5a726e1592d22f6e9", # noqa: E501
|
||||
},
|
||||
Release.BAZELISK: {
|
||||
"darwin-amd64": "023225736cea5dc88f2b0807d5b1af4eb0f69a4ed45e3994b2c18c263bc80e48", # noqa: E501
|
||||
"darwin-arm64": "dea3f3f5de2dbc5e269e0132cdd369d5efe738f7b973d5d4eb2b4f7055a97b39", # noqa: E501
|
||||
@@ -119,6 +137,39 @@ def _download(url: str, local_path: Path) -> Optional[int]:
|
||||
return None
|
||||
|
||||
|
||||
def _download_with_retries(url: str, dest: Path) -> None:
|
||||
"""Downloads `url` to `dest`, retrying transient HTTP errors."""
|
||||
retries = 5
|
||||
while True:
|
||||
err = _download(url, dest)
|
||||
if err is None:
|
||||
return
|
||||
retries -= 1
|
||||
if retries == 0:
|
||||
exit(f"Failed to download {url}: HTTP {err}.")
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def _extract_binary_from_archive(
|
||||
archive_path: Path, archive_url: str, binary_name: str, dest: Path
|
||||
) -> None:
|
||||
"""Extracts `binary_name` from a `.tar.gz` or `.zip` archive into `dest`."""
|
||||
if archive_url.endswith(".tar.gz"):
|
||||
with tarfile.open(archive_path, mode="r:gz") as tf:
|
||||
member = tf.getmember(binary_name)
|
||||
extracted = tf.extractfile(member)
|
||||
if extracted is None:
|
||||
exit(f"Entry {binary_name} in {archive_url} is not a file.")
|
||||
with extracted, dest.open("wb") as out:
|
||||
shutil.copyfileobj(extracted, out)
|
||||
elif archive_url.endswith(".zip"):
|
||||
with zipfile.ZipFile(archive_path) as zf:
|
||||
with zf.open(binary_name) as extracted, dest.open("wb") as out:
|
||||
shutil.copyfileobj(extracted, out)
|
||||
else:
|
||||
exit(f"Unsupported archive format: {archive_url}")
|
||||
|
||||
|
||||
def _get_cached_binary(name: str, url: str, want_hash: str) -> str:
|
||||
"""Returns the path to the cached binary.
|
||||
|
||||
@@ -127,6 +178,7 @@ def _get_cached_binary(name: str, url: str, want_hash: str) -> str:
|
||||
"""
|
||||
cache_dir = Path.home().joinpath(".cache", "carbon-lang-scripts")
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
is_archive = url.endswith((".tar.gz", ".zip"))
|
||||
|
||||
# Hold a lock while checksumming and downloading the path. Otherwise,
|
||||
# parallel runs by prek may conflict with one another with
|
||||
@@ -134,21 +186,49 @@ def _get_cached_binary(name: str, url: str, want_hash: str) -> str:
|
||||
with open(cache_dir.joinpath(f"{name}.lock"), "w") as lock_file:
|
||||
fcntl.lockf(lock_file.fileno(), fcntl.LOCK_EX)
|
||||
|
||||
# Check if there's a cached file that can be used.
|
||||
local_path = cache_dir.joinpath(name)
|
||||
hash_sidecar = cache_dir.joinpath(f"{name}.sha256")
|
||||
if is_archive:
|
||||
if local_path.is_file() and hash_sidecar.is_file():
|
||||
archive_sha, sep, bin_sha = (
|
||||
hash_sidecar.read_text(encoding="utf-8")
|
||||
.strip()
|
||||
.partition(":")
|
||||
)
|
||||
if (
|
||||
sep
|
||||
and archive_sha == want_hash
|
||||
and bin_sha == _get_hash(local_path)
|
||||
):
|
||||
return str(local_path)
|
||||
|
||||
with tempfile.TemporaryDirectory(dir=cache_dir) as tmp_dir:
|
||||
archive_path = Path(tmp_dir) / "archive"
|
||||
_download_with_retries(url, archive_path)
|
||||
found_hash = _get_hash(archive_path)
|
||||
if want_hash != found_hash:
|
||||
exit(
|
||||
f"Downloaded {url} but found sha256 "
|
||||
f"{found_hash} ({archive_path.stat().st_size} bytes), "
|
||||
f"wanted {want_hash}"
|
||||
)
|
||||
extracted_tmp = Path(tmp_dir) / name
|
||||
_extract_binary_from_archive(
|
||||
archive_path, url, name, extracted_tmp
|
||||
)
|
||||
extracted_tmp.chmod(0o755)
|
||||
bin_hash = _get_hash(extracted_tmp)
|
||||
extracted_tmp.replace(local_path)
|
||||
hash_sidecar.write_text(
|
||||
f"{want_hash}:{bin_hash}\n", encoding="utf-8"
|
||||
)
|
||||
return str(local_path)
|
||||
|
||||
# Check if there's a cached file that can be used.
|
||||
if local_path.is_file() and want_hash == _get_hash(local_path):
|
||||
return str(local_path)
|
||||
|
||||
# Download the file.
|
||||
retries = 5
|
||||
while True:
|
||||
err = _download(url, local_path)
|
||||
if err is None:
|
||||
break
|
||||
retries -= 1
|
||||
if retries == 0:
|
||||
exit(f"Failed to download {url}: HTTP {err}.")
|
||||
time.sleep(1)
|
||||
_download_with_retries(url, local_path)
|
||||
local_path.chmod(0o755)
|
||||
|
||||
# Verify the downloaded hash.
|
||||
@@ -187,6 +267,18 @@ def _select_hash(hashes: dict[str, str], version: str) -> str:
|
||||
return hashes[version]
|
||||
|
||||
|
||||
def _release_asset_url(
|
||||
release: Release, info: ReleaseInfo, platform_label: str
|
||||
) -> str:
|
||||
base = info.url.rstrip("/")
|
||||
if info.version_in_name is not None:
|
||||
return (
|
||||
f"{base}/{release.value}{info.separator}"
|
||||
f"{info.version_in_name}{info.separator}{platform_label}"
|
||||
)
|
||||
return f"{base}/{release.value}{info.separator}{platform_label}"
|
||||
|
||||
|
||||
def get_release(release: Release) -> str:
|
||||
"""Install a tool to carbon-lang's cache and return its path.
|
||||
|
||||
@@ -195,12 +287,17 @@ def get_release(release: Release) -> str:
|
||||
info = _RELEASES[release]
|
||||
shas = _RELEASE_SHAS[release]
|
||||
|
||||
# Translate platform information into Bazel's release form.
|
||||
# Translate platform information into the release asset's naming scheme.
|
||||
ext = _get_platform_ext()
|
||||
if info.version_in_name is not None:
|
||||
asset_ext = ".zip" if platform.system() == "Windows" else ".tar.gz"
|
||||
else:
|
||||
asset_ext = ext
|
||||
platform_label = (
|
||||
f"{platform.system().lower()}{info.separator}{_get_machine()}{ext}"
|
||||
f"{platform.system().lower()}{info.separator}"
|
||||
f"{_get_machine()}{asset_ext}"
|
||||
)
|
||||
url = f"{info.url}/{release.value}{info.separator}{platform_label}"
|
||||
url = _release_asset_url(release, info, platform_label)
|
||||
want_hash = _select_hash(shas, platform_label)
|
||||
|
||||
return _get_cached_binary(f"{release.value}{ext}", url, want_hash)
|
||||
@@ -214,7 +311,7 @@ def calculate_release_shas() -> None:
|
||||
|
||||
print(f" {release}: {{")
|
||||
for platform_label in shas.keys():
|
||||
url = f"{info.url}/{release.value}{info.separator}{platform_label}"
|
||||
url = _release_asset_url(release, info, platform_label)
|
||||
with tempfile.NamedTemporaryFile() as f:
|
||||
path = Path(f.name)
|
||||
_download(url, path)
|
||||
|
||||
Reference in new issue
Block a user