Files
carbon-lang/.github/workflows/benchmarks.yaml
T
Chandler Carruth f789ad4bdd Fix benchmarks.yaml workflow failure due to syntax errors (#8156)
We were setting up `BMF_FILE` environment variable in the job rather
than the step, and we don't have a `runner` variable available at that
point. Sink it into the step where it can resolve successfully.

Assisted-by: Antigravity with Gemini
2026-11-18 17:07:56 +00:00

198 lines
6.9 KiB
YAML

# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
# Exceptions. See /LICENSE for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
name: Benchmarks
on:
push:
branches: [trunk]
# Chain nightly benchmarks directly after `Nightly Release` finishes so there
# is only a single nightly cron schedule, no runner contention with the
# release build, and benchmarks run on the exact release tag commit.
# zizmor: ignore[dangerous-triggers] - {Nightly Release only runs on trunk}
workflow_run:
workflows: ['Nightly Release']
types:
- completed
workflow_dispatch:
inputs:
tier:
description: 'Benchmark suite tier to execute'
required: false
default: 'nightly'
type: choice
options:
- continuous
- nightly
permissions:
contents: read # For actions/checkout.
jobs:
benchmark:
name: Benchmark (${{ matrix.tier }}, ${{ matrix.runner }})
if: |
github.event_name != 'workflow_run' ||
(github.event.workflow_run.conclusion == 'success' &&
github.event.workflow_run.head_repository.full_name == github.repository &&
github.event.workflow_run.head_branch == 'trunk' &&
github.event.workflow_run.event != 'pull_request')
# Coalesce rapid pushes on `trunk` per runner without canceling an
# in-progress benchmark measurement.
concurrency:
group:
${{ github.workflow }}-${{ github.event_name }}-${{ matrix.runner }}-${{
github.ref }}
cancel-in-progress: false
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
runner: ['ubuntu-22.04', 'ubuntu-22.04-arm', 'macos-26']
tier:
- ${{ (github.event_name == 'workflow_run' && 'nightly') ||
(github.event_name == 'workflow_dispatch' && inputs.tier) ||
'continuous' }}
exclude:
- runner: 'macos-26'
tier: 'continuous'
runs-on: ${{ matrix.runner }}
steps:
- name: Harden Runner
uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4
with:
egress-policy: block
# When adding endpoints, see README.md.
# prettier-ignore
allowed-endpoints: >
*.blob.storage.azure.net:443
*.githubapp.com:443
*.sourceforge.net:443
api.bencher.dev:443
api.github.com:443
api.ipify.org:443
bcr.bazel.build:443
download.gnome.org:443
downloads.sourceforge.net:443
files.pythonhosted.org:443
formulae.brew.sh:443
github.com:443
go.dev:443
index.crates.io:443
mirror.bazel.build:443
mirrors.kernel.org:443
nodejs.org:443
oauth2.googleapis.com:443
objects.githubusercontent.com:443
pkg-containers.githubusercontent.com:443
pypi.org:443
raw.githubusercontent.com:443
registry.npmjs.org:443
release-assets.githubusercontent.com:443
releases.astral.sh:443
releases.bazel.build:443
static.crates.io:443
static.rust-lang.org:443
storage.googleapis.com:443
uploads.github.com:443
www.googleapis.com:443
- name: Checkout repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref:
${{ github.event_name == 'workflow_run' &&
github.event.workflow_run.head_sha || '' }}
persist-credentials: false
- id: filter
if: github.event_name == 'push'
uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3
with:
filters: |
has_code:
- '!{**/*.md,LICENSE,CODEOWNERS,.git*}'
- name: Set up remote cache access
id: cache-setup
if: steps.filter.outputs.has_code != 'false'
env:
REMOTE_CACHE_KEY: ${{ secrets.CARBON_BUILDS_GITHUB }}
run: |
if [[ -n "$REMOTE_CACHE_KEY" ]]; then
echo "$REMOTE_CACHE_KEY" | base64 -d > "$HOME/remote_cache_key.json"
echo "remote_cache_upload=--google_credentials=$HOME/remote_cache_key.json" \
>> "$GITHUB_OUTPUT"
else
echo "remote_cache_upload=--remote_upload_local_results=false" \
>> "$GITHUB_OUTPUT"
fi
- uses: ./.github/actions/build-setup-common
if: steps.filter.outputs.has_code != 'false'
with:
matrix_runner: ${{ matrix.runner }}
remote_cache_upload:
${{ steps.cache-setup.outputs.remote_cache_upload }}
- name: Run benchmarks
id: run-bencher
if: steps.filter.outputs.has_code != 'false'
env:
# 'libtool_check_unique failed to generate' workaround.
# https://github.com/bazelbuild/bazel/issues/14113#issuecomment-999794586
BAZEL_USE_CPP_ONLY_TOOLCHAIN: 1
BENCH_TIER: ${{ matrix.tier }}
BMF_FILE: ${{ runner.temp }}/bmf-${{ matrix.runner }}.json
run: |
./toolchain/benchmarking/run_bencher.py run \
--ci \
--tier="$BENCH_TIER" \
--output="$BMF_FILE" \
--github-output="$GITHUB_OUTPUT"
- name: Install Bencher CLI
if: steps.run-bencher.outputs.has_benchmarks == 'true'
uses: bencherdev/bencher@7de3c69a85380826613050826b4bce062be2f07d # v0.6.16
- name: Upload results to Bencher
if: steps.run-bencher.outputs.has_benchmarks == 'true'
env:
BENCHER_API_KEY: ${{ secrets.BENCHER_API_KEY }}
BENCH_TIER: ${{ matrix.tier }}
BMF_FILE: ${{ runner.temp }}/bmf-${{ matrix.runner }}.json
RUNNER: ${{ matrix.runner }}
TARGET_SHA:
${{ github.event_name == 'workflow_run' &&
github.event.workflow_run.head_sha || github.sha }}
run: |
if [[ "$GITHUB_REF_NAME" == "trunk" && "$BENCH_TIER" == "nightly" ]]; then
bencher_branch="nightly"
else
bencher_branch="$GITHUB_REF_NAME"
fi
threshold_output="$(./toolchain/benchmarking/run_bencher.py threshold-args)"
extra_args=("--thresholds-reset")
if [[ "$bencher_branch" == "action-test" ]]; then
extra_args+=("--start-point-reset")
else
while IFS= read -r arg; do
[[ -n "$arg" ]] && extra_args+=("$arg")
done <<<"$threshold_output"
fi
bencher run \
--project carbon-lang \
--branch "$bencher_branch" \
--hash "$TARGET_SHA" \
--testbed "$RUNNER" \
--adapter json \
--file "$BMF_FILE" \
"${extra_args[@]}"
if [[ "$RUNNER" == "ubuntu-22.04" ]]; then
./toolchain/benchmarking/run_bencher.py sync-measures
fi