mirror of
https://github.com/carbon-language/carbon-lang.git
synced 2026-09-26 22:02:30 +01:00
Jon Ross-Perkins
0405fff68c
Noticed due to https://github.com/carbon-language/carbon-lang/actions/runs/8207272518 ``` Node.js 16 actions are deprecated. Please update the following actions to use Node.js 20: actions/checkout@v3. For more information see: https://github.blog/changelog/2023-09-22-github-actions-transitioning-from-node-16-to-node-20/. ``` I'm expecting we can just update the version number on these.
423 lines
17 KiB
YAML
423 lines
17 KiB
YAML
# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
|
|
# Exceptions. See /LICENSE for license information.
|
|
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
|
|
name: test
|
|
|
|
on:
|
|
push:
|
|
branches: [trunk, action-test]
|
|
pull_request:
|
|
merge_group:
|
|
|
|
# Cancel previous workflows on the PR when there are multiple fast commits.
|
|
# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#concurrency
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
test:
|
|
strategy:
|
|
matrix:
|
|
# On PRs and in the merge queue test a recent version of each supported
|
|
# OS. On push (post-submit), also run on `macos-12` to get Intel macOS
|
|
# coverage.
|
|
runner:
|
|
${{ fromJSON(github.event_name != 'push' && '["ubuntu-22.04",
|
|
"macos-14"]' || '["ubuntu-22.04", "macos-14", "macos-12"]') }}
|
|
build_mode: [fastbuild, opt]
|
|
include:
|
|
# The clang-tidy config doesn't work on macos (missing `truncate`).
|
|
- runner: ubuntu-22.04
|
|
build_mode: clang-tidy
|
|
runs-on: ${{ matrix.runner }}
|
|
|
|
env:
|
|
os: ${{ startsWith(matrix.runner, 'ubuntu') && 'ubuntu' || 'macos' }}
|
|
|
|
steps:
|
|
# Ubuntu images start with 23GB available, and this adds 14GB more. For
|
|
# comparison, MacOS images have >100GB free.
|
|
- name: Free up disk space (Ubuntu)
|
|
if: env.os == 'ubuntu'
|
|
uses: jlumbroso/free-disk-space@v1.2.0
|
|
with:
|
|
android: true
|
|
dotnet: true
|
|
haskell: true
|
|
# Although we could delete more, if we run into a limit, it provides a
|
|
# little flexibility to get space while trying to shrink the build.
|
|
# There's also support for docker images at head (1.2.0 is still
|
|
# the latest release).
|
|
large-packages: false
|
|
swap-storage: false
|
|
|
|
# Checkout the pull request head or the branch.
|
|
- name: Checkout pull request
|
|
if: github.event_name == 'pull_request'
|
|
uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ github.event.pull_request.head.sha }}
|
|
|
|
- name: Checkout branch
|
|
if: github.event_name != 'pull_request'
|
|
uses: actions/checkout@v4
|
|
|
|
# Tests should only run on applicable paths, but we still need to have an
|
|
# action run for the merge queue. We filter steps based on the paths here,
|
|
# and condition steps on the output.
|
|
- id: filter
|
|
uses: dorny/paths-filter@v2
|
|
with:
|
|
filters: |
|
|
has_code:
|
|
- '!{**/*.md,LICENSE,CODEOWNERS,.git*}'
|
|
|
|
# Setup Python and related tools.
|
|
- uses: actions/setup-python@v4
|
|
if: steps.filter.outputs.has_code == 'true'
|
|
with:
|
|
# Match the min version listed in docs/project/contribution_tools.md
|
|
# or the oldest version available on the OS.
|
|
python-version: ${{ matrix.runner == 'macos-14' && '3.11' || '3.9' }}
|
|
|
|
# Install and cache LLVM 16 from Homebrew.
|
|
# TODO: We can potentially remove this and simplify things when the
|
|
# Homebrew version of LLVM updates to 16 here:
|
|
# https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md
|
|
- name: Cache Homebrew (macOS)
|
|
if: steps.filter.outputs.has_code == 'true' && env.os == 'macos'
|
|
id: cache-homebrew-macos
|
|
uses: actions/cache@v3
|
|
env:
|
|
cache-name: cache-homebrew
|
|
with:
|
|
# Cover all the critical parts of Homebrew here. Homebrew on Arm macOS
|
|
# uses its own prefix making this easy to cover, but we need a few
|
|
# different paths for Intel.
|
|
path: |
|
|
${{
|
|
runner.arch == 'ARM64' && '/opt/homebrew' ||
|
|
'
|
|
/usr/local/Homebrew
|
|
/usr/local/Cellar
|
|
/usr/local/Frameworks
|
|
/usr/local/bin
|
|
/usr/local/opt
|
|
'
|
|
}}
|
|
# Note the key needs to include all the packages we're adding.
|
|
key: Homebrew-Cache-${{ matrix.runner }}-${{ runner.arch }}
|
|
|
|
- name: Install LLVM and Clang with Homebrew (macOS)
|
|
if:
|
|
steps.filter.outputs.has_code == 'true' && env.os == 'macos' &&
|
|
steps.cache-homebrew-macos.outputs.cache-hit != 'true'
|
|
run: |
|
|
echo '*** Prune brew leaves'
|
|
# We prune all the leaf packages to have a minimal environment. This
|
|
# both minimizes the install space and avoids accidental dependencies
|
|
# on installed packages.
|
|
brew leaves
|
|
LEAVES=$(brew leaves | egrep -v '^(bazelisk|gh|git|git-lfs|gnu-tar|go@.*|jq|pipx|node@.*|openssl@.*|wget|yq|zlib)$')
|
|
brew uninstall -f --ignore-dependencies $LEAVES
|
|
echo '*** Installing LLVM deps'
|
|
brew install --force-bottle --only-dependencies llvm@16
|
|
echo '*** Installing LLVM itself'
|
|
brew install --force-bottle --force --verbose llvm@16
|
|
echo '*** brew info llvm@16'
|
|
brew info llvm@16
|
|
echo '*** brew autoremove'
|
|
brew autoremove
|
|
echo '*** brew info'
|
|
brew info
|
|
echo '*** brew leaves'
|
|
brew leaves
|
|
echo '*** brew config'
|
|
brew config
|
|
|
|
- name: Setup LLVM and Clang (macOS)
|
|
if: steps.filter.outputs.has_code == 'true' && env.os == 'macos'
|
|
run: |
|
|
LLVM_PATH="$(brew --prefix llvm@16)"
|
|
echo "Using ${LLVM_PATH}"
|
|
echo "${LLVM_PATH}/bin" >> $GITHUB_PATH
|
|
echo '*** ls "${LLVM_PATH}"'
|
|
ls "${LLVM_PATH}"
|
|
echo '*** ls "${LLVM_PATH}/bin"'
|
|
ls "${LLVM_PATH}/bin"
|
|
|
|
# Cache and install a recent version of LLVM. This uses the GitHub action
|
|
# cache to avoid directly downloading on each iteration and improve
|
|
# reliability.
|
|
- name: Cache LLVM and Clang installation (Ubuntu)
|
|
if: steps.filter.outputs.has_code == 'true' && env.os == 'ubuntu'
|
|
id: cache-llvm-ubuntu
|
|
uses: actions/cache@v3
|
|
env:
|
|
cache-name: cache-llvm
|
|
with:
|
|
path: ~/llvm
|
|
key: LLVM-16-Cache-${{ env.os }}-${{ runner.arch }}
|
|
|
|
- name: Download LLVM and Clang installation (Ubuntu)
|
|
if:
|
|
steps.filter.outputs.has_code == 'true' && env.os == 'ubuntu' &&
|
|
steps.cache-llvm-ubuntu.outputs.cache-hit != 'true'
|
|
run: |
|
|
cd ~
|
|
LLVM_RELEASE=clang+llvm-16.0.4-x86_64-linux-gnu-ubuntu-22.04
|
|
echo "*** Downloading $LLVM_RELEASE"
|
|
wget --show-progress=off "https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.4/$LLVM_RELEASE.tar.xz"
|
|
echo "*** Extracting $LLVM_RELEASE"
|
|
tar -xJf "$LLVM_RELEASE.tar.xz"
|
|
echo "*** Moving to 'llvm'"
|
|
mv "$LLVM_RELEASE" llvm
|
|
echo "*** Testing `clang++ --version`"
|
|
~/llvm/bin/clang++ --version
|
|
|
|
# The installation contains *huge* parts of LLVM we don't need for the
|
|
# toolchain. Prune them here to keep our cache small.
|
|
echo "*** Cleaning the 'llvm' directory"
|
|
rm llvm/lib/{*.a,*.so,*.so.*,*.bc}
|
|
rm llvm/bin/{flang-*,mlir-*,clang-{scan-deps,check,repl},*-test,llvm-{lto*,reduce,bolt*,exegesis,jitlink},bugpoint,opt,llc}
|
|
echo "*** Size of the 'llvm' directory"
|
|
du -hs llvm
|
|
|
|
- name: Setup LLVM and Clang paths (Ubuntu)
|
|
if: steps.filter.outputs.has_code == 'true' && env.os == 'ubuntu'
|
|
run: |
|
|
LLVM_PATH=~/llvm
|
|
echo "Using ${LLVM_PATH}"
|
|
echo "${LLVM_PATH}/bin" >> $GITHUB_PATH
|
|
echo '*** ls "${LLVM_PATH}"'
|
|
ls "${LLVM_PATH}"
|
|
echo '*** ls "${LLVM_PATH}/bin"'
|
|
ls "${LLVM_PATH}/bin"
|
|
|
|
# Print the various tool paths and versions to help in debugging.
|
|
- name: Print tool debugging info
|
|
if: steps.filter.outputs.has_code == 'true'
|
|
run: |
|
|
echo '*** PATH'
|
|
echo $PATH
|
|
echo '*** bazelisk'
|
|
which bazelisk
|
|
bazelisk --version
|
|
echo '*** python'
|
|
which python
|
|
python --version
|
|
echo '*** clang'
|
|
which clang
|
|
clang --version
|
|
echo '*** clang++'
|
|
which clang++
|
|
clang++ --version
|
|
|
|
# Disable uploads when the remote cache is read-only.
|
|
- name: Set up remote cache access (read-only)
|
|
if:
|
|
steps.filter.outputs.has_code == 'true' && github.event_name ==
|
|
'pull_request'
|
|
run: |
|
|
echo "remote_cache_upload=--remote_upload_local_results=false" \
|
|
>> $GITHUB_ENV
|
|
|
|
# Provide a cache key when the remote cache is read-write.
|
|
- name: Set up remote cache access (read-write)
|
|
if:
|
|
steps.filter.outputs.has_code == 'true' && github.event_name !=
|
|
'pull_request'
|
|
env:
|
|
REMOTE_CACHE_KEY: ${{ secrets.CARBON_BUILDS_GITHUB }}
|
|
run: |
|
|
echo "$REMOTE_CACHE_KEY" | base64 -d > $HOME/remote_cache_key.json
|
|
echo "remote_cache_upload=--google_credentials=$HOME/remote_cache_key.json" \
|
|
>> $GITHUB_ENV
|
|
|
|
# Add our bazel configuration and print basic info to ease debugging.
|
|
- name: Configure Bazel and print info
|
|
if: steps.filter.outputs.has_code == 'true'
|
|
env:
|
|
# Add a cache version for changes that bazel won't otherwise detect,
|
|
# like llvm version changes.
|
|
CACHE_VERSION: 1
|
|
run: |
|
|
cat >user.bazelrc <<EOF
|
|
# Enable remote cache for our CI but minimize downloads.
|
|
build --remote_cache=https://storage.googleapis.com/carbon-builds-github-v${CACHE_VERSION}
|
|
build --remote_download_minimal
|
|
|
|
# We import a special key into every action in order to key the Bazel
|
|
# remote cache in a way that avoids collisions between different
|
|
# runners. Anything that might change the system external to Bazel but
|
|
# not be fully captured by the sand-boxing of the build should be used
|
|
# as part of the key. We don't need to use the CPU target for example,
|
|
# as that is captured by Bazel's configuration of each action. And the
|
|
# Clang version is incorporated by our Clang toolchain setup. But we
|
|
# do need to capture any differences between GitHub runner OSes that
|
|
# don't impact the Bazel configuration to avoid collisions between
|
|
# those.
|
|
build --action_env=BAZEL_REMOTE_CACHE_KEY=github-action-${{ matrix.runner }}
|
|
|
|
build ${{ env.remote_cache_upload }}
|
|
|
|
# Set an artificially high jobs count. This flag controls the number
|
|
# of concurrency Bazel itself uses, which is essential for actions
|
|
# that are internally blocked on for example downloading results form
|
|
# the cache above. Without setting this high, Bazel will pick a small
|
|
# number based on the available host CPUs and the reality will be a
|
|
# long chain of largely serialized download events with little or no
|
|
# usage of the host machine. Fortunately, local actions are
|
|
# *separately* gated on '--local_*_resources' that will avoid a large
|
|
# jobs value overwhelming the host. There is a bug to make downloads
|
|
# behave completely asynchronously and remove the need for this filed
|
|
# back in 2018 but work seemed to not finish:
|
|
# https://github.com/bazelbuild/bazel/issues/6394
|
|
#
|
|
# There is a new effort (yay!) but until then it seems worth using the
|
|
# workaround of a high jobs value. The biggest downside (increased
|
|
# heap usage) seems like it isn't currently a big loss for our builds.
|
|
#
|
|
# Higher values like 50 have led to CI failures with network errors
|
|
# and IOExceptions, see
|
|
# https://discord.com/channels/655572317891461132/707150492370862090/1151605725576056934
|
|
build --jobs=32
|
|
|
|
# General build options.
|
|
build --verbose_failures
|
|
test --test_output=errors
|
|
EOF
|
|
bazelisk info
|
|
|
|
# Just for visibility, print space before and after the build.
|
|
- name: Disk space before build
|
|
if: steps.filter.outputs.has_code == 'true'
|
|
run: df -h
|
|
|
|
- name: Verify MODULE.bazel.lock
|
|
if: steps.filter.outputs.has_code == 'true'
|
|
run: |
|
|
exit_code=0
|
|
bazelisk mod deps --lockfile_mode=error || exit_code=$?
|
|
if (( $exit_code != 0 )); then
|
|
bazelisk mod deps --lockfile_mode=update
|
|
echo "MODULE.bazel.lock is out of date! Use below file for update."
|
|
echo "Platforms may require merging output, for example by applying"
|
|
echo "an update, re-running triggers, and applying the next update."
|
|
echo "============================================================"
|
|
cat MODULE.bazel.lock
|
|
echo "============================================================"
|
|
exit 1
|
|
fi
|
|
|
|
# Build and run all targets on branch pushes to ensure we always have a
|
|
# clean tree. We don't expect this to be an interactive path and so don't
|
|
# optimize the latency of this step.
|
|
- name: Compute impacted pull request targets (for push)
|
|
if:
|
|
steps.filter.outputs.has_code == 'true' && github.event_name == 'push'
|
|
env:
|
|
TARGETS_FILE: ${{ runner.temp }}/targets
|
|
run: |
|
|
echo "//..." >$TARGETS_FILE
|
|
|
|
# Compute the set of possible rules impacted by this change using
|
|
# Bazel-based diffing. This lets PRs and the merge queue have a much more
|
|
# efficient test CI action by avoiding even enumerating (and downloading)
|
|
# all of the unaffected Bazel targets.
|
|
- name: Compute impacted pull request targets
|
|
if:
|
|
steps.filter.outputs.has_code == 'true' && github.event_name != 'push'
|
|
env:
|
|
# Compute the base SHA from the different event structures.
|
|
GIT_BASE_SHA:
|
|
${{ github.event_name == 'pull_request' &&
|
|
github.event.pull_request.base.sha ||
|
|
github.event.merge_group.base_sha }}
|
|
TARGETS_FILE: ${{ runner.temp }}/targets
|
|
run: |
|
|
# First fetch the relevant base into the git repository.
|
|
git fetch --depth=1 origin $GIT_BASE_SHA
|
|
|
|
# Then use `target-determinator` as wrapped by our script.
|
|
./scripts/target_determinator.py $GIT_BASE_SHA >$TARGETS_FILE
|
|
|
|
# Bazel requires a test target to run the test command. There may be
|
|
# no targets or there may only be non-test targets that we want to
|
|
# build, so simply inject an explicit no-op test target.
|
|
echo "//scripts:no_op_test" >> $TARGETS_FILE
|
|
|
|
# Build and run just the tests impacted by the PR or merge group.
|
|
- name: Test (${{ matrix.build_mode }})
|
|
if:
|
|
steps.filter.outputs.has_code == 'true' && matrix.build_mode !=
|
|
'clang-tidy'
|
|
env:
|
|
# 'libtool_check_unique failed to generate' workaround.
|
|
# https://github.com/bazelbuild/bazel/issues/14113#issuecomment-999794586
|
|
BAZEL_USE_CPP_ONLY_TOOLCHAIN: 1
|
|
TARGETS_FILE: ${{ runner.temp }}/targets
|
|
run: |
|
|
for i in {1..5}; do
|
|
if (( $i == 4 )); then
|
|
# Decrease the jobs sharply if we see repeated failures to try to
|
|
# work around transient network errors even if it makes things
|
|
# slower.
|
|
echo "build --jobs=4" >>user.bazelrc
|
|
fi
|
|
|
|
bazel_exit=0
|
|
bazelisk test -c ${{ matrix.build_mode }} \
|
|
--target_pattern_file=$TARGETS_FILE || bazel_exit=$?
|
|
|
|
# If we succeed, we're done.
|
|
if (( $bazel_exit == 0 )); then
|
|
break
|
|
fi
|
|
|
|
# Several error codes are reliably permanent, break immediately.
|
|
# `1` -- The build failed.
|
|
# `2` -- Command line or environment problem.
|
|
# `3` -- Tests failed or timed out, we don't retry at this layer
|
|
# on execution timeout.
|
|
# `4` -- No tests found, which should be impossible here.
|
|
# `8` -- Explicitly interrupted build.
|
|
#
|
|
# Note that `36` is documented as "likely permanent", but we retry
|
|
# it as most of our transient failures actually produce that error
|
|
# code.
|
|
if (( $bazel_exit == 1 || $bazel_exit == 2 || $bazel_exit == 3 || \
|
|
$bazel_exit == 4 || $bazel_exit == 8 || $bazel_exit == 8 ))
|
|
then
|
|
break
|
|
fi
|
|
|
|
echo "Retrying a failed build as it may be transient..."
|
|
# Also sleep a bit to try to skip over transient machine load.
|
|
sleep $i
|
|
done
|
|
|
|
# Propagate the Bazel exit code.
|
|
exit $bazel_exit
|
|
|
|
# Run in the clang-tidy config. This is done as part of tests so that we
|
|
# aren't duplicating bazel/llvm setup.
|
|
#
|
|
# The `-k` flag is used to print all clang-tidy errors.
|
|
- name: clang-tidy
|
|
if:
|
|
steps.filter.outputs.has_code == 'true' && matrix.build_mode ==
|
|
'clang-tidy'
|
|
env:
|
|
TARGETS_FILE: ${{ runner.temp }}/targets
|
|
run: |
|
|
bazelisk build --config=clang-tidy -k \
|
|
--target_pattern_file=$TARGETS_FILE
|
|
|
|
# See "Disk space before build".
|
|
- name: Disk space after build
|
|
if: steps.filter.outputs.has_code == 'true'
|
|
run: df -h
|