From 07cee1b82a302267af38e6fc5d9c21bf1335d1f9 Mon Sep 17 00:00:00 2001 From: Chandler Carruth Date: Wed, 16 Jun 2021 16:25:52 -0700 Subject: [PATCH] Switch to the Bazel disk cache (locally and in CI). (#579) The Bazel disk cache provides a stable, durable, and supported persistent cache of the intermediate steps of Bazel builds. This is like `ccache` but covering more steps and more reliable. It is based on the same fundamental infrastructure that enables a *remote* build cache, and at some point we may want to enable that so that our CI can pre-seed a cache for users. But for now, this just enables local support for this. For local builds this will ensure that intermediate artifacts are re-used when changing branches, syncing, across Bazel server restarts or even upgrades. It should dramatically reduce the need to re-build large and slow components like all of Clang and LLVM. Largely, those should only be rebuilt when we update our pinned revision of upstream and then cached persistently on a given machine. Enabling this by default requires hard-coding a path, and so I've also added built-in support for importing a user config file if present so that folks can customize this (and any other Bazel features) as desired. This also wires the disk cache up for the GitHub cache in our CI. By using the designated Bazel disk cache for this, we get a *much* simpler and *much* more robust solution. So much so that I've re-arranged the key structure so we can more aggressively re-use old caches. The only downside is that nothing ever gets removed automatically from the cache. I've added a layer that will trim any unused file roughly once a day. For CI, the benefits of this are huge. None of the previous hacks are needed to locate and preserve the cache. The design of the cache is deeply durable and so we don't need to aggressively invalidate it. Last but not least, it focuses the cache on the raw artifacts, which compress exceptionally well. With the other changes to shrink the build outputs and this one combined, I expect we will be able to enable all of the build targets across all four config/os combinations without any significant pressure on our space quota for GitHub action caches. --- .bazelrc | 13 +++++++ .github/workflows/tests.yaml | 67 ++++++++++++++++++------------------ .gitignore | 3 ++ 3 files changed, 50 insertions(+), 33 deletions(-) diff --git a/.bazelrc b/.bazelrc index 4a2c465678a7..5da6e3def3c8 100644 --- a/.bazelrc +++ b/.bazelrc @@ -5,6 +5,15 @@ build --crosstool_top=@bazel_cc_toolchain build --host_crosstool_top=@bazel_cc_toolchain +# Default to using a disk cache to minimize re-building LLVM and Clang which we +# try to avoid updating too frequently to minimize rebuild cost. The location +# here can be overridden in the user configuration where needed. +build --disk_cache=~/.cache/carbon-lang-build-cache + +# Enable some safety when using the build cache, likely to be defaulted in +# future Bazel releases. +build --experimental_guard_against_concurrent_changes + # Allow the toolchain to configure itself differently in the host build from # the target build. Even when the host and target platforms are ostensibly the # same (and use identical toolchains), it is beneficial to be able to configure @@ -45,3 +54,7 @@ build --test_env=ASAN_SYMBOLIZER_PATH # Linux and macOS this seems strictly better than the Bazel default of just # `en_US`. build --action_env=LANG=en_US.UTF-8 + +# Allow users to override any of the flags desired by importing a user-specific +# RC file here if present. +try-import %workspace%/user.bazelrc diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index b4c6874e1c9d..9b0fb9d8dfb4 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -35,8 +35,7 @@ jobs: steps: - name: Create environment variables run: | - echo "BAZEL_OUTPUT_BASE_PATH=$HOME/.cache/bazel-output-base" >> $GITHUB_ENV - echo "BAZEL=bazelisk --output_base=$HOME/.cache/bazel-output-base" >> $GITHUB_ENV + echo "BAZEL_DISK_CACHE_PATH=$HOME/.cache/carbon-lang-build-cache" >> $GITHUB_ENV - name: Checkout uses: actions/checkout@v2 @@ -108,58 +107,50 @@ jobs: echo "::set-output name=date::$(/bin/date -u "+%Y%m%d")" # Preserve and restore the Bazel cache across builds. - # - # These cached entries are keyed on this file (which configures the - # command and other inputs) as well as the date. Because Bazel is - # hermetic, it doesn't matter if we get a "stale" entry here, but it makes - # the cache less efficient. So we use the date to get a fresh baseline - # cache once per day. We include this file so that changing the build - # structure itself returns us to a clean state. We can also increment the - # counter in this comment to forcibly rotate this hash in the event we get - # a corrupted cache that breaks Bazel. This is a risk because we very - # selectively prune directories out of the internals of the Bazel output - # base to reduce the storage of the cache. At any point, this might break - # invariants and require us to clear our caches. There is no way to - # actually delete them and so instead we can simply bump the counter here. - # - # Current cache version: 2 - name: Cache Bazel build data uses: actions/cache@v2 with: - # We exclude the install symlink so that Bazel reconnects to the - # current installation. This helps repair any damage done by other - # exclusions as well. Then we exclude the `external` tree because - # those are all downloaded and rapidly rebuilt, so no benefit from - # downloading them with the cached state. path: | - ${{ env.BAZEL_OUTPUT_BASE_PATH }} - !${{ env.BAZEL_OUTPUT_BASE_PATH }}/install - !${{ env.BAZEL_OUTPUT_BASE_PATH }}/external + ${{ env.BAZEL_DISK_CACHE_PATH }} + # The `bazel-1` string here and below in the key acts as a counter + # that can be incremented to rebase the cache in the event of + # persistent corruption or other rare bug. The `-1-` prefix to the + # date can be incremented when adding new targets or updating this + # configuration and a new cache should be created but there is no + # reason to discard prior caches. key: | - bazel-${{ matrix.os }}-${{ matrix.build_mode }}-${{ hashFiles('.github/workflows/tests.yaml') - }}-${{ steps.date.outputs.date }} + bazel-1-${{ matrix.os }}-${{ matrix.build_mode }}-1-${{ steps.date.outputs.date }} # When we get a cache miss, try finding the most recent previous day's # cache to start. restore-keys: | - bazel-${{ matrix.os }}-${{ matrix.build_mode }}-${{ hashFiles('.github/workflows/tests.yaml') - }}- + bazel-1-${{ matrix.os }}-${{ matrix.build_mode }}- + + # Reset all the access-times for the bazel disk cache to 1984. We really + # just need a time far in the past to ensure even with `relatime` or + # similar, the filesystem will track how much of the disk cache is + # actually still *used* in the course of the build. This lets us GC + # anything unused before potentially creating a new cache entry. + - name: Reset Bazel disk cache atimes + run: | + mkdir -p ${{ env.BAZEL_DISK_CACHE_PATH }} + find ${{ env.BAZEL_DISK_CACHE_PATH }}/ -type f -exec touch -a -t 198401010000 '{}' '+' # Print Bazel diagnostics to make debugging easier. - name: Print Bazel info run: | - ${{ env.BAZEL }} info + bazelisk info # Build all targets first to isolate build failures. - name: Build (${{ matrix.build_mode }}) run: | - ${{ env.BAZEL }} build -c ${{ matrix.build_mode }} --verbose_failures \ + bazelisk build -c ${{ matrix.build_mode }} --verbose_failures \ --deleted_packages=migrate_cpp,migrate_cpp/cpp_refactoring \ //...:all # Run all test targets. - name: Test (${{ matrix.build_mode }}) run: | - ${{ env.BAZEL }} test -c ${{ matrix.build_mode }} --test_output errors \ + bazelisk test -c ${{ matrix.build_mode }} --test_output errors \ --deleted_packages=migrate_cpp,migrate_cpp/cpp_refactoring \ --verbose_failures //...:all @@ -167,4 +158,14 @@ jobs: # don't interact with it. - name: Shutdown Bazel run: | - ${{ env.BAZEL }} shutdown + bazelisk shutdown + + # Since we reset the `atime`s of the disk cache to ancient history, + # everything that ended up accessed by the build and test should have a + # recent `atime`. Delete any files whose `atime` is more than 7 days old. + # We print some statistics to try to help with any debugging later on. + - name: Remove unused disk cache files + run: | + find ${{ env.BAZEL_DISK_CACHE_PATH }}/ -type f | wc -l + find ${{ env.BAZEL_DISK_CACHE_PATH }}/ -type f -atime +7 | wc -l + find ${{ env.BAZEL_DISK_CACHE_PATH }}/ -type f -atime +7 -delete diff --git a/.gitignore b/.gitignore index 23c51710f009..cb9665302205 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,9 @@ # Directories created by clangd /.cache/clangd +# User-specific Bazel configuration. +/user.bazelrc + # Compilation database used by clangd compile_commands.json