From 21c825a3f7dac3cc96a6a1440228aaa184b67b44 Mon Sep 17 00:00:00 2001 From: Chandler Carruth Date: Mon, 26 Oct 2026 06:38:17 +0000 Subject: [PATCH] Hot-path optimizations for `(Canonical)ValueStore::Add` (#8031) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract the allocation path (which is inherently cold due to growth). Making the outlining effective requires both blocking further inlining and using `preserve_most` so that the existence of the call doesn't disrupt the register allocation. We also mark the insertion statements with callbacks as `always_inline` so it being cold doesn't cause it to be left out-of-line and further disrupt register allocation. It's a bit strange to force inlining of a cold-path function, but the register impact of the call extends back into the hot-path. Agy's benchmarks (`./scripts/bench_runner.py --runs=12`, `-c opt --copt=-march=x86-64-v3`, vs. `trunk`): - `BM_ValidIdentifiers<3, 5, true>`: -6.277% CPU time - `BM_ValidIdentifiers<3, 16, true>`: -4.525% CPU time - `BM_CompileApiFileDenseDecls/262144`: -2.012% CPU time (47.5 ms ± 6.358% -> 46.54 ms ± 1.773%, p=0.0167) Assisted-by: Antigravity with Gemini --- common/set.h | 9 +++++++-- toolchain/base/value_store.h | 10 ++++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/common/set.h b/common/set.h index f38c7bfd709d..f10379bf4b10 100644 --- a/common/set.h +++ b/common/set.h @@ -366,7 +366,9 @@ auto SetBase::Insert(LookupKeyT lookup_key, return Insert( lookup_key, [&key_cb](LookupKeyT /*lookup_key*/, void* key_storage) { - new (key_storage) KeyT(key_cb()); + // Despite being on the cold path, inlining is essential to keep + // fast-path register allocation efficient. + [[clang::always_inline]] new (key_storage) KeyT(key_cb()); }, key_context); } @@ -386,7 +388,10 @@ auto SetBase::Insert(LookupKeyT lookup_key, return InsertResult(false, entry->key()); } - insert_cb(lookup_key, static_cast(&entry->key_storage)); + // Despite being on the cold path, inlining is essential to keep fast-path + // register allocation efficient. + [[clang::always_inline]] insert_cb(lookup_key, + static_cast(&entry->key_storage)); return InsertResult(true, entry->key()); } diff --git a/toolchain/base/value_store.h b/toolchain/base/value_store.h index 0e9de4e6496f..bea2e2509675 100644 --- a/toolchain/base/value_store.h +++ b/toolchain/base/value_store.h @@ -263,8 +263,10 @@ class ValueStore CARBON_DCHECK(static_cast(chunk_index) <= chunks_.size(), "{0} <= {1}", chunk_index, chunks_.size()); - if (static_cast(chunk_index) == chunks_.size()) { - chunks_.emplace_back(); + if (LLVM_UNLIKELY(static_cast(chunk_index) == chunks_.size())) { + // Sink all the growth and allocation logic into an out-of-line helper as + // this is inherently a cold-path. + AllocateNextChunk(); } CARBON_DCHECK(pos == chunks_[chunk_index].size()); @@ -520,6 +522,10 @@ class ValueStore return RawIndexToChunkIndices(GetRawIndex(id)); } + [[clang::noinline, clang::preserve_most]] auto AllocateNextChunk() -> void { + chunks_.emplace_back(); + } + // Number of elements added to the store. The number should never exceed what // fits in an `int32_t`, which is checked in non-optimized builds in Add(). int32_t size_ = 0;