diff --git a/common/set.h b/common/set.h index f38c7bfd709d..f10379bf4b10 100644 --- a/common/set.h +++ b/common/set.h @@ -366,7 +366,9 @@ auto SetBase::Insert(LookupKeyT lookup_key, return Insert( lookup_key, [&key_cb](LookupKeyT /*lookup_key*/, void* key_storage) { - new (key_storage) KeyT(key_cb()); + // Despite being on the cold path, inlining is essential to keep + // fast-path register allocation efficient. + [[clang::always_inline]] new (key_storage) KeyT(key_cb()); }, key_context); } @@ -386,7 +388,10 @@ auto SetBase::Insert(LookupKeyT lookup_key, return InsertResult(false, entry->key()); } - insert_cb(lookup_key, static_cast(&entry->key_storage)); + // Despite being on the cold path, inlining is essential to keep fast-path + // register allocation efficient. + [[clang::always_inline]] insert_cb(lookup_key, + static_cast(&entry->key_storage)); return InsertResult(true, entry->key()); } diff --git a/toolchain/base/value_store.h b/toolchain/base/value_store.h index 0e9de4e6496f..bea2e2509675 100644 --- a/toolchain/base/value_store.h +++ b/toolchain/base/value_store.h @@ -263,8 +263,10 @@ class ValueStore CARBON_DCHECK(static_cast(chunk_index) <= chunks_.size(), "{0} <= {1}", chunk_index, chunks_.size()); - if (static_cast(chunk_index) == chunks_.size()) { - chunks_.emplace_back(); + if (LLVM_UNLIKELY(static_cast(chunk_index) == chunks_.size())) { + // Sink all the growth and allocation logic into an out-of-line helper as + // this is inherently a cold-path. + AllocateNextChunk(); } CARBON_DCHECK(pos == chunks_[chunk_index].size()); @@ -520,6 +522,10 @@ class ValueStore return RawIndexToChunkIndices(GetRawIndex(id)); } + [[clang::noinline, clang::preserve_most]] auto AllocateNextChunk() -> void { + chunks_.emplace_back(); + } + // Number of elements added to the store. The number should never exceed what // fits in an `int32_t`, which is checked in non-optimized builds in Add(). int32_t size_ = 0;