Hot-path optimizations for (Canonical)ValueStore::Add (#8031)

Extract the allocation path (which is inherently cold due to growth).
Making the outlining effective requires both blocking further inlining
and using `preserve_most` so that the existence of the call doesn't
disrupt the register allocation.

We also mark the insertion statements with callbacks as `always_inline`
so it being cold doesn't cause it to be left out-of-line and further
disrupt register allocation. It's a bit strange to force inlining of a
cold-path function, but the register impact of the call extends back
into the hot-path.

Agy's benchmarks (`./scripts/bench_runner.py --runs=12`, `-c opt
--copt=-march=x86-64-v3`, vs. `trunk`):
- `BM_ValidIdentifiers<3, 5, true>`: -6.277% CPU time
- `BM_ValidIdentifiers<3, 16, true>`: -4.525% CPU time
- `BM_CompileApiFileDenseDecls<Lang::Carbon, Phase::Lex>/262144`:
-2.012% CPU time (47.5 ms ± 6.358% -> 46.54 ms ± 1.773%, p=0.0167)

Assisted-by: Antigravity with Gemini

common/set.h

@@ -366,7 +366,9 @@ auto SetBase<InputKeyT, InputKeyContextT>::Insert(LookupKeyT lookup_key,
   return Insert(
       lookup_key,
       [&key_cb](LookupKeyT /*lookup_key*/, void* key_storage) {
-        new (key_storage) KeyT(key_cb());
+        // Despite being on the cold path, inlining is essential to keep
+        // fast-path register allocation efficient.
+        [[clang::always_inline]] new (key_storage) KeyT(key_cb());
       },
       key_context);
 }
@@ -386,7 +388,10 @@ auto SetBase<InputKeyT, InputKeyContextT>::Insert(LookupKeyT lookup_key,
     return InsertResult(false, entry->key());
   }
 
-  insert_cb(lookup_key, static_cast<void*>(&entry->key_storage));
+  // Despite being on the cold path, inlining is essential to keep fast-path
+  // register allocation efficient.
+  [[clang::always_inline]] insert_cb(lookup_key,
+                                     static_cast<void*>(&entry->key_storage));
   return InsertResult(true, entry->key());
 }
 

toolchain/base/value_store.h

@@ -263,8 +263,10 @@ class ValueStore
 
     CARBON_DCHECK(static_cast<size_t>(chunk_index) <= chunks_.size(),
                   "{0} <= {1}", chunk_index, chunks_.size());
-    if (static_cast<size_t>(chunk_index) == chunks_.size()) {
-      chunks_.emplace_back();
+    if (LLVM_UNLIKELY(static_cast<size_t>(chunk_index) == chunks_.size())) {
+      // Sink all the growth and allocation logic into an out-of-line helper as
+      // this is inherently a cold-path.
+      AllocateNextChunk();
     }
 
     CARBON_DCHECK(pos == chunks_[chunk_index].size());
@@ -520,6 +522,10 @@ class ValueStore
     return RawIndexToChunkIndices(GetRawIndex(id));
   }
 
+  [[clang::noinline, clang::preserve_most]] auto AllocateNextChunk() -> void {
+    chunks_.emplace_back();
+  }
+
   // Number of elements added to the store. The number should never exceed what
   // fits in an `int32_t`, which is checked in non-optimized builds in Add().
   int32_t size_ = 0;