From b39c7c93aaa5b95a7defd66a73d1e198ed975a1f Mon Sep 17 00:00:00 2001 From: Chandler Carruth Date: Fri, 27 Jun 2025 17:52:58 -0700 Subject: [PATCH] Add hashtable benchmark coverage for integers with low zero bits (#5735) These have unique challenges for our hashing scheme, and so its useful to make sure the hash functions we use can handle them. Some other work on Abseil's hash tables uncovered that this might be risky and may have surfaced some improvements to reduce the impact here, but the first step seems to try and start covering this path in the benchmarks. --------- Co-authored-by: Jon Ross-Perkins --- common/BUILD | 1 + common/map_benchmark.cpp | 17 ++++++ common/raw_hashtable_benchmark_helpers.cpp | 43 ++++++++++++++ common/raw_hashtable_benchmark_helpers.h | 69 ++++++++++++++++++++++ 4 files changed, 130 insertions(+) diff --git a/common/BUILD b/common/BUILD index 9fb63616e6a1..44cba49a8d8d 100644 --- a/common/BUILD +++ b/common/BUILD @@ -476,6 +476,7 @@ cc_library( "@abseil-cpp//absl/base:no_destructor", "@abseil-cpp//absl/hash", "@abseil-cpp//absl/random", + "@boost_unordered", "@google_benchmark//:benchmark", "@llvm-project//llvm:Support", ], diff --git a/common/map_benchmark.cpp b/common/map_benchmark.cpp index 9b7c311bd680..e80aa7e743f9 100644 --- a/common/map_benchmark.cpp +++ b/common/map_benchmark.cpp @@ -19,6 +19,7 @@ using RawHashtable::CarbonHashDI; using RawHashtable::GetKeysAndHitKeys; using RawHashtable::GetKeysAndMissKeys; using RawHashtable::HitArgs; +using RawHashtable::LowZeroBitInt; using RawHashtable::ReportTableMetrics; using RawHashtable::SizeArgs; using RawHashtable::ValueToBool; @@ -328,6 +329,22 @@ static void BM_MapLookupHit(benchmark::State& state) { } MAP_BENCHMARK_ONE_OP(BM_MapLookupHit, HitArgs); +// We also do some minimal benchmarking with integers that have a +// large number of low zero bits shifted into them. These present particular +// challenges to the hashing strategy Carbon's hash tables use and so they help +// form stress tests and benchmark to make sure the hash function quality +// remains reasonable even under adverse conditions. We can't go past a certain +// limit here without our hash tables becoming impossibly slow due to complete +// collapse of the hash functions -- if we ever need to hash integers with more +// than 32 low zero bits, we'll ask that code to use a custom hash algorithm. +// +// We don't benchmark these everywhere as they only provide marginal information +// beyond the core types, and checking just this operation covers that +// sufficiently. +MAP_BENCHMARK_ONE_OP_SIZE(BM_MapLookupHit, HitArgs, LowZeroBitInt<12>, int); +MAP_BENCHMARK_ONE_OP_SIZE(BM_MapLookupHit, HitArgs, LowZeroBitInt<24>, int); +MAP_BENCHMARK_ONE_OP_SIZE(BM_MapLookupHit, HitArgs, LowZeroBitInt<32>, int); + // This is an update throughput benchmark in practice. While whether the key was // a hit is kept in the critical path, we only use keys that are hits and so // expect that to be fully predicted and speculated. diff --git a/common/raw_hashtable_benchmark_helpers.cpp b/common/raw_hashtable_benchmark_helpers.cpp index d421d3fd846f..11f431beae16 100644 --- a/common/raw_hashtable_benchmark_helpers.cpp +++ b/common/raw_hashtable_benchmark_helpers.cpp @@ -221,6 +221,16 @@ static absl::NoDestructor> raw_int_keys{[] { return keys; }()}; +template +static absl::NoDestructor>> + raw_low_zero_bit_int_keys{[] { + llvm::OwningArrayRef> keys(MaxNumKeys); + for (auto [index, key] : llvm::enumerate(keys)) { + key = LowZeroBitInt(index + 1); + } + return keys; + }()}; + namespace { // Allow generically dispatching over the specific key types we build and @@ -231,6 +241,12 @@ auto GetRawKeys() -> llvm::ArrayRef { return *raw_str_keys; } else if constexpr (std::is_pointer_v) { return *raw_ptr_keys; + } else if constexpr (std::is_same_v>) { + return *raw_low_zero_bit_int_keys<12>; + } else if constexpr (std::is_same_v>) { + return *raw_low_zero_bit_int_keys<24>; + } else if constexpr (std::is_same_v>) { + return *raw_low_zero_bit_int_keys<32>; } else { return *raw_int_keys; } @@ -305,6 +321,15 @@ template auto GetKeysAndMissKeys(ssize_t size) template auto GetKeysAndMissKeys(ssize_t size) -> std::pair, llvm::ArrayRef>; +template auto GetKeysAndMissKeys>(ssize_t size) + -> std::pair>, + llvm::ArrayRef>>; +template auto GetKeysAndMissKeys>(ssize_t size) + -> std::pair>, + llvm::ArrayRef>>; +template auto GetKeysAndMissKeys>(ssize_t size) + -> std::pair>, + llvm::ArrayRef>>; template auto GetKeysAndHitKeys(ssize_t table_keys_size, ssize_t lookup_keys_size) @@ -322,6 +347,18 @@ template auto GetKeysAndHitKeys(ssize_t size, ssize_t lookup_keys_size) -> std::pair, llvm::ArrayRef>; +template auto GetKeysAndHitKeys>(ssize_t size, + ssize_t lookup_keys_size) + -> std::pair>, + llvm::ArrayRef>>; +template auto GetKeysAndHitKeys>(ssize_t size, + ssize_t lookup_keys_size) + -> std::pair>, + llvm::ArrayRef>>; +template auto GetKeysAndHitKeys>(ssize_t size, + ssize_t lookup_keys_size) + -> std::pair>, + llvm::ArrayRef>>; template auto DumpHashStatistics(llvm::ArrayRef keys) -> void { @@ -381,6 +418,12 @@ auto DumpHashStatistics(llvm::ArrayRef keys) -> void { } } template auto DumpHashStatistics(llvm::ArrayRef keys) -> void; +template auto DumpHashStatistics(llvm::ArrayRef> keys) + -> void; +template auto DumpHashStatistics(llvm::ArrayRef> keys) + -> void; +template auto DumpHashStatistics(llvm::ArrayRef> keys) + -> void; template auto DumpHashStatistics(llvm::ArrayRef keys) -> void; template auto DumpHashStatistics(llvm::ArrayRef keys) -> void; diff --git a/common/raw_hashtable_benchmark_helpers.h b/common/raw_hashtable_benchmark_helpers.h index a130b2425d61..99108a1aa44e 100644 --- a/common/raw_hashtable_benchmark_helpers.h +++ b/common/raw_hashtable_benchmark_helpers.h @@ -8,6 +8,8 @@ #include #include +#include +#include #include #include #include @@ -55,6 +57,42 @@ auto GetKeysAndHitKeys(ssize_t table_keys_size, ssize_t lookup_keys_size) template auto DumpHashStatistics(llvm::ArrayRef keys) -> void; +// A type that works like an `int` but shifting in the specified number of low +// zero bits. This is only intended for testing hash tables with especially +// difficult to hash integer values, it isn't meant to be used otherwise. +template +struct LowZeroBitInt { + int64_t shifted_value = 0; + + explicit constexpr LowZeroBitInt() = default; + explicit constexpr LowZeroBitInt(int64_t value) + : shifted_value(value << LowZeroBits) {} + + friend auto operator<<(llvm::raw_ostream& out, const LowZeroBitInt& value) + -> llvm::raw_ostream& { + return out << value.shifted_value; + } + + constexpr auto operator==(const LowZeroBitInt& rhs) const -> bool = default; + constexpr auto operator<=>(const LowZeroBitInt& rhs) const + -> std::strong_ordering = default; + + friend auto CarbonHashValue(const LowZeroBitInt& value, uint64_t seed) + -> HashCode { + return HashValue(value.shifted_value, seed); + } + + template + friend auto AbslHashValue(H h, const LowZeroBitInt& value) -> H { + return H::combine(std::move(h), value.shifted_value); + } + + friend auto hash_value(const LowZeroBitInt& value) -> size_t { + boost::hash hasher; + return hasher(value.shifted_value); + } +}; + // Convert values used in hashtable benchmarking to a bool. This is used to form // dependencies between values stored in the hashtable between benchmark // iterations. @@ -151,6 +189,19 @@ struct CarbonHashDI { } }; +template +struct CarbonHashDI> { + using IntT = LowZeroBitInt; + static auto getEmptyKey() -> IntT { return IntT(-1); } + static auto getTombstoneKey() -> IntT { return IntT(-2); } + static auto getHashValue(const IntT val) -> unsigned { + return static_cast(HashValue(val)); + } + static auto isEqual(const IntT lhs, const IntT rhs) -> bool { + return lhs == rhs; + } +}; + template struct CarbonHashDI { static constexpr uintptr_t Log2MaxAlign = 12; @@ -238,4 +289,22 @@ auto ReportTableMetrics(const TableT& table, benchmark::State& state) -> void { } // namespace Carbon::RawHashtable +namespace llvm { + +// Enable LLVM to hash our special stress testing integer type. +template +struct DenseMapInfo> { + using IntT = Carbon::RawHashtable::LowZeroBitInt; + static auto getEmptyKey() -> IntT { return IntT(-1); } + static auto getTombstoneKey() -> IntT { return IntT(-2); } + static auto getHashValue(const IntT val) -> unsigned { + return DenseMapInfo::getHashValue(val.shifted_value); + } + static auto isEqual(const IntT lhs, const IntT rhs) -> bool { + return lhs == rhs; + } +}; + +} // namespace llvm + #endif // CARBON_COMMON_RAW_HASHTABLE_BENCHMARK_HELPERS_H_