diff --git a/common/map.h b/common/map.h index 5922cf9a342b..86ae0e74f986 100644 --- a/common/map.h +++ b/common/map.h @@ -66,6 +66,7 @@ class MapView using KeyT = typename ImplT::KeyT; using ValueT = typename ImplT::ValueT; using KeyContextT = typename ImplT::KeyContextT; + using MetricsT = typename ImplT::MetricsT; // This type represents the result of lookup operations. It encodes whether // the lookup was a success as well as accessors for the key and value. @@ -117,15 +118,11 @@ class MapView requires(std::invocable); // This routine is relatively inefficient and only intended for use in - // benchmarking or logging of performance anomalies. The specific count - // returned has no specific guarantees beyond being informative in benchmarks. - // It counts how many of the keys in the hashtable have required probing - // beyond their initial group of slots. - // - // TODO: Replace with a more general metrics routine that covers other - // important aspects such as load factor, and average probe *distance*. - auto CountProbedKeys(KeyContextT key_context = KeyContextT()) -> ssize_t { - return ImplT::CountProbedKeys(key_context); + // benchmarking or logging of performance anomalies. The specific metrics + // returned have no specific guarantees beyond being informative in + // benchmarks. + auto ComputeMetrics(KeyContextT key_context = KeyContextT()) -> MetricsT { + return ImplT::ComputeMetricsImpl(key_context); } private: @@ -165,6 +162,7 @@ class MapBase : protected RawHashtable::BaseImpl; using LookupKVResult = typename ViewT::LookupKVResult; + using MetricsT = typename ImplT::MetricsT; // The result type for insertion operations both indicates whether an insert // was needed (as opposed to finding an existing element), and provides access @@ -233,9 +231,9 @@ class MapBase : protected RawHashtable::BaseImpl ssize_t { - return ViewT(*this).CountProbedKeys(key_context); + auto ComputeMetrics(KeyContextT key_context = KeyContextT()) const + -> MetricsT { + return ViewT(*this).ComputeMetrics(key_context); } // Insert a key and value into the map. If the key is already present, the new diff --git a/common/map_benchmark.cpp b/common/map_benchmark.cpp index 7c0ed9203733..63723a705607 100644 --- a/common/map_benchmark.cpp +++ b/common/map_benchmark.cpp @@ -18,6 +18,7 @@ using RawHashtable::CarbonHashDI; using RawHashtable::GetKeysAndHitKeys; using RawHashtable::GetKeysAndMissKeys; using RawHashtable::HitArgs; +using RawHashtable::ReportTableMetrics; using RawHashtable::SizeArgs; using RawHashtable::ValueToBool; @@ -159,6 +160,15 @@ template using MapWrapper = MapWrapperOverride; +template +auto ReportMetrics(const MapWrapper& m_wrapper, benchmark::State& state) + -> void { + // Report some extra statistics about the Carbon type. + if constexpr (IsCarbonMap) { + ReportTableMetrics(m_wrapper.m, state); + } +} + // NOLINTBEGIN(bugprone-macro-parentheses): Parentheses are incorrect here. #define MAP_BENCHMARK_ONE_OP_SIZE(NAME, APPLY, KT, VT) \ BENCHMARK(NAME>)->Apply(APPLY); \ @@ -223,6 +233,8 @@ static void BM_MapContainsHit(benchmark::State& state) { i += static_cast(result); } } + + ReportMetrics(m, state); } MAP_BENCHMARK_ONE_OP(BM_MapContainsHit, HitArgs); @@ -250,6 +262,8 @@ static void BM_MapContainsMiss(benchmark::State& state) { i += static_cast(!result); } } + + ReportMetrics(m, state); } MAP_BENCHMARK_ONE_OP(BM_MapContainsMiss, SizeArgs); @@ -302,6 +316,8 @@ static void BM_MapLookupHit(benchmark::State& state) { i += static_cast(result); } } + + ReportMetrics(m, state); } MAP_BENCHMARK_ONE_OP(BM_MapLookupHit, HitArgs); @@ -339,6 +355,8 @@ static void BM_MapUpdateHit(benchmark::State& state) { CARBON_DCHECK(!inserted); } } + + ReportMetrics(m, state); } MAP_BENCHMARK_ONE_OP(BM_MapUpdateHit, HitArgs); @@ -454,19 +472,13 @@ static void BM_MapInsertSeq(benchmark::State& state) { if constexpr (IsCarbonMap) { // Re-build a map outside of the timing loop to look at the statistics // rather than the timing. - MapT m; + MapWrapperT m; for (auto k : keys) { - bool inserted = m.Insert(k, MakeValue()).is_inserted(); + bool inserted = m.BenchInsert(k, MakeValue()); CARBON_DCHECK(inserted) << "Must be a successful insert!"; } - // While this count is "iteration invariant" (it should be exactly the same - // for every iteration as the set of keys is the same), we don't use that - // because it will scale this by the number of iterations. We want to - // display the probe count of this benchmark *parameter*, not the probe - // count that resulted from the number of iterations. That means we use the - // normal counter API without flags. - state.counters["Probed"] = m.CountProbedKeys(); + ReportMetrics(m, state); // Uncomment this call to print out statistics about the index-collisions // among these keys for debugging: diff --git a/common/raw_hashtable.h b/common/raw_hashtable.h index eaff986edf08..21a5f03ea64d 100644 --- a/common/raw_hashtable.h +++ b/common/raw_hashtable.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -255,6 +256,38 @@ struct StorageEntry { alignas(KeyT) std::byte key_storage[sizeof(KeyT)]; }; +struct Metrics { + // How many keys are present in the table. + ssize_t key_count = 0; + // How many slots of the table are reserved due to deleted markers required to + // preserve probe sequences. + ssize_t deleted_count = 0; + // How many bytes of allocated storage are used by the table. Note, does not + // include the table object or any small-size buffer. + ssize_t storage_bytes = 0; + + // How many keys have required probing beyond the initial group. These are the + // keys with a probe distance > 0. + ssize_t probed_key_count = 0; + // The probe distance averaged over every key. If every key is in its initial + // group, this will be zero as no keys will have a larger probe distance. In + // general, we want this to be as close to zero as possible. + double probe_avg_distance = 0.0; + // The maximum probe distance found for a single key in the table. + ssize_t probe_max_distance = 0; + // The average number of probing comparisons required to locate a specific key + // in the table. This is how many comparisons are required *before* the key is + // located, or the *failed* comparisons. We always have to do one successful + // comparison at the end. This successful comparison isn't counted because + // that focuses this metric on the overhead the table is introducing, and + // keeps a "perfect" table with an average of `0.0` here similar to the + // perfect average of `0.0` average probe distance. + double probe_avg_compares = 0.0; + // The maximum number of probing comparisons required to locate a specific + // key in the table. + ssize_t probe_max_compares = 0; +}; + // A placeholder empty type used to model pointers to the allocated buffer of // storage. // @@ -301,6 +334,7 @@ class ViewImpl { using ValueT = InputValueT; using KeyContextT = InputKeyContextT; using EntryT = StorageEntry; + using MetricsT = Metrics; friend class BaseImpl; @@ -335,9 +369,10 @@ class ViewImpl { auto ForEachEntry(EntryCallbackT entry_callback, GroupCallbackT group_callback) const -> void; - // Counts the number of keys in the hashtable that required probing beyond the - // initial group. - auto CountProbedKeys(KeyContextT key_context) const -> ssize_t; + // Returns a collection of informative metrics on the the current state of the + // table, useful for performance analysis. These include relatively slow to + // compute metrics requiring deep inspection of the table's state. + auto ComputeMetricsImpl(KeyContextT key_context) const -> MetricsT; private: ViewImpl(ssize_t alloc_size, Storage* storage) @@ -358,6 +393,11 @@ class ViewImpl { return alloc_size; } + // Compute the allocated table's byte size. + static constexpr auto AllocByteSize(ssize_t alloc_size) -> ssize_t { + return EntriesOffset(alloc_size) + sizeof(EntryT) * alloc_size; + } + auto metadata() const -> uint8_t* { return reinterpret_cast(storage_); } @@ -388,6 +428,7 @@ class BaseImpl { using KeyContextT = InputKeyContextT; using ViewImplT = ViewImpl; using EntryT = typename ViewImplT::EntryT; + using MetricsT = typename ViewImplT::MetricsT; BaseImpl(int small_alloc_size, Storage* small_storage) : small_alloc_size_(small_alloc_size) { @@ -447,9 +488,6 @@ class BaseImpl { template <> struct SmallStorage<0> {}; - static constexpr auto AllocByteSize(ssize_t alloc_size) -> ssize_t { - return ViewImplT::EntriesOffset(alloc_size) + sizeof(EntryT) * alloc_size; - } static auto Allocate(ssize_t alloc_size) -> Storage*; static auto Deallocate(Storage* storage, ssize_t alloc_size) -> void; @@ -709,26 +747,72 @@ ViewImpl::ForEachEntry( } template -auto ViewImpl::CountProbedKeys( - KeyContextT key_context) const -> ssize_t { +auto ViewImpl::ComputeMetricsImpl( + KeyContextT key_context) const -> Metrics { uint8_t* local_metadata = metadata(); EntryT* local_entries = entries(); ssize_t local_size = alloc_size_; - ssize_t count = 0; + + Metrics metrics; + + // Compute the ones we can directly. + metrics.deleted_count = llvm::count( + llvm::ArrayRef(local_metadata, local_size), MetadataGroup::Deleted); + metrics.storage_bytes = AllocByteSize(local_size); + + // We want to process present slots specially to collect metrics on their + // probing behavior. for (ssize_t group_index = 0; group_index < local_size; group_index += GroupSize) { auto g = MetadataGroup::Load(local_metadata, group_index); auto present_matched_range = g.MatchPresent(); for (ssize_t byte_index : present_matched_range) { + ++metrics.key_count; ssize_t index = group_index + byte_index; HashCode hash = key_context.HashKey(local_entries[index].key(), ComputeSeed()); - ssize_t hash_index = hash.ExtractIndexAndTag<7>().first & - ComputeProbeMaskFromSize(local_size); - count += static_cast(hash_index != group_index); + auto [hash_index, tag] = hash.ExtractIndexAndTag<7>(); + ProbeSequence s(hash_index, local_size); + metrics.probed_key_count += + static_cast(s.index() != group_index); + + // For each probed key, go through the probe sequence to find both the + // probe distance and how many comparisons are required. + ssize_t distance = 0; + ssize_t compares = 0; + for (; s.index() != group_index; s.Next()) { + auto probe_g = MetadataGroup::Load(local_metadata, s.index()); + auto probe_matched_range = probe_g.Match(tag); + compares += std::distance(probe_matched_range.begin(), + probe_matched_range.end()); + distance += 1; + } + + auto probe_g = MetadataGroup::Load(local_metadata, s.index()); + auto probe_matched_range = probe_g.Match(tag); + CARBON_CHECK(!probe_matched_range.empty()); + for (ssize_t match_index : probe_matched_range) { + if (match_index >= byte_index) { + // Note we only count the compares that will *fail* as part of + // probing. The last successful compare isn't interesting, it is + // always needed. + break; + } + compares += 1; + } + metrics.probe_avg_distance += distance; + metrics.probe_max_distance = + std::max(metrics.probe_max_distance, distance); + metrics.probe_avg_compares += compares; + metrics.probe_max_compares = + std::max(metrics.probe_max_compares, compares); } } - return count; + if (metrics.key_count > 0) { + metrics.probe_avg_compares /= metrics.key_count; + metrics.probe_avg_distance /= metrics.key_count; + } + return metrics; } template @@ -888,8 +972,8 @@ template auto BaseImpl::Allocate( ssize_t alloc_size) -> Storage* { return reinterpret_cast(__builtin_operator_new( - AllocByteSize(alloc_size), static_cast(Alignment), - std::nothrow_t())); + ViewImplT::AllocByteSize(alloc_size), + static_cast(Alignment), std::nothrow_t())); } // Deallocates a table's storage that was allocated with the `Allocate` @@ -897,7 +981,7 @@ auto BaseImpl::Allocate( template auto BaseImpl::Deallocate( Storage* storage, ssize_t alloc_size) -> void { - ssize_t allocated_size = AllocByteSize(alloc_size); + ssize_t allocated_size = ViewImplT::AllocByteSize(alloc_size); // We don't need the size, but make sure it always compiles. static_cast(allocated_size); __builtin_operator_delete(storage, diff --git a/common/raw_hashtable_benchmark_helpers.h b/common/raw_hashtable_benchmark_helpers.h index b5566dfb062e..a130b2425d61 100644 --- a/common/raw_hashtable_benchmark_helpers.h +++ b/common/raw_hashtable_benchmark_helpers.h @@ -203,6 +203,39 @@ struct CarbonHashDI { } }; +template +auto ReportTableMetrics(const TableT& table, benchmark::State& state) -> void { + // While this count is "iteration invariant" (it should be exactly the same + // for every iteration as the set of keys is the same), we don't use that + // because it will scale this by the number of iterations. We want to + // display the metrics for this benchmark *parameter*, not what resulted + // from the number of iterations. That means we use the normal counter API + // without flags. + auto metrics = table.ComputeMetrics(); + state.counters["P-compares"] = metrics.probe_avg_compares; + state.counters["P-distance"] = metrics.probe_avg_distance; + state.counters["P-fraction"] = + static_cast(metrics.probed_key_count) / metrics.key_count; + state.counters["Pmax-distance"] = metrics.probe_max_distance; + state.counters["Pmax-compares"] = metrics.probe_max_compares; + state.counters["Probed"] = metrics.probed_key_count; + + state.counters["Storage"] = metrics.storage_bytes; + + // Also compute how 'efficient' the storage is, 1.0 being zero bytes outside + // of key and value. + ssize_t element_size; + if constexpr (requires { TableT::ValueT; }) { + element_size = + sizeof(typename TableT::KeyT) + sizeof(typename TableT::ValueT); + } else { + element_size = sizeof(typename TableT::KeyT); + } + state.counters["Storage eff"] = + static_cast(metrics.key_count * element_size) / + metrics.storage_bytes; +} + } // namespace Carbon::RawHashtable #endif // CARBON_COMMON_RAW_HASHTABLE_BENCHMARK_HELPERS_H_ diff --git a/common/set.h b/common/set.h index 7a34e1919a95..cbefc5a60c19 100644 --- a/common/set.h +++ b/common/set.h @@ -59,6 +59,7 @@ class SetView : RawHashtable::ViewImpl { public: using KeyT = typename ImplT::KeyT; using KeyContextT = typename ImplT::KeyContextT; + using MetricsT = typename ImplT::MetricsT; // This type represents the result of lookup operations. It encodes whether // the lookup was a success as well as accessors for the key. @@ -97,15 +98,11 @@ class SetView : RawHashtable::ViewImpl { requires(std::invocable); // This routine is relatively inefficient and only intended for use in - // benchmarking or logging of performance anomalies. The specific count - // returned has no specific guarantees beyond being informative in benchmarks. - // It counts how many of the keys in the hashtable have required probing - // beyond their initial group of slots. - // - // TODO: Replace with a more general metrics routine that covers other - // important aspects such as load factor, and average probe *distance*. - auto CountProbedKeys(KeyContextT key_context = KeyContextT()) -> ssize_t { - return ImplT::CountProbedKeys(key_context); + // benchmarking or logging of performance anomalies. The specific metrics + // returned have no specific guarantees beyond being informative in + // benchmarks. + auto ComputeMetrics(KeyContextT key_context = KeyContextT()) -> MetricsT { + return ImplT::ComputeMetricsImpl(key_context); } private: @@ -140,6 +137,7 @@ class SetBase using KeyContextT = typename ImplT::KeyContextT; using ViewT = SetView; using LookupResult = typename ViewT::LookupResult; + using MetricsT = typename ImplT::MetricsT; // The result type for insertion operations both indicates whether an insert // was needed (as opposed to the key already being in the set), and provides @@ -193,9 +191,9 @@ class SetBase } // Convenience forwarder to the view type. - auto CountProbedKeys(KeyContextT key_context = KeyContextT()) const - -> ssize_t { - return ViewT(*this).CountProbedKeys(key_context); + auto ComputeMetrics(KeyContextT key_context = KeyContextT()) const + -> MetricsT { + return ViewT(*this).ComputeMetrics(key_context); } // Insert a key into the set. If the key is already present, no insertion is diff --git a/common/set_benchmark.cpp b/common/set_benchmark.cpp index 048603ca5378..2702cb62e853 100644 --- a/common/set_benchmark.cpp +++ b/common/set_benchmark.cpp @@ -16,6 +16,7 @@ using RawHashtable::CarbonHashDI; using RawHashtable::GetKeysAndHitKeys; using RawHashtable::GetKeysAndMissKeys; using RawHashtable::HitArgs; +using RawHashtable::ReportTableMetrics; using RawHashtable::SizeArgs; using RawHashtable::ValueToBool; @@ -362,13 +363,7 @@ static void BM_SetInsertSeq(benchmark::State& state) { CARBON_DCHECK(inserted) << "Must be a successful insert!"; } - // While this count is "iteration invariant" (it should be exactly the same - // for every iteration as the set of keys is the same), we don't use that - // because it will scale this by the number of iterations. We want to - // display the probe count of this benchmark *parameter*, not the probe - // count that resulted from the number of iterations. That means we use the - // normal counter API without flags. - state.counters["Probed"] = s.CountProbedKeys(); + ReportTableMetrics(s, state); // Uncomment this call to print out statistics about the index-collisions // among these keys for debugging: