From 5cb7be2794263b292e6e0ab8addcdabaa34e8c59 Mon Sep 17 00:00:00 2001 From: Chandler Carruth Date: Wed, 23 Aug 2023 10:12:27 -0700 Subject: [PATCH] Expand lexing benchmarks a little bit. (#3123) This adds a benchmark that tries to synthesize mixtures of symbols, keywords, and identifiers. It tweaks the distribution of identifier lengths based on some empirical measurements of, for example LLVM's codebase. Also establish a framework for skewing the symbol distribution, although that one is based entirely on intuition and not measurements. It should be adjusted as we have measurements. The ratios between symbols, keywords, and identifiers is also unmeasured, but several different ratios are covered. Neither literals nor grouping symbols are included yet, as both present some additional challenges in forming them, and this seemed like a plausible increment in expanding the benchmark coverage. --- .../lexer/tokenized_buffer_benchmark.cpp | 227 +++++++++++++----- 1 file changed, 167 insertions(+), 60 deletions(-) diff --git a/toolchain/lexer/tokenized_buffer_benchmark.cpp b/toolchain/lexer/tokenized_buffer_benchmark.cpp index 4ec8a4f94f0f..fe935cb5933e 100644 --- a/toolchain/lexer/tokenized_buffer_benchmark.cpp +++ b/toolchain/lexer/tokenized_buffer_benchmark.cpp @@ -15,6 +15,117 @@ namespace Carbon::Testing { namespace { +auto IdentifierStartChars() -> llvm::ArrayRef { + static llvm::SmallVector chars = [] { + llvm::SmallVector chars; + chars.push_back('_'); + for (char c : llvm::seq_inclusive('A', 'Z')) { + chars.push_back(c); + } + for (char c : llvm::seq_inclusive('a', 'z')) { + chars.push_back(c); + } + return chars; + }(); + return chars; +} + +auto IdentifierChars() -> llvm::ArrayRef { + static llvm::SmallVector chars = [] { + llvm::ArrayRef start_chars = IdentifierStartChars(); + llvm::SmallVector chars(start_chars.begin(), start_chars.end()); + for (char c : llvm::seq_inclusive('0', '9')) { + chars.push_back(c); + } + return chars; + }(); + return chars; +} + +// Generates a random identifier string using the provided RNG BitGen. +// +// Optionally, can specify a min and max length for the generated identifier. +// +// Optionally, can request a uniform distribution of lengths. When this is false +// (the default) the routine tries to generate a distribution that roughly +// matches what we observe in C++ code. +auto GenerateRandomIdentifier(absl::BitGen& gen, int min_length = 1, + int max_length = 64, bool uniform_lengths = false) + -> std::string { + llvm::ArrayRef start_chars = IdentifierStartChars(); + llvm::ArrayRef chars = IdentifierChars(); + + int length = + uniform_lengths + ? absl::Uniform(gen, min_length, max_length) + // None of the Abseil distributions are *great* fits for observed data + // on identifier length, but log-uniform is vaguely close. A better + // distribution would have two peaks -- one at 1 and the other at 4, + // with a minor dip between and a fairly slow log-uniform falloff into + // the long tail. Lacking more nuanced distribution functions, we work + // with a basic log-uniform. + : absl::LogUniform(gen, min_length, max_length); + + std::string id_result; + llvm::raw_string_ostream os(id_result); + llvm::StringRef id; + do { + // Erase any prior attempts to find an identifier. + id_result.clear(); + os << start_chars[absl::Uniform(gen, 0, start_chars.size())]; + for (int j : llvm::seq(0, length)) { + static_cast(j); + os << chars[absl::Uniform(gen, 0, chars.size())]; + } + // Check if we ended up forming an integer type literal or a keyword, and + // try again. + id = llvm::StringRef(id_result); + } while ( + llvm::any_of(TokenKind::KeywordTokens, + [id](auto token) { return id == token.fixed_spelling(); }) || + ((id.consume_front("i") || id.consume_front("u") || + id.consume_front("f")) && + llvm::all_of(id, [](const char c) { return llvm::isDigit(c); }))); + return id_result; +} + +// Build our own table of symbols so we can use repetitions to skew the +// distribution. +auto GetSymbolTokenTableImpl() -> llvm::SmallVector { + llvm::SmallVector table; +#define CARBON_SYMBOL_TOKEN(TokenName, Spelling) \ + table.push_back(TokenKind::TokenName); +#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) +#define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) +#include "toolchain/lexer/token_kind.def" + table.insert(table.end(), 32, TokenKind::Semi); + table.insert(table.end(), 16, TokenKind::Comma); + table.insert(table.end(), 12, TokenKind::Period); + table.insert(table.end(), 8, TokenKind::Colon); + table.insert(table.end(), 8, TokenKind::Equal); + table.insert(table.end(), 4, TokenKind::Amp); + table.insert(table.end(), 4, TokenKind::ColonExclaim); + table.insert(table.end(), 4, TokenKind::EqualEqual); + table.insert(table.end(), 4, TokenKind::ExclaimEqual); + table.insert(table.end(), 4, TokenKind::MinusGreater); + table.insert(table.end(), 4, TokenKind::Star); + return table; +} + +auto GetSymbolTokenTable() -> llvm::ArrayRef { + static auto symbol_token_table_storage = GetSymbolTokenTableImpl(); + return symbol_token_table_storage; +} + +// Generate random symbols. This skews the distribution as best it can towards +// what we expect in real world source code, but doesn't include grouping +// symbols for simplicity. +auto GenerateRandomSymbol(absl::BitGen& gen) -> llvm::StringRef { + llvm::ArrayRef table = GetSymbolTokenTable(); + auto index = absl::Uniform(gen, 0, table.size()); + return table[index].fixed_spelling(); +} + class LexerBenchHelper { public: explicit LexerBenchHelper(llvm::StringRef text) @@ -73,66 +184,60 @@ void BM_ValidKeywords(benchmark::State& state) { } BENCHMARK(BM_ValidKeywords); -auto IdentifierStartChars() -> llvm::ArrayRef { - static llvm::SmallVector chars = [] { - llvm::SmallVector chars; - chars.push_back('_'); - for (char c : llvm::seq_inclusive('A', 'Z')) { - chars.push_back(c); - } - for (char c : llvm::seq_inclusive('a', 'z')) { - chars.push_back(c); - } - return chars; - }(); - return chars; -} - -auto IdentifierChars() -> llvm::ArrayRef { - static llvm::SmallVector chars = [] { - llvm::ArrayRef start_chars = IdentifierStartChars(); - llvm::SmallVector chars(start_chars.begin(), start_chars.end()); - for (char c : llvm::seq_inclusive('0', '9')) { - chars.push_back(c); - } - return chars; - }(); - return chars; -} - -void BM_ValidIdentifiers(benchmark::State& state) { +void BM_ValidIdentifiers(benchmark::State& state, bool uniform_lengths) { int min_length = state.range(0); int max_length = state.range(1); absl::BitGen gen; std::string source; llvm::raw_string_ostream os(source); llvm::ListSeparator sep(" "); - for (int i : llvm::seq(0, NumTokens)) { - static_cast(i); - int length = absl::Uniform(absl::IntervalClosedClosed, gen, min_length, - max_length); + for (int i = 0; i < NumTokens; ++i) { + os << sep + << GenerateRandomIdentifier(gen, min_length, max_length, + uniform_lengths); + } + + LexerBenchHelper helper(source); + for (auto _ : state) { + TokenizedBuffer buffer = helper.Lex(); + CARBON_CHECK(!buffer.has_errors()) << helper.DiagnoseErrors(); + } + + state.counters["TokenRate"] = benchmark::Counter( + NumTokens, benchmark::Counter::kIsIterationInvariantRate); +} +// Benchmark the non-uniform distribution we observe in C++ code. +BENCHMARK_CAPTURE(BM_ValidIdentifiers, Representative, + /*uniform_lengths=*/false) + ->Args({1, 64}); + +// Also benchmark a few uniform distribution ranges of identifier widths to +// cover different patterns that emerge with small, medium, and longer +// identifiers. +BENCHMARK_CAPTURE(BM_ValidIdentifiers, Uniform, + /*uniform_lengths=*/true) + ->Args({3, 5}) + ->Args({3, 16}) + ->Args({12, 64}); + +void BM_ValidMix(benchmark::State& state) { + int symbol_percent = state.range(0); + int keyword_percent = state.range(1); + absl::BitGen gen; + std::string source; + llvm::raw_string_ostream os(source); + llvm::ListSeparator sep(" "); + for (int i = 0; i < NumTokens; ++i) { os << sep; - int id_start = source.size(); - llvm::StringRef id; - do { - // Erase any prior attempts to find an identifier. - source.resize(id_start); - llvm::ArrayRef start_chars = IdentifierStartChars(); - os << start_chars[absl::Uniform(gen, 0, start_chars.size())]; - llvm::ArrayRef chars = IdentifierChars(); - for (int j : llvm::seq(0, length)) { - static_cast(j); - os << chars[absl::Uniform(gen, 0, chars.size())]; - } - // Check if we ended up forming an integer type literal or a keyword, and - // try again. - id = llvm::StringRef(source).substr(id_start); - } while (llvm::any_of( - TokenKind::KeywordTokens, - [id](auto token) { return id == token.fixed_spelling(); }) || - ((id.consume_front("i") || id.consume_front("u") || - id.consume_front("f")) && - llvm::all_of(id, [](const char c) { return llvm::isDigit(c); }))); + int percent_bucket = absl::Uniform(gen, 0, 100); + if (percent_bucket < symbol_percent) { + os << GenerateRandomSymbol(gen); + } else if (percent_bucket < symbol_percent + keyword_percent) { + int index = absl::Uniform(gen, 0, TokenKind::KeywordTokens.size()); + os << TokenKind::KeywordTokens[index].fixed_spelling(); + } else { + os << GenerateRandomIdentifier(gen); + } } LexerBenchHelper helper(source); @@ -147,13 +252,15 @@ void BM_ValidIdentifiers(benchmark::State& state) { state.counters["TokenRate"] = benchmark::Counter( NumTokens, benchmark::Counter::kIsIterationInvariantRate); } -BENCHMARK(BM_ValidIdentifiers) - // Benchmark a few ranges of identifier widths to cover different patterns - // that emerge with small, medium, and longer identifiers. - ->Args({1, 3}) - ->Args({3, 5}) - ->Args({3, 16}) - ->Args({12, 64}); +// The distributions between symbols, keywords, and identifiers here are +// guesses. Eventually, we should collect more data to help tune these, but +// hopefully the performance isn't too sensitive and we can just cover a wide +// range here. +BENCHMARK(BM_ValidMix) + ->Args({10, 40}) + ->Args({25, 30}) + ->Args({50, 20}) + ->Args({75, 10}); } // namespace } // namespace Carbon::Testing