diff --git a/toolchain/benchmarking/BUILD b/toolchain/benchmarking/BUILD index 36168bce70c1..528ff5018462 100644 --- a/toolchain/benchmarking/BUILD +++ b/toolchain/benchmarking/BUILD @@ -70,6 +70,7 @@ cc_binary( "//testing/base:global_exe_path", "//toolchain/base:install_paths_test_helpers", "//toolchain/driver", + "//toolchain/driver:clang_runner", "//toolchain/testing:compile_helper", "@google_benchmark//:benchmark", "@llvm-project//llvm:Support", diff --git a/toolchain/benchmarking/compile_benchmark.cpp b/toolchain/benchmarking/compile_benchmark.cpp index 16be6667537e..5ca6c2962503 100644 --- a/toolchain/benchmarking/compile_benchmark.cpp +++ b/toolchain/benchmarking/compile_benchmark.cpp @@ -5,56 +5,22 @@ #include #include +#include #include +#include "llvm/Support/VirtualFileSystem.h" #include "testing/base/global_exe_path.h" #include "toolchain/base/install_paths_test_helpers.h" #include "toolchain/benchmarking/source_gen.h" +#include "toolchain/driver/clang_runner.h" #include "toolchain/driver/driver.h" #include "toolchain/testing/compile_helper.h" namespace Carbon::Testing { namespace { -// Helper used to benchmark compilation across different phases. -// -// Handles setting up the compiler's driver, locating the prelude, and managing -// a VFS in which the compilations occur. -class CompileBenchmark { - public: - CompileBenchmark() - : installation_(InstallPaths::MakeForBazelRunfiles(GetExePath())), - driver_(fs_, &installation_, /*input_stream=*/nullptr, &llvm::outs(), - &llvm::errs()) { - AddPreludeFilesToVfs(installation_, fs_); - } - - // Setup a set of source files in the VFS for the driver. Each string input is - // materialized into a virtual file and a list of the virtual filenames is - // returned. - auto SetUpFiles(llvm::ArrayRef sources) - -> llvm::SmallVector { - llvm::SmallVector file_names; - file_names.reserve(sources.size()); - for (auto [i, source] : llvm::enumerate(sources)) { - file_names.push_back(llvm::formatv("file_{0}.carbon", i).str()); - fs_->addFile(file_names.back(), /*ModificationTime=*/0, - llvm::MemoryBuffer::getMemBuffer(source)); - } - return file_names; - } - - auto driver() -> Driver& { return driver_; } - auto gen() -> SourceGen& { return gen_; } - - private: - llvm::IntrusiveRefCntPtr fs_ = - new llvm::vfs::InMemoryFileSystem; - const InstallPaths installation_; - Driver driver_; - - SourceGen gen_; -}; +// A using declaration and abbreviation to keep the benchmark names short. +using Lang = SourceGen::Language; // An enumerator used to select compilation phases to benchmark. enum class Phase : uint8_t { @@ -63,18 +29,90 @@ enum class Phase : uint8_t { Check, }; -// Maps the enumerator for a compilation phase into a specific `compile` command -// line flag. -static auto PhaseFlag(Phase phase) -> llvm::StringRef { - switch (phase) { - case Phase::Lex: - return "--phase=lex"; - case Phase::Parse: - return "--phase=parse"; - case Phase::Check: - return "--phase=check"; +// Helper used to benchmark compilation. +// +// Handles setting up the compiler's driver or ClangRunner, locating the prelude +// or system headers, and managing a VFS in which the compilations occur. +template +class CompileBenchmark { + public: + CompileBenchmark() + : fs_(new llvm::vfs::InMemoryFileSystem), + installation_(InstallPaths::MakeForBazelRunfiles(GetExePath())), + gen_(L) { + if constexpr (L == Lang::Carbon) { + driver_.emplace(fs_, &installation_, /*input_stream=*/nullptr, + &llvm::outs(), &llvm::errs()); + AddPreludeFilesToVfs(installation_, fs_); + } else { + overlay_fs_ = + new llvm::vfs::OverlayFileSystem(llvm::vfs::getRealFileSystem()); + overlay_fs_->pushOverlay(fs_); + runner_.emplace(&installation_, overlay_fs_); + } } -} + + // Setup a set of source files in the VFS for the driver. Each string input is + // materialized into a virtual file and a list of the virtual filenames is + // returned + auto SetUpFiles(llvm::ArrayRef sources) + -> llvm::SmallVector { + llvm::SmallVector file_names; + file_names.reserve(sources.size()); + for (auto [i, source] : llvm::enumerate(sources)) { + file_names.push_back( + llvm::formatv("file_{0}.{1}", i, L == Lang::Carbon ? "carbon" : "cpp") + .str()); + fs_->addFile(file_names.back(), /*ModificationTime=*/0, + llvm::MemoryBuffer::getMemBuffer(source)); + } + return file_names; + } + + auto RunCompile(llvm::StringRef file_name, Phase phase) -> bool { + if constexpr (L == Lang::Carbon) { + llvm::StringRef phase_flag; + switch (phase) { + case Phase::Lex: + phase_flag = "--phase=lex"; + break; + case Phase::Parse: + phase_flag = "--phase=parse"; + break; + case Phase::Check: + phase_flag = "--phase=check"; + break; + } + return driver_->RunCommand({"compile", phase_flag, file_name}).success; + } + + // We only support check and lex phases for C++ as it doesn't have a + // meaningful parse phase in Clang. + switch (phase) { + case Phase::Check: { + auto result = runner_->RunWithNoRuntimes({"-fsyntax-only", file_name}); + return result.ok() && *result; + } + case Phase::Lex: { + auto result = + runner_->RunWithNoRuntimes({"-E", "-o", "/dev/null", file_name}); + return result.ok() && *result; + } + case Phase::Parse: + CARBON_FATAL("No parse phase to benchmark in Clang"); + } + } + + auto gen() -> SourceGen& { return gen_; } + + private: + llvm::IntrusiveRefCntPtr fs_; + llvm::IntrusiveRefCntPtr overlay_fs_; + const InstallPaths installation_; + std::optional driver_; + std::optional runner_; + SourceGen gen_; +}; // Benchmark on multiple files of the same size but with different source code // in order to avoid branch prediction perfectly learning a particular file's @@ -94,37 +132,47 @@ static auto ComputeFileCount(int target_lines) -> int { #endif } -template +template static auto BM_CompileApiFileDenseDecls(benchmark::State& state) -> void { - CompileBenchmark bench; + CompileBenchmark bench; + CompileHelper carbon_compile_helper; + int target_lines = state.range(0); int num_files = ComputeFileCount(target_lines); + if constexpr (L == Lang::Cpp) { + // Reduce the number of files with C++ to balance the longer compile times. + num_files /= 2; + } + llvm::SmallVector sources; sources.reserve(num_files); - // Create a collection of random source files. Compute average statistics for - // counters for compilation speed. - CompileHelper compile_helper; double total_bytes = 0.0; - double total_tokens = 0.0; double total_lines = 0.0; + double total_tokens = 0.0; + for (auto _ : llvm::seq(num_files)) { sources.push_back(bench.gen().GenApiFileDenseDecls( target_lines, SourceGen::DenseDeclParams{})); const auto& source = sources.back(); total_bytes += source.size(); - total_tokens += compile_helper.GetTokenizedBuffer(source).size(); total_lines += llvm::count(source, '\n'); + if constexpr (L == Lang::Carbon) { + total_tokens += carbon_compile_helper.GetTokenizedBuffer(source).size(); + } }; + state.counters["Bytes"] = benchmark::Counter(total_bytes / sources.size(), benchmark::Counter::kIsIterationInvariantRate); - state.counters["Tokens"] = - benchmark::Counter(total_tokens / sources.size(), - benchmark::Counter::kIsIterationInvariantRate); state.counters["Lines"] = benchmark::Counter(total_lines / sources.size(), benchmark::Counter::kIsIterationInvariantRate); + if constexpr (L == Lang::Carbon) { + state.counters["Tokens"] = + benchmark::Counter(total_tokens / sources.size(), + benchmark::Counter::kIsIterationInvariantRate); + } // Set up the sources as files for compilation. llvm::SmallVector file_names = bench.SetUpFiles(sources); @@ -139,10 +187,8 @@ static auto BM_CompileApiFileDenseDecls(benchmark::State& state) -> void { // the generated code that we're benchmarking. benchmark::DoNotOptimize(i); - bool success = bench.driver() - .RunCommand({"compile", PhaseFlag(P), file_names[i]}) - .success; - CARBON_DCHECK(success); + bool success = bench.RunCompile(file_names[i], P); + CARBON_CHECK(success); // We use the compilation success to step through the file names, // establishing a dependency between each lookup. This doesn't fully allow @@ -155,13 +201,20 @@ static auto BM_CompileApiFileDenseDecls(benchmark::State& state) -> void { // Benchmark from 256-line test cases through 256k line test cases, and for each // phase of compilation. -BENCHMARK(BM_CompileApiFileDenseDecls) +BENCHMARK(BM_CompileApiFileDenseDecls) ->RangeMultiplier(4) ->Range(256, static_cast(256 * 1024)); -BENCHMARK(BM_CompileApiFileDenseDecls) +BENCHMARK(BM_CompileApiFileDenseDecls) ->RangeMultiplier(4) ->Range(256, static_cast(256 * 1024)); -BENCHMARK(BM_CompileApiFileDenseDecls) +BENCHMARK(BM_CompileApiFileDenseDecls) + ->RangeMultiplier(4) + ->Range(256, static_cast(256 * 1024)); + +BENCHMARK(BM_CompileApiFileDenseDecls) + ->RangeMultiplier(4) + ->Range(256, static_cast(256 * 1024)); +BENCHMARK(BM_CompileApiFileDenseDecls) ->RangeMultiplier(4) ->Range(256, static_cast(256 * 1024)); diff --git a/toolchain/benchmarking/source_gen.cpp b/toolchain/benchmarking/source_gen.cpp index 0f6212991a01..296d49f3e35f 100644 --- a/toolchain/benchmarking/source_gen.cpp +++ b/toolchain/benchmarking/source_gen.cpp @@ -467,8 +467,9 @@ static auto IdentifierChars() -> llvm::ArrayRef { } static constexpr llvm::StringRef NonCarbonCppKeywords[] = { - "asm", "do", "double", "float", "int", "long", "new", "signed", - "std", "try", "unix", "unsigned", "xor", "NAN", "M_E", "M_PI", + "asm", "catch", "do", "double", "float", "int", "long", "new", + "operator", "signed", "std", "this", "throw", "try", "typename", "unix", + "unsigned", "using", "xor", "M_E", "M_El", "M_PI", "NAN", "NULL", }; // Returns a random identifier string of the specified length.