diff --git a/toolchain/benchmarking/BUILD b/toolchain/benchmarking/BUILD index 686518d5c6e5..81d79b73f86f 100644 --- a/toolchain/benchmarking/BUILD +++ b/toolchain/benchmarking/BUILD @@ -63,11 +63,15 @@ cc_binary( name = "compile_benchmark", testonly = 1, srcs = ["compile_benchmark.cpp"], + # The subprocess benchmarks execute the installed toolchain binaries. + data = ["//toolchain/install:install_data"], deps = [ ":source_gen_lib", "//common:all_llvm_targets", + "//common:filesystem", "//testing/base:benchmark_main", "//testing/base:global_exe_path", + "//toolchain/base:install_paths", "//toolchain/base:install_paths_test_helpers", "//toolchain/driver", "//toolchain/driver:clang_runner", @@ -83,8 +87,10 @@ sh_test( srcs = [":compile_benchmark"], args = [ "--benchmark_dry_run", - # The `$$` is repeated for Bazel escaping of `$`. - "--benchmark_filter=/256$$", + # Dry-run the smallest (256-line) case of every benchmark. The optional + # `/real_time` suffix is present on the subprocess benchmarks, which use + # real time. The `$$` is repeated for Bazel escaping of `$`. + "--benchmark_filter=/256(/real_time)?$$", ], env = cc_env(), ) diff --git a/toolchain/benchmarking/compile_benchmark.cpp b/toolchain/benchmarking/compile_benchmark.cpp index 5ca6c2962503..878f7487f54e 100644 --- a/toolchain/benchmarking/compile_benchmark.cpp +++ b/toolchain/benchmarking/compile_benchmark.cpp @@ -1,15 +1,28 @@ // Part of the Carbon Language project, under the Apache License v2.0 with LLVM // Exceptions. See /LICENSE for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// Benchmarks for compiling Carbon and C++ source at varying sizes, across each +// phase of compilation. Each benchmark runs in one of two modes: `InProcess`, +// which drives the compiler libraries directly over an in-memory filesystem, or +// `Subprocess`, which executes the installed compiler binary on real files to +// capture the end-to-end command-line cost including process startup. #include #include #include #include +#include +#include "common/filesystem.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/Program.h" #include "llvm/Support/VirtualFileSystem.h" #include "testing/base/global_exe_path.h" +#include "toolchain/base/install_paths.h" #include "toolchain/base/install_paths_test_helpers.h" #include "toolchain/benchmarking/source_gen.h" #include "toolchain/driver/clang_runner.h" @@ -29,14 +42,48 @@ enum class Phase : uint8_t { Check, }; -// Helper used to benchmark compilation. -// -// Handles setting up the compiler's driver or ClangRunner, locating the prelude -// or system headers, and managing a VFS in which the compilations occur. +// Selects how compilation is driven: in-process via the compiler libraries, or +// by executing the installed compiler binary as a subprocess. +enum class Mode : uint8_t { + InProcess, + Subprocess, +}; + +// Returns the `carbon compile --phase=` flag for a given compilation phase. +static auto PhaseFlag(Phase phase) -> llvm::StringRef { + switch (phase) { + case Phase::Lex: + return "--phase=lex"; + case Phase::Parse: + return "--phase=parse"; + case Phase::Check: + return "--phase=check"; + } +} + +// Returns the file name for the `index`-th generated source file, using the +// language-appropriate extension. template -class CompileBenchmark { +static auto SourceFileName(size_t index) -> std::string { + return llvm::formatv("file_{0}.{1}", index, + L == Lang::Carbon ? "carbon" : "cpp") + .str(); +} + +// In-process compilation harness. +// +// Drives the compiler's driver or `ClangRunner` directly, locating the prelude +// or system headers and managing an in-memory VFS in which the compilations +// occur. This measures the compiler as a library, without process startup cost. +template +class InProcessCompiler { public: - CompileBenchmark() + // File-count bounds tuned for in-process (library) compilation; see + // `ComputeFileCount`. + static constexpr int MinFiles = 8; + static constexpr int MaxFiles = 128; + + InProcessCompiler() : fs_(new llvm::vfs::InMemoryFileSystem), installation_(InstallPaths::MakeForBazelRunfiles(GetExePath())), gen_(L) { @@ -52,17 +99,15 @@ class CompileBenchmark { } } - // Setup a set of source files in the VFS for the driver. Each string input is - // materialized into a virtual file and a list of the virtual filenames is - // returned + // Sets up a set of source files in the VFS for the driver. Each string input + // is materialized into a virtual file and a list of the virtual filenames is + // returned. auto SetUpFiles(llvm::ArrayRef sources) -> llvm::SmallVector { llvm::SmallVector file_names; file_names.reserve(sources.size()); for (auto [i, source] : llvm::enumerate(sources)) { - file_names.push_back( - llvm::formatv("file_{0}.{1}", i, L == Lang::Carbon ? "carbon" : "cpp") - .str()); + file_names.push_back(SourceFileName(i)); fs_->addFile(file_names.back(), /*ModificationTime=*/0, llvm::MemoryBuffer::getMemBuffer(source)); } @@ -71,19 +116,8 @@ class CompileBenchmark { auto RunCompile(llvm::StringRef file_name, Phase phase) -> bool { if constexpr (L == Lang::Carbon) { - llvm::StringRef phase_flag; - switch (phase) { - case Phase::Lex: - phase_flag = "--phase=lex"; - break; - case Phase::Parse: - phase_flag = "--phase=parse"; - break; - case Phase::Check: - phase_flag = "--phase=check"; - break; - } - return driver_->RunCommand({"compile", phase_flag, file_name}).success; + return driver_->RunCommand({"compile", PhaseFlag(phase), file_name}) + .success; } // We only support check and lex phases for C++ as it doesn't have a @@ -114,6 +148,109 @@ class CompileBenchmark { SourceGen gen_; }; +// Subprocess compilation harness. +// +// Materializes generated sources as real files in a temporary directory and +// compiles them by executing the installed `carbon` or `clang` binary as a +// subprocess. This measures the full end-to-end cost users see on the command +// line, including process startup, rather than only the compiler-as-a-library +// cost measured by `InProcessCompiler`. +template +class SubprocessCompiler { + public: + // Subprocess launches are expensive, so use fewer files than the in-process + // harness to keep benchmark time reasonable; see `ComputeFileCount`. + static constexpr int MinFiles = 4; + static constexpr int MaxFiles = 64; + + SubprocessCompiler() + : installation_(InstallPaths::MakeForBazelRunfiles(GetExePath())), + gen_(L) { + carbon_path_ = (installation_.root() / "carbon-busybox").string(); + clang_path_ = installation_.clang_path().string(); + + // Leave stdin connected to the parent, but discard the compiler's stdout + // and stderr so benchmark output isn't cluttered. + redirects_[0] = std::nullopt; + redirects_[1] = llvm::StringRef("/dev/null"); + redirects_[2] = llvm::StringRef("/dev/null"); + } + + // Sets up a set of source files on the real filesystem. Each string input is + // written into a file in a temporary directory and a list of the full file + // paths is returned. + auto SetUpFiles(llvm::ArrayRef sources) + -> llvm::SmallVector { + auto tmp_dir_result = Filesystem::MakeTmpDir(); + CARBON_CHECK(tmp_dir_result.ok(), "Failed to create temp dir: {0}", + tmp_dir_result.error()); + tmp_dir_ = std::move(*tmp_dir_result); + + llvm::SmallVector file_names; + file_names.reserve(sources.size()); + for (auto [i, source] : llvm::enumerate(sources)) { + std::string file_name = SourceFileName(i); + auto write_result = tmp_dir_->WriteFileFromString(file_name, source); + CARBON_CHECK(write_result.ok(), "Failed to write file: {0}", + write_result.error()); + file_names.push_back((tmp_dir_->path() / file_name).string()); + } + return file_names; + } + + auto RunCompile(llvm::StringRef file_name, Phase phase) -> bool { + llvm::StringRef program; + llvm::SmallVector args; + + if constexpr (L == Lang::Carbon) { + program = carbon_path_; + args = {"carbon", "compile", PhaseFlag(phase), file_name}; + } else { + program = clang_path_; + args = {"clang++", "--driver-mode=g++"}; + + // We only support check and lex phases for C++ as it doesn't have a + // meaningful parse phase in Clang. + switch (phase) { + case Phase::Check: + args.push_back("-fsyntax-only"); + break; + case Phase::Lex: + args.push_back("-E"); + args.push_back("-o"); + args.push_back("/dev/null"); + break; + case Phase::Parse: + CARBON_FATAL("No parse phase to benchmark in Clang"); + } + args.push_back(file_name); + } + + std::string err_msg; + bool execution_failed = false; + int exit_code = llvm::sys::ExecuteAndWait( + program, args, /*Env=*/std::nullopt, redirects_, + /*SecondsToWait=*/0, /*MemoryLimit=*/0, &err_msg, &execution_failed); + return exit_code == 0 && !execution_failed; + } + + auto gen() -> SourceGen& { return gen_; } + + private: + const InstallPaths installation_; + SourceGen gen_; + std::string carbon_path_; + std::string clang_path_; + std::optional redirects_[3]; + std::optional tmp_dir_; +}; + +// Selects the compilation harness for a given language and mode. +template +using CompilerFor = + std::conditional_t, + SubprocessCompiler>; + // Benchmark on multiple files of the same size but with different source code // in order to avoid branch prediction perfectly learning a particular file's // structure and shape, and to get closer to a cache-cold benchmark number which @@ -122,23 +259,29 @@ class CompileBenchmark { // on a single source file that may have unrepresentative content. // // For simplicity, we compute a number of files from the target line count as a -// heuristic. -static auto ComputeFileCount(int target_lines) -> int { +// heuristic, clamped to the `[min_files, max_files]` range provided by the +// compilation harness. +static auto ComputeFileCount(int target_lines, int min_files, + [[maybe_unused]] int max_files) -> int { + int file_count = (1024 * 1024) / target_lines; #ifndef NDEBUG - // Use a smaller number of files in debug builds where compiles are slower. - return std::max(1, std::min(8, (1024 * 1024) / target_lines)); + // Use a smaller number of files in debug builds where compiles are slower, + // capping at the release-mode minimum. + return std::max(1, std::min(min_files, file_count)); #else - return std::max(8, std::min(128, (1024 * 1024) / target_lines)); + return std::max(min_files, std::min(max_files, file_count)); #endif } -template +template static auto BM_CompileApiFileDenseDecls(benchmark::State& state) -> void { - CompileBenchmark bench; + using Compiler = CompilerFor; + Compiler bench; CompileHelper carbon_compile_helper; int target_lines = state.range(0); - int num_files = ComputeFileCount(target_lines); + int num_files = + ComputeFileCount(target_lines, Compiler::MinFiles, Compiler::MaxFiles); if constexpr (L == Lang::Cpp) { // Reduce the number of files with C++ to balance the longer compile times. num_files /= 2; @@ -160,7 +303,7 @@ static auto BM_CompileApiFileDenseDecls(benchmark::State& state) -> void { if constexpr (L == Lang::Carbon) { total_tokens += carbon_compile_helper.GetTokenizedBuffer(source).size(); } - }; + } state.counters["Bytes"] = benchmark::Counter(total_bytes / sources.size(), @@ -188,7 +331,7 @@ static auto BM_CompileApiFileDenseDecls(benchmark::State& state) -> void { benchmark::DoNotOptimize(i); bool success = bench.RunCompile(file_names[i], P); - CARBON_CHECK(success); + CARBON_CHECK(success, "Compilation failed for file: {0}", file_names[i]); // We use the compilation success to step through the file names, // establishing a dependency between each lookup. This doesn't fully allow @@ -199,24 +342,52 @@ static auto BM_CompileApiFileDenseDecls(benchmark::State& state) -> void { } } -// Benchmark from 256-line test cases through 256k line test cases, and for each -// phase of compilation. -BENCHMARK(BM_CompileApiFileDenseDecls) - ->RangeMultiplier(4) - ->Range(256, static_cast(256 * 1024)); -BENCHMARK(BM_CompileApiFileDenseDecls) - ->RangeMultiplier(4) - ->Range(256, static_cast(256 * 1024)); -BENCHMARK(BM_CompileApiFileDenseDecls) - ->RangeMultiplier(4) - ->Range(256, static_cast(256 * 1024)); +// A thin wrapper for the subprocess benchmarks: they reuse the shared +// implementation above, but register under their own (terser) name rather than +// spelling out `Mode::Subprocess` at each registration. +template +static auto BM_CompileExecApiFileDenseDecls(benchmark::State& state) -> void { + BM_CompileApiFileDenseDecls(state); +} +// Applies the shared range configuration used by every compile benchmark: +// 256-line test cases through 256k-line test cases. +static auto ConfigureCompileBenchmark(benchmark::Benchmark* b) -> void { + b->RangeMultiplier(4)->Range(256, static_cast(256 * 1024)); +} + +// In-process benchmarks measure the compiler as a library across each phase of +// compilation. The mode defaults to `Mode::InProcess`. +BENCHMARK(BM_CompileApiFileDenseDecls) + ->Apply(ConfigureCompileBenchmark); +BENCHMARK(BM_CompileApiFileDenseDecls) + ->Apply(ConfigureCompileBenchmark); +BENCHMARK(BM_CompileApiFileDenseDecls) + ->Apply(ConfigureCompileBenchmark); BENCHMARK(BM_CompileApiFileDenseDecls) - ->RangeMultiplier(4) - ->Range(256, static_cast(256 * 1024)); + ->Apply(ConfigureCompileBenchmark); BENCHMARK(BM_CompileApiFileDenseDecls) - ->RangeMultiplier(4) - ->Range(256, static_cast(256 * 1024)); + ->Apply(ConfigureCompileBenchmark); + +// Subprocess benchmarks measure the same work end-to-end by executing the +// installed compiler binary, including process startup cost. They use real +// (wall-clock) time because the compilation happens in a child process whose +// CPU time isn't attributed to the benchmark process. +BENCHMARK(BM_CompileExecApiFileDenseDecls) + ->Apply(ConfigureCompileBenchmark) + ->UseRealTime(); +BENCHMARK(BM_CompileExecApiFileDenseDecls) + ->Apply(ConfigureCompileBenchmark) + ->UseRealTime(); +BENCHMARK(BM_CompileExecApiFileDenseDecls) + ->Apply(ConfigureCompileBenchmark) + ->UseRealTime(); +BENCHMARK(BM_CompileExecApiFileDenseDecls) + ->Apply(ConfigureCompileBenchmark) + ->UseRealTime(); +BENCHMARK(BM_CompileExecApiFileDenseDecls) + ->Apply(ConfigureCompileBenchmark) + ->UseRealTime(); } // namespace } // namespace Carbon::Testing