From 1a26b5773243f7533776b525a78be0ca16acff40 Mon Sep 17 00:00:00 2001 From: Lucile Rose Nihlen Date: Tue, 9 Jun 2026 16:24:01 -0400 Subject: [PATCH] add first implementation of `carbon build` subcommand (#7239) This is the first draft of the implementation of [p6333](https://docs.carbon-lang.dev/proposals/p6333.html). The `build` subcommand shared logic with the `compile` and `link` subcommands, so I've moved some of the functionality in `compile` and `link` to shared `CompileDriver` and `LinkDriver` classes, respectively. This also required exposing the `CompileOptions` and `LinkOptions` subcommand structs for re-use. There's still some work to do on the proposal, most notably the package include automatic path resolution and import, and the refactors to `carbon compile`. --- toolchain/diagnostics/coverage_test.cpp | 5 + toolchain/diagnostics/kind.def | 5 + toolchain/driver/BUILD | 12 + toolchain/driver/build_subcommand.cpp | 127 +++ toolchain/driver/build_subcommand.h | 39 + toolchain/driver/compile_driver.cpp | 659 +++++++++++ toolchain/driver/compile_driver.h | 263 +++++ toolchain/driver/compile_options.cpp | 525 +++++++++ toolchain/driver/compile_options.h | 124 +++ toolchain/driver/compile_subcommand.cpp | 1331 +---------------------- toolchain/driver/compile_subcommand.h | 72 +- toolchain/driver/driver.cpp | 3 + toolchain/driver/driver_test.cpp | 32 +- toolchain/driver/link_driver.cpp | 85 ++ toolchain/driver/link_driver.h | 31 + toolchain/driver/link_options.cpp | 82 ++ toolchain/driver/link_options.h | 35 + toolchain/driver/link_subcommand.cpp | 120 +- toolchain/driver/link_subcommand.h | 16 +- 19 files changed, 2045 insertions(+), 1521 deletions(-) create mode 100644 toolchain/driver/build_subcommand.cpp create mode 100644 toolchain/driver/build_subcommand.h create mode 100644 toolchain/driver/compile_driver.cpp create mode 100644 toolchain/driver/compile_driver.h create mode 100644 toolchain/driver/compile_options.cpp create mode 100644 toolchain/driver/compile_options.h create mode 100644 toolchain/driver/link_driver.cpp create mode 100644 toolchain/driver/link_driver.h create mode 100644 toolchain/driver/link_options.cpp create mode 100644 toolchain/driver/link_options.h diff --git a/toolchain/diagnostics/coverage_test.cpp b/toolchain/diagnostics/coverage_test.cpp index ba1bb896686f..e59ddeffd71f 100644 --- a/toolchain/diagnostics/coverage_test.cpp +++ b/toolchain/diagnostics/coverage_test.cpp @@ -31,6 +31,11 @@ constexpr Kind UntestedKinds[] = { // Diagnosing erroneous install conditions, but test environments are // typically correct. + Kind::BuildFailureRunningClangToLink, + Kind::BuildOutputFileOpenError, + Kind::BuildPreludeManifestError, + Kind::BuildTempDirectoryCreationError, + Kind::BuildTempDirectoryDeletionError, Kind::CompilePreludeManifestError, Kind::ConfigFailedToReadDigest, Kind::ConfigFailedToSetupTarget, diff --git a/toolchain/diagnostics/kind.def b/toolchain/diagnostics/kind.def index 247837e9a7d1..b84727dd444a 100644 --- a/toolchain/diagnostics/kind.def +++ b/toolchain/diagnostics/kind.def @@ -25,6 +25,11 @@ CARBON_DIAGNOSTIC_KIND(DriverInstallInvalid) CARBON_DIAGNOSTIC_KIND(DriverRuntimesCacheInvalid) CARBON_DIAGNOSTIC_KIND(DriverPrebuiltRuntimesInvalid) CARBON_DIAGNOSTIC_KIND(DriverCommandLineParseFailed) +CARBON_DIAGNOSTIC_KIND(BuildFailureRunningClangToLink) +CARBON_DIAGNOSTIC_KIND(BuildOutputFileOpenError) +CARBON_DIAGNOSTIC_KIND(BuildPreludeManifestError) +CARBON_DIAGNOSTIC_KIND(BuildTempDirectoryCreationError) +CARBON_DIAGNOSTIC_KIND(BuildTempDirectoryDeletionError) CARBON_DIAGNOSTIC_KIND(CompilePhaseFlagConflict) CARBON_DIAGNOSTIC_KIND(CompilePreludeManifestError) CARBON_DIAGNOSTIC_KIND(CompileInputNotRegularFile) diff --git a/toolchain/driver/BUILD b/toolchain/driver/BUILD index 3135252f5ee6..393f94dfc114 100644 --- a/toolchain/driver/BUILD +++ b/toolchain/driver/BUILD @@ -131,8 +131,14 @@ cc_library( srcs = [ "build_runtimes_subcommand.cpp", "build_runtimes_subcommand.h", + "build_subcommand.cpp", + "build_subcommand.h", "clang_subcommand.cpp", "clang_subcommand.h", + "compile_driver.cpp", + "compile_driver.h", + "compile_options.cpp", + "compile_options.h", "compile_subcommand.cpp", "compile_subcommand.h", "config_subcommand.cpp", @@ -144,6 +150,10 @@ cc_library( "format_subcommand.h", "language_server_subcommand.cpp", "language_server_subcommand.h", + "link_driver.cpp", + "link_driver.h", + "link_options.cpp", + "link_options.h", "link_subcommand.cpp", "link_subcommand.h", "lld_subcommand.cpp", @@ -169,6 +179,7 @@ cc_library( "//common:command_line", "//common:error", "//common:filesystem", + "//common:hashing", "//common:ostream", "//common:pretty_stack_trace_function", "//common:raw_string_ostream", @@ -201,6 +212,7 @@ cc_library( "@llvm-project//llvm:MC", "@llvm-project//llvm:Passes", "@llvm-project//llvm:Support", + "@llvm-project//llvm:Target", "@llvm-project//llvm:TargetParser", ], ) diff --git a/toolchain/driver/build_subcommand.cpp b/toolchain/driver/build_subcommand.cpp new file mode 100644 index 000000000000..e38a3dee1d7e --- /dev/null +++ b/toolchain/driver/build_subcommand.cpp @@ -0,0 +1,127 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "toolchain/driver/build_subcommand.h" + +#include + +#include "common/filesystem.h" +#include "llvm/ADT/ScopeExit.h" +#include "llvm/ADT/SmallString.h" +#include "toolchain/driver/compile_driver.h" +#include "toolchain/driver/link_driver.h" + +namespace Carbon { + +auto BuildSubcommandOptions::Build(CommandLine::CommandBuilder& b) -> void { + compile_options.BuildForBuildSubcommand(b); + link_options.BuildForBuildSubcommand(b); + + b.AddFlag( + { + .name = "use-temp-dir", + .help = R"""( +Use a temporary directory for intermediate compilation artifacts. + +When enabled (the default), carbon will compile all input files and necessary +dependencies into a temporary directory, before linking them into the final +output binary. If false, carbon will store the compilation artifacts as hashes +of the compiled input name in the current working directory. +)""", + }, + [&](auto& arg_b) { + arg_b.Default(true); + arg_b.Set(&use_temp_dir); + }); +} + +static constexpr CommandLine::CommandInfo SubcommandInfo = { + .name = "build", + .help = R"""( +Compile and then link Carbon and C++ source code into a single executable. +)""", +}; + +BuildSubcommand::BuildSubcommand() : DriverSubcommand(SubcommandInfo) {} + +auto BuildSubcommand::BuildOptions(CommandLine::CommandBuilder& b) -> void { + options_.Build(b); +} + +auto BuildSubcommand::Run(DriverEnv& driver_env) -> DriverResult { + if (driver_env.fuzzing && !options_.compile_options.clang_args.empty()) { + // Parsing specific Clang arguments can reach deep into + // external libraries that aren't fuzz clean. + TestAndDiagnoseIfFuzzingExternalLibraries(driver_env, "build"); + return {.success = false}; + } + + std::optional temp_dir = std::nullopt; + auto temp_dir_path = std::filesystem::path(""); + if (options_.use_temp_dir) { + if (auto d = Filesystem::MakeTmpDir(); !d.ok()) { + CARBON_DIAGNOSTIC(BuildTempDirectoryCreationError, Error, "{0}", + std::string); + driver_env.emitter.Emit(BuildTempDirectoryCreationError, + PrintToString(d.error())); + return {.success = false}; + } else { + temp_dir = std::move(*d); + temp_dir_path = temp_dir->path(); + } + } + + auto on_exit = llvm::scope_exit([&]() { + // Clean up the temporary directory created for compile results. + if (temp_dir) { + auto remove_result = std::move(*temp_dir).Remove(); + if (!remove_result.ok()) { + CARBON_DIAGNOSTIC(BuildTempDirectoryDeletionError, Error, "{0}", + std::string); + driver_env.emitter.Emit(BuildTempDirectoryDeletionError, + PrintToString(remove_result.error())); + } + } + }); + + auto compile_driver = CompileDriver(&options_.compile_options); + if (!compile_driver.Initialize( + driver_env, [&](llvm::StringRef input_filename) -> std::string { + return (temp_dir_path / + llvm::formatv("{0:x16}.o", HashValue(input_filename)).str()) + .string(); + })) { + return {.success = false}; + } + + auto compile_result = compile_driver.Compile(driver_env); + if (!compile_result.success) { + return compile_result; + } + + // Compute the needed LinkOptions for the LinkDriver from the output of the + // compilation process. + options_.link_options.codegen_options = + options_.compile_options.codegen_options; + + llvm::SmallString<256> output_filename; + if (options_.link_options.output_filename.empty()) { + output_filename = llvm::sys::path::filename( + compile_driver.units()[compile_driver.first_input_index()] + ->input_filename()); + llvm::sys::path::replace_extension(output_filename, ""); + options_.link_options.output_filename = output_filename; + } + + auto input_builder = [&](const std::unique_ptr& unit) { + return unit->output_filename(); + }; + append_range(options_.link_options.object_filenames, + llvm::map_range(compile_driver.units(), input_builder)); + + auto link_driver = LinkDriver(&options_.link_options); + return link_driver.Link(driver_env); +} + +} // namespace Carbon diff --git a/toolchain/driver/build_subcommand.h b/toolchain/driver/build_subcommand.h new file mode 100644 index 000000000000..8a764e939427 --- /dev/null +++ b/toolchain/driver/build_subcommand.h @@ -0,0 +1,39 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef CARBON_TOOLCHAIN_DRIVER_BUILD_SUBCOMMAND_H_ +#define CARBON_TOOLCHAIN_DRIVER_BUILD_SUBCOMMAND_H_ + +#include "common/command_line.h" +#include "toolchain/driver/compile_options.h" +#include "toolchain/driver/driver_env.h" +#include "toolchain/driver/driver_subcommand.h" +#include "toolchain/driver/link_options.h" + +namespace Carbon { + +// Options for the build subcommand. +struct BuildSubcommandOptions { + auto Build(CommandLine::CommandBuilder& b) -> void; + + CompileOptions compile_options; + LinkOptions link_options; + bool use_temp_dir; +}; + +class BuildSubcommand : public DriverSubcommand { + public: + explicit BuildSubcommand(); + + auto BuildOptions(CommandLine::CommandBuilder& b) -> void override; + + auto Run(DriverEnv& driver_env) -> DriverResult override; + + private: + BuildSubcommandOptions options_; +}; + +} // namespace Carbon + +#endif // CARBON_TOOLCHAIN_DRIVER_BUILD_SUBCOMMAND_H_ diff --git a/toolchain/driver/compile_driver.cpp b/toolchain/driver/compile_driver.cpp new file mode 100644 index 000000000000..c6d27c44de10 --- /dev/null +++ b/toolchain/driver/compile_driver.cpp @@ -0,0 +1,659 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "toolchain/driver/compile_driver.h" + +#include +#include +#include +#include +#include +#include + +#include "common/error.h" +#include "common/ostream.h" +#include "common/pretty_stack_trace_function.h" +#include "common/vlog.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/ScopeExit.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/MC/TargetRegistry.h" +#include "llvm/Passes/OptimizationLevel.h" +#include "llvm/Passes/PassBuilder.h" +#include "llvm/Passes/StandardInstrumentations.h" +#include "toolchain/base/clang_invocation.h" +#include "toolchain/base/timings.h" +#include "toolchain/check/check.h" +#include "toolchain/codegen/codegen.h" +#include "toolchain/diagnostics/emitter.h" +#include "toolchain/diagnostics/format_providers.h" +#include "toolchain/lex/lex.h" +#include "toolchain/lower/lower.h" +#include "toolchain/lower/options.h" +#include "toolchain/parse/parse.h" +#include "toolchain/parse/tree_and_subtrees.h" +#include "toolchain/sem_ir/ids.h" +#include "toolchain/source/source_buffer.h" + +namespace Carbon { + +CompilationUnit::CompilationUnit(SemIR::CheckIRId check_ir_id, + int total_ir_count, DriverEnv* driver_env, + const CompileOptions* options, + Diagnostics::Consumer* consumer, + llvm::StringRef input_filename, + std::string output_filename, + const llvm::Target* target) + : check_ir_id_(check_ir_id), + total_ir_count_(total_ir_count), + driver_env_(driver_env), + options_(options), + target_(target), + input_filename_(input_filename), + output_filename_(std::move(output_filename)), + vlog_stream_(driver_env_->vlog_stream) { + if (vlog_stream_ != nullptr || options_->stream_errors) { + consumer_ = consumer; + } else { + sorting_consumer_ = Diagnostics::SortingConsumer(*consumer); + consumer_ = &*sorting_consumer_; + } +} + +auto CompilationUnit::IncludeInDumps() -> bool { + return cache_->include_in_dumps().Get(check_ir_id_); +} + +auto CompilationUnit::SetMultiUnitCache(MultiUnitCache* cache) -> void { + CARBON_CHECK(!cache_, "Called SetMultiUnitCache twice"); + cache_ = cache; + + if (options_->dump_mem_usage && IncludeInDumps()) { + CARBON_CHECK(!mem_usage_); + mem_usage_ = MemUsage(); + } + if (options_->dump_timings && IncludeInDumps()) { + CARBON_CHECK(!timings_); + timings_ = Timings(); + } +} + +auto CompilationUnit::RunLex() -> void { + CARBON_CHECK(cache_, "Must call SetMultiUnitCache first"); + CARBON_CHECK(!tokens_, "Called RunLex twice"); + + LogCall("SourceBuffer::MakeFromFileOrStdin", "source", [&] { + source_ = SourceBuffer::MakeFromFileOrStdin(*driver_env_->fs, + input_filename_, *consumer_); + }); + + if (!source_) { + success_ = false; + return; + } + + if (mem_usage_) { + mem_usage_->Add("source_", source_->text().size(), source_->text().size()); + } + + CARBON_VLOG("*** SourceBuffer ***\n```\n{0}\n```\n", source_->text()); + + LogCall("Lex::Lex", "lex", [&] { + Lex::LexOptions options; + options.consumer = consumer_; + options.vlog_stream = vlog_stream_; + if (options_->dump_tokens && IncludeInDumps()) { + options.dump_stream = driver_env_->output_stream; + options.omit_file_boundary_tokens = options_->omit_file_boundary_tokens; + } + tokens_ = Lex::Lex(value_stores_, *source_, options); + }); + if (mem_usage_) { + mem_usage_->Collect("tokens_", *tokens_); + } + if (tokens_->has_errors()) { + success_ = false; + } +} + +auto CompilationUnit::RunParse() -> void { + LogCall("Parse::Parse", "parse", [&] { + Parse::ParseOptions options; + options.consumer = consumer_; + options.vlog_stream = vlog_stream_; + if (options_->dump_parse_tree && IncludeInDumps()) { + options.dump_stream = driver_env_->output_stream; + options.dump_preorder_parse_tree = options_->preorder_parse_tree; + } + parse_tree_ = Parse::Parse(*tokens_, options); + }); + if (mem_usage_) { + mem_usage_->Collect("parse_tree_", *parse_tree_); + } + if (parse_tree_->has_errors()) { + success_ = false; + } +} + +auto CompilationUnit::GetCheckUnit() -> Check::Unit { + CARBON_CHECK(parse_tree_, "Must call RunParse first"); + CARBON_CHECK(!sem_ir_, "Called GetCheckUnit twice"); + + tree_and_subtrees_getter_ = [this]() -> const Parse::TreeAndSubtrees& { + return this->GetParseTreeAndSubtrees(); + }; + sem_ir_.emplace(&*parse_tree_, check_ir_id_, parse_tree_->packaging_decl(), + value_stores_, input_filename_); + if (!llvm_context_) { + llvm_context_ = std::make_unique(); + } + return {.consumer = consumer_, + .value_stores = &value_stores_, + .timings = timings_ ? &*timings_ : nullptr, + .sem_ir = &*sem_ir_, + .llvm_context = llvm_context_.get(), + .total_ir_count = total_ir_count_}; +} + +auto CompilationUnit::PostCheck() -> void { + CARBON_CHECK(sem_ir_, "Must call GetCheckUnit first"); + + // We've finished all steps that can produce diagnostics. Emit the + // diagnostics now, so that the developer sees them sooner and doesn't need + // to wait for code generation. + consumer_->Flush(); + + if (mem_usage_) { + mem_usage_->Collect("sem_ir_", *sem_ir_); + } + + if (sem_ir_->has_errors()) { + success_ = false; + } +} + +auto CompilationUnit::RunLower() -> void { + LogCall("Lower::LowerToLLVM", "lower", [&] { + if (!llvm_context_) { + llvm_context_ = std::make_unique(); + } + Lower::LowerToLLVMOptions options; + options.llvm_verifier_stream = + options_->run_llvm_verifier ? driver_env_->error_stream : nullptr; + options.want_debug_info = options_->include_debug_info; + options.vlog_stream = vlog_stream_; + options.opt_level = options_->opt_level; + options.mangle_string_fingerprint = options_->mangle_string_fingerprint; + module_ = Lower::LowerToLLVM(*llvm_context_, driver_env_->fs, + cache_->tree_and_subtrees_getters(), *sem_ir_, + total_ir_count_, options); + }); +} + +auto CompilationUnit::MakeTargetMachine( + const clang::CompilerInvocation& clang_invocation) -> void { + CARBON_CHECK(module_, "Must call RunLower first"); + CARBON_CHECK(!target_machine_, "Should not call this multiple times"); + + // Set the target on the module. + // TODO: We should do this earlier. Lower should be passed the target triple + // so it can create the module with this already set. + llvm::Triple target_triple(options_->codegen_options->target); + module_->setTargetTriple(target_triple); + + // TODO: Provide flags to control these. + constexpr llvm::StringLiteral CPU = "generic"; + constexpr llvm::StringLiteral Features = ""; + + const auto& codegen_opts = clang_invocation.getCodeGenOpts(); + + // TODO: Make the code in Clang's BackendUtil.cpp externally accessible and + // call it from here. This is doing a subset of the same work to translate + // Clang code generation options into target options. + llvm::TargetOptions target_opts; + target_opts.UseInitArray = codegen_opts.UseInitArray; + target_opts.FunctionSections = codegen_opts.FunctionSections; + target_opts.DataSections = codegen_opts.DataSections; + target_opts.UniqueSectionNames = codegen_opts.UniqueSectionNames; + target_machine_.reset(target_->createTargetMachine( + target_triple, CPU, Features, target_opts, llvm::Reloc::PIC_)); +} + +auto CompilationUnit::RunOptimize( + const clang::CompilerInvocation& clang_invocation) -> void { + CARBON_CHECK(module_, "Must call RunLower first"); + + // TODO: A lot of the work done here duplicates work done by Clang setting up + // its pass manager. Moreover, we probably want to pick up Clang's + // customizations and make use of its flags for controlling LLVM passes. We + // should consider whether we would be better off running Clang's pass + // pipeline rather than building one of our own, or factoring out enough of + // Clang's pipeline builder that we can reuse and further customize it. + + MakeTargetMachine(clang_invocation); + + // TODO: There's no way to set these automatically from an + // llvm::OptimizationLevel. Add such a mechanism to LLVM and use it from + // here. For now we reconstruct what Clang does by default. + llvm::PipelineTuningOptions pto; + bool opt_for_speed = options_->opt_level == Lower::OptimizationLevel::Speed; + bool opt_for_size_or_speed = + opt_for_speed || options_->opt_level == Lower::OptimizationLevel::Size; + // Loop unrolling is enabled by `--optimize=size` but isn't actually performed + // because we add `optsize` attributes to the function definitions we emit. + pto.LoopUnrolling = opt_for_size_or_speed; + pto.LoopInterleaving = opt_for_size_or_speed; + pto.LoopVectorization = opt_for_speed; + pto.SLPVectorization = opt_for_size_or_speed; + + llvm::LoopAnalysisManager lam; + llvm::FunctionAnalysisManager fam; + llvm::CGSCCAnalysisManager cgam; + llvm::ModuleAnalysisManager mam; + + llvm::PassInstrumentationCallbacks pic; + + // Register standard pass instrumentations. This adds support for things like + // `-print-after-all`. + llvm::StandardInstrumentations si(module_->getContext(), + /*DebugLogging=*/false); + si.registerCallbacks(pic); + + llvm::PassBuilder builder(target_machine_.get(), pto, + /*PGOOpt=*/std::nullopt, &pic); + + // TODO: Add an AssignmentTrackingPass for at least `--optimize=debug`. + + // Set up target library information and add an analysis pass to supply it. + std::unique_ptr tlii(llvm::driver::createTLII( + module_->getTargetTriple(), llvm::driver::VectorLibrary::NoLibrary)); + fam.registerPass([&] { return llvm::TargetLibraryAnalysis(*tlii); }); + + builder.registerModuleAnalyses(mam); + builder.registerCGSCCAnalyses(cgam); + builder.registerFunctionAnalyses(fam); + builder.registerLoopAnalyses(lam); + builder.crossRegisterProxies(lam, fam, cgam, mam); + + llvm::ModulePassManager pass_manager = builder.buildPerModuleDefaultPipeline( + CompileOptions::GetLLVMOptimizationLevel(options_->opt_level)); + + if (vlog_stream_) { + CARBON_VLOG("*** Running pass pipeline: "); + pass_manager.printPipeline( + *vlog_stream_, [&pic](llvm::StringRef class_name) { + auto pass_name = pic.getPassNameForClassName(class_name); + return pass_name.empty() ? class_name : pass_name; + }); + CARBON_VLOG(" ***\n"); + } + + LogCall("ModulePassManager::run", "optimize", + [&] { pass_manager.run(*module_, mam); }); + + if (vlog_stream_) { + CARBON_VLOG("*** Optimized llvm::Module ***\n"); + module_->print(*vlog_stream_, /*AAW=*/nullptr, + /*ShouldPreserveUseListOrder=*/false, + /*IsForDebug=*/true); + } +} + +auto CompilationUnit::PostLower() -> void { + CARBON_CHECK(module_, "Must call RunLower first"); + if (options_->dump_llvm_ir && IncludeInDumps()) { + module_->print(*driver_env_->output_stream, /*AAW=*/nullptr, + /*ShouldPreserveUseListOrder=*/true); + } +} + +auto CompilationUnit::RunCodeGen() -> void { + CARBON_CHECK(module_, "Must call RunLower first"); + LogCall("CodeGen", "codegen", [&] { success_ = RunCodeGenHelper(); }); +} + +auto CompilationUnit::PostCompile() -> void { + if (options_->dump_shared_values && IncludeInDumps()) { + Yaml::Print(*driver_env_->output_stream, + value_stores_.OutputYaml(input_filename_)); + } + if (mem_usage_) { + mem_usage_->Collect("value_stores_", value_stores_); + Yaml::Print(*driver_env_->output_stream, + mem_usage_->OutputYaml(input_filename_)); + } + if (timings_) { + Yaml::Print(*driver_env_->output_stream, + timings_->OutputYaml(input_filename_)); + } + + // The diagnostics consumer must be flushed before compilation artifacts are + // destructed, because diagnostics can refer to their state. + consumer_->Flush(); +} + +auto CompilationUnit::RunCodeGenHelper() -> bool { + CARBON_CHECK(module_, "Must call RunLower first"); + CARBON_CHECK(target_machine_, "Must call MakeTargetMachine first"); + + CodeGen codegen(module_.get(), target_machine_.get(), consumer_); + if (vlog_stream_) { + CARBON_VLOG("*** Assembly ***\n"); + codegen.EmitAssembly(*vlog_stream_); + } + + if (output_filename_ == "-") { + // TODO: The output file name, forcing object output, and requesting + // textual assembly output are all somewhat linked flags. We should add + // some validation that they are used correctly. + if (options_->force_obj_output) { + if (!codegen.EmitObject(*driver_env_->output_stream)) { + return false; + } + } else { + if (!codegen.EmitAssembly(*driver_env_->output_stream)) { + return false; + } + } + } else { + llvm::SmallString<256> output_filename = llvm::StringRef(output_filename_); + if (output_filename.empty()) { + if (!source_->is_regular_file()) { + // Don't invent file names like `-.o` or `/dev/stdin.o`. + // TODO: Consider rephrasing the diagnostic to use the file as the + // `Emit` location. + CARBON_DIAGNOSTIC(CompileInputNotRegularFile, Error, + "output file name must be specified for input `{0}` " + "that is not a regular file", + std::string); + driver_env_->emitter.Emit(CompileInputNotRegularFile, input_filename_); + return false; + } + output_filename = input_filename_; + llvm::sys::path::replace_extension(output_filename, + options_->asm_output ? ".s" : ".o"); + } + CARBON_VLOG("Writing output to: {0}\n", output_filename); + + std::error_code ec; + llvm::raw_fd_ostream output_file(output_filename, ec, + llvm::sys::fs::OF_None); + if (ec) { + // TODO: Consider rephrasing the diagnostic to use the file as the `Emit` + // location. + CARBON_DIAGNOSTIC(CompileOutputFileOpenError, Error, + "could not open output file `{0}`: {1}", std::string, + std::string); + driver_env_->emitter.Emit(CompileOutputFileOpenError, + output_filename.str().str(), ec.message()); + return false; + } + if (options_->asm_output) { + if (!codegen.EmitAssembly(output_file)) { + return false; + } + } else { + if (!codegen.EmitObject(output_file)) { + return false; + } + } + } + return true; +} + +auto CompilationUnit::GetParseTreeAndSubtrees() + -> const Parse::TreeAndSubtrees& { + if (!parse_tree_and_subtrees_) { + parse_tree_and_subtrees_ = Parse::TreeAndSubtrees(*tokens_, *parse_tree_); + if (mem_usage_) { + mem_usage_->Collect("parse_tree_and_subtrees_", + *parse_tree_and_subtrees_); + } + } + return *parse_tree_and_subtrees_; +} + +auto CompilationUnit::LogCall(llvm::StringLiteral logging_label, + llvm::StringLiteral timing_label, + llvm::function_refvoid> fn) -> void { + PrettyStackTraceFunction trace_file([&](llvm::raw_ostream& out) { + out << "Filename: " << input_filename_ << "\n"; + }); + CARBON_VLOG("*** {0}: {1} ***\n", logging_label, input_filename_); + Timings::ScopedTiming timing(timings_ ? &*timings_ : nullptr, timing_label); + fn(); + CARBON_VLOG("*** {0} done ***\n", logging_label); +} + +CompileDriver::CompileDriver(CompileOptions* options) : options_(options) {} + +auto CompileDriver::Initialize( + DriverEnv& driver_env, + llvm::function_refstd::string> map_input) -> bool { + if (!options_->ValidatePhase(driver_env.emitter)) { + return false; + } + + // Validate the target before passing it to Clang. + const llvm::Target* target; + if (auto t = options_->ValidateTarget(driver_env.emitter); !t.ok()) { + return false; + } else { + target = *t; + } + + if (auto i = options_->BuildClangInvocation(driver_env); !i.ok()) { + return false; + } else { + clang_invocation_ = std::move(*i); + } + + // Find the files comprising the prelude if we are importing it. + // TODO: Replace this with a search for library api files in a + // package-specific search path based on the library name. + llvm::SmallVector prelude; + if (options_->prelude_import && !options_->custom_core && + options_->phase >= CompileOptions::Phase::Check) { + if (auto find = driver_env.installation->ReadPreludeManifest(); + !find.ok()) { + // TODO: Change ReadPreludeManifest to produce diagnostics. + CARBON_DIAGNOSTIC(CompilePreludeManifestError, Error, "{0}", std::string); + driver_env.emitter.Emit(CompilePreludeManifestError, + PrintToString(find.error())); + return false; + } else { + prelude = std::move(*find); + } + } + + // Prepare CompilationUnits before building scope exit handlers. + int unit_index = -1; + int total_unit_count = prelude.size() + options_->input_filenames.size(); + auto unit_builder = [&](llvm::StringRef filename) { + ++unit_index; + return std::make_unique( + SemIR::CheckIRId(unit_index), total_unit_count, &driver_env, options_, + &driver_env.consumer, filename, map_input(filename), target); + }; + llvm::append_range(units_, llvm::map_range(prelude, unit_builder)); + input_filenames_index_ = units_.size(); + llvm::append_range(units_, + llvm::map_range(options_->input_filenames, unit_builder)); + CARBON_CHECK(units_.size() == static_cast(total_unit_count)); + + // Add the cache to all units. This must be done after all units are created. + cache_ = std::make_unique(options_, units_); + for (auto& unit : units_) { + unit->SetMultiUnitCache(cache_.get()); + } + + return true; +} + +auto CompileDriver::Compile(DriverEnv& driver_env) -> DriverResult { + auto on_exit = llvm::scope_exit([&]() { + // Finish compilation units. This flushes their diagnostics in the order in + // which they were specified on the command line. + for (auto& unit : units_) { + unit->PostCompile(); + } + + driver_env.consumer.Flush(); + }); + + PrettyStackTraceFunction flush_on_crash([&](llvm::raw_ostream& out) { + // When crashing, flush diagnostics. If sorting diagnostics, they can be + // redirected to the crash stream; if streaming, the original stream is + // flushed. + // TODO: Eventually we'll want to limit the count. + if (options_->stream_errors) { + out << "Flushing diagnostics\n"; + } else { + out << "Pending diagnostics:\n"; + driver_env.consumer.set_stream(&out); + } + + for (auto& unit : units_) { + unit->FlushForStackTrace(); + } + driver_env.consumer.Flush(); + driver_env.consumer.set_stream(driver_env.error_stream); + }); + + // Returns a DriverResult object. Called whenever Compile returns. + auto make_result = [&]() { + DriverResult result = {.success = true}; + for (const auto& unit : units_) { + result.success &= unit->success(); + result.per_file_success.push_back( + {unit->input_filename().str(), unit->success()}); + } + return result; + }; + + // Lex. + for (auto& unit : units_) { + unit->RunLex(); + } + if (options_->phase == CompileOptions::Phase::Lex) { + return make_result(); + } + cache_->ApplyPerFileIncludeInDumps(); + // Parse and check phases examine `has_source` because they want to proceed if + // lex failed, but not if source doesn't exist. Later steps are skipped if + // anything failed, so don't need this. + + // Parse. + for (auto& unit : units_) { + if (unit->has_source()) { + unit->RunParse(); + } + } + if (options_->phase == CompileOptions::Phase::Parse) { + return make_result(); + } + + // Gather Check::Units. + llvm::SmallVector check_units; + check_units.reserve(units_.size()); + for (auto& unit : units_) { + if (unit->has_source()) { + check_units.push_back(unit->GetCheckUnit()); + } + } + + // Execute the actual checking. + CARBON_VLOG_TO(driver_env.vlog_stream, "*** Check::CheckParseTrees ***\n"); + Check::CheckParseTreesOptions options; + options.prelude_import = options_->prelude_import; + options.vlog_stream = driver_env.vlog_stream; + options.fuzzing = driver_env.fuzzing; + options.mangle_string_fingerprint = options_->mangle_string_fingerprint; + if (options.vlog_stream || options_->dump_sem_ir || options_->dump_cpp_ast || + options_->dump_raw_sem_ir) { + options.include_in_dumps = &cache_->include_in_dumps(); + if (options_->dump_sem_ir) { + options.dump_stream = driver_env.output_stream; + } + if (options_->dump_cpp_ast) { + options.dump_cpp_ast_stream = driver_env.output_stream; + } + if (options.vlog_stream || options_->dump_sem_ir) { + options.dump_sem_ir_ranges = options_->dump_sem_ir_ranges; + } + if (options_->dump_raw_sem_ir) { + options.raw_dump_stream = driver_env.output_stream; + options.dump_raw_sem_ir_builtins = options_->builtin_sem_ir; + } + options.sem_ir_crash_dump = options_->sem_ir_crash_dump; + } + + Check::CheckParseTrees(check_units, cache_->tree_and_subtrees_getters(), + driver_env.fs, options, clang_invocation_); + CARBON_VLOG_TO(driver_env.vlog_stream, + "*** Check::CheckParseTrees done ***\n"); + for (auto& unit : units_) { + if (unit->has_source()) { + unit->PostCheck(); + } + } + if (options_->phase == CompileOptions::Phase::Check) { + return make_result(); + } + + // Unlike previous steps, errors block further progress. + if (llvm::any_of(units_, + [&](const auto& unit) { return !unit->success(); })) { + CARBON_VLOG_TO(driver_env.vlog_stream, + "*** Stopping before lowering due to errors ***\n"); + return make_result(); + } + + // Lower and optimize. + for (const auto& unit : units_) { + unit->RunLower(); + + if (options_->phase != CompileOptions::Phase::Lower) { + unit->RunOptimize(*clang_invocation_); + } + + unit->PostLower(); + } + if (options_->phase == CompileOptions::Phase::Lower || + options_->phase == CompileOptions::Phase::Optimize) { + return make_result(); + } + CARBON_CHECK(options_->phase == CompileOptions::Phase::CodeGen, + "CodeGen should be the last stage"); + + bool output_last_input_only = options_->output_last_input_only; + if (!output_last_input_only && units_.size() > 1 && + !options_->output_filename.empty() && options_->output_filename != "-") { + // TODO: Command line structure should change to make this implicit + // (passing non-compiling inputs differently), and the warning should be + // removed. + CARBON_DIAGNOSTIC( + CompileMultipleInputsWithOutput, Warning, + "only outputting {0} to {1}, skipping output of {2} input " + "file{2:s}; pass `--output-last-input-only` to silence this warning", + std::string, std::string, Diagnostics::IntAsSelect); + driver_env.emitter.Emit(CompileMultipleInputsWithOutput, + units_.back()->input_filename().str(), + options_->output_filename.str(), units_.size() - 1); + output_last_input_only = true; + } + + // Codegen. + if (output_last_input_only) { + units_.back()->RunCodeGen(); + } else { + for (const auto& unit : units_) { + unit->RunCodeGen(); + } + } + return make_result(); +} + +} // namespace Carbon diff --git a/toolchain/driver/compile_driver.h b/toolchain/driver/compile_driver.h new file mode 100644 index 000000000000..d70a4b3938eb --- /dev/null +++ b/toolchain/driver/compile_driver.h @@ -0,0 +1,263 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef CARBON_TOOLCHAIN_DRIVER_COMPILE_DRIVER_H_ +#define CARBON_TOOLCHAIN_DRIVER_COMPILE_DRIVER_H_ + +#include + +#include "llvm/ADT/SmallVector.h" +#include "llvm/Target/TargetMachine.h" +#include "toolchain/diagnostics/sorting_consumer.h" +#include "toolchain/driver/compile_options.h" +#include "toolchain/driver/driver_env.h" +#include "toolchain/driver/driver_subcommand.h" + +namespace Carbon { + +class MultiUnitCache; + +// Ties together information for a file being compiled. +class CompilationUnit { + public: + // `driver_env`, `options`, `consumer`, and `target` must be non-null. + explicit CompilationUnit(SemIR::CheckIRId check_ir_id, int total_ir_count, + DriverEnv* driver_env, const CompileOptions* options, + Diagnostics::Consumer* consumer, + llvm::StringRef input_filename, + std::string output_filename, + const llvm::Target* target); + + // Sets the multi-unit cache and initializes dependent member state. + auto SetMultiUnitCache(MultiUnitCache* cache) -> void; + + // Loads source and lexes it. Returns true on success. + auto RunLex() -> void; + + // Parses tokens. Returns true on success. + auto RunParse() -> void; + + // Returns information needed to check this unit. + auto GetCheckUnit() -> Check::Unit; + + // Runs post-check logic. Returns true if checking succeeded for the IR. + auto PostCheck() -> void; + + // Lower SemIR to LLVM IR. + auto RunLower() -> void; + + // Runs the optimization pipeline. + auto RunOptimize(const clang::CompilerInvocation& clang_invocation) -> void; + + // Runs post-lowering-to-LLVM-IR logic. This is always called if we do any + // lowering work, after we've finished building the IR in RunLower() and, + // optionally, RunOptimize(). + auto PostLower() -> void; + + auto RunCodeGen() -> void; + + // Runs post-compile logic. This is always called, and called after all other + // actions on the CompilationUnit. + auto PostCompile() -> void; + + // Flushes diagnostics, specifically as part of generating stack trace + // information. + auto FlushForStackTrace() -> void { consumer_->Flush(); } + + auto input_filename() -> llvm::StringRef { return input_filename_; } + auto output_filename() -> llvm::StringRef { return output_filename_; } + auto has_include_in_dumps() -> bool { + return tokens_ && tokens_->has_include_in_dumps(); + } + auto success() -> bool { return success_; } + auto has_source() -> bool { return source_.has_value(); } + auto get_trees_and_subtrees() -> Parse::GetTreeAndSubtreesFn { + return *tree_and_subtrees_getter_; + } + + private: + // Do codegen. Returns true on success. + auto RunCodeGenHelper() -> bool; + + // The TreeAndSubtrees is mainly used for debugging and diagnostics, and has + // significant overhead. Avoid constructing it when unused. + auto GetParseTreeAndSubtrees() -> const Parse::TreeAndSubtrees&; + + // Wraps a call with log statements to indicate start and end. Typically logs + // with the actual function name, but marks timings with the appropriate + // phase. + auto LogCall(llvm::StringLiteral logging_label, + llvm::StringLiteral timing_label, + llvm::function_refvoid> fn) -> void; + + // Returns true if the current file should be included in debug dumps. + auto IncludeInDumps() -> bool; + + // Builds the LLVM target machine. + auto MakeTargetMachine(const clang::CompilerInvocation& clang_invocation) + -> void; + + // The index of the unit amongst all units. + SemIR::CheckIRId check_ir_id_; + // The number of units in total. + int total_ir_count_; + + DriverEnv* driver_env_; + const CompileOptions* options_; + const llvm::Target* target_; + + SharedValueStores value_stores_; + + // The input filename from the command line. For most diagnostics, we + // typically use `source_->filename()`, which includes a `-` -> `` + // translation. However, logging and some diagnostics use the command line + // argument. + std::string input_filename_; + std::string output_filename_; + + // Copied from driver_ for CARBON_VLOG. + llvm::raw_pwrite_stream* vlog_stream_; + + // Diagnostics are sent to consumer_, with optional sorting. + std::optional sorting_consumer_; + Diagnostics::Consumer* consumer_; + + bool success_ = true; + + // Initialized by `SetMultiUnitCache`. + MultiUnitCache* cache_ = nullptr; + // Tracks memory usage of the compile. + std::optional mem_usage_; + // Tracks timings of the compile. + std::optional timings_; + + // These are initialized as steps are run. + std::optional source_; + std::optional tokens_; + std::optional parse_tree_; + std::optional parse_tree_and_subtrees_; + std::optionalconst Parse::TreeAndSubtrees&>> + tree_and_subtrees_getter_; + std::unique_ptr llvm_context_; + std::optional sem_ir_; + std::unique_ptr module_; + std::unique_ptr target_machine_; +}; + +// Caches lists that are shared cross-unit. Accessors do lazy caching because +// they may not be used. +class MultiUnitCache { + public: + using IncludeInDumpsStore = FixedSizeValueStore; + using TreeAndSubtreesGettersStore = Parse::GetTreeAndSubtreesStore; + + // This relies on construction after `units` are all initialized, which is + // reflected by the `ArrayRef` here. + explicit MultiUnitCache( + const CompileOptions* options, + llvm::ArrayRef> units) + : options_(options), units_(units) {} + + // If `include_in_dumps` is in use, we need to apply per-file include + // settings. + auto ApplyPerFileIncludeInDumps() -> void { + if (!include_in_dumps_) { + // No cached value to update. + return; + } + for (const auto& [i, unit] : llvm::enumerate(units_)) { + if (unit->has_include_in_dumps()) { + include_in_dumps_->Set(SemIR::CheckIRId(i), true); + } + } + } + + auto include_in_dumps() -> const IncludeInDumpsStore& { + if (!include_in_dumps_) { + include_in_dumps_.emplace( + IncludeInDumpsStore::MakeWithExplicitSize(units_.size(), false)); + for (const auto& [i, unit] : llvm::enumerate(units_)) { + // If this is first accessed after lexing is complete, we need to apply + // per-file includes. Otherwise, this is based only on the exclude + // option. + bool include = + unit->has_include_in_dumps() || + llvm::none_of(options_->exclude_dump_file_prefixes, + [&](auto prefix) { + return unit->input_filename().starts_with(prefix); + }); + include_in_dumps_->Set(SemIR::CheckIRId(i), include); + } + } + return *include_in_dumps_; + } + + auto tree_and_subtrees_getters() -> const TreeAndSubtreesGettersStore& { + if (!tree_and_subtrees_getters_) { + tree_and_subtrees_getters_.emplace( + TreeAndSubtreesGettersStore::MakeWithExplicitSize(units_.size(), + nullptr)); + for (const auto& [i, unit] : llvm::enumerate(units_)) { + if (unit->has_source()) { + tree_and_subtrees_getters_->Set(SemIR::CheckIRId(i), + unit->get_trees_and_subtrees()); + } + } + } + return *tree_and_subtrees_getters_; + } + + private: + const CompileOptions* options_; + + // The units being compiled. + llvm::ArrayRef> units_; + + // For each unit, whether it's included in dumps. Used cross-phase. + std::optional include_in_dumps_; + + // For each unit, the `TreeAndSubtrees` getter. Used by lowering. + std::optional tree_and_subtrees_getters_; +}; + +// Helper class to compile C++ and Carbon input files. Used by the `build` and +// `compile` subcommands. +class CompileDriver { + public: + explicit CompileDriver(CompileOptions* options); + + // Configure the toolchain to compile all input files and dependencies. + // The `map_input` function maps an input file name to an output static + // object name. + // Returns `false` on configuration error. + auto Initialize( + DriverEnv& driver_env, + llvm::function_refstd::string> map_input) -> bool; + + // Performs the compilation process on each input specified in the + // `CompileOptions` provided at construction time. + auto Compile(DriverEnv& driver_env) -> DriverResult; + + // Returns the index in the `units()` array of the first input file + // specified by the user on the command line. This may not be the first + // element in the array due to dependency prepends. + auto first_input_index() -> size_t { return input_filenames_index_; } + + // Provides read-only access to the array of CompilationUnits, useful for + // subsequent processing of the compiled output products. + auto units() -> llvm::ArrayRef> { + return units_; + } + + private: + CompileOptions* options_; + size_t input_filenames_index_ = 0; + llvm::SmallVector, 256> units_; + std::unique_ptr cache_; + std::shared_ptr clang_invocation_; +}; + +} // namespace Carbon + +#endif // CARBON_TOOLCHAIN_DRIVER_COMPILE_DRIVER_H_ diff --git a/toolchain/driver/compile_options.cpp b/toolchain/driver/compile_options.cpp new file mode 100644 index 000000000000..835369bcdc24 --- /dev/null +++ b/toolchain/driver/compile_options.cpp @@ -0,0 +1,525 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "toolchain/driver/compile_options.h" + +#include + +#include "toolchain/base/clang_invocation.h" + +namespace Carbon { + +namespace { + +// Provides command-line options common to both `compile` and `link` +// subcommands. +auto BuildSharedOptions(CommandLine::CommandBuilder& b, CompileOptions* options) + -> void { + b.AddStringPositionalArg( + { + .name = "FILE", + .help = R"""( +The input Carbon source file to compile. +)""", + }, + [&](auto& arg_b) { + arg_b.Required(true); + arg_b.Append(&options->input_filenames); + }); + + b.AddStringOption( + { + .name = "clang-arg", + .value_name = "CLANG-ARG", + .help = R"""( +An argument to pass to the Clang compiler for use when compiling imported C++ +code. + +All flags that are accepted by the Clang driver are supported. However, you +cannot specify arguments that would result in additional compilations being +performed. Use `carbon clang` instead to compile additional source files. +)""", + }, + [&](auto& arg_b) { arg_b.Append(&options->clang_args); }); + + b.AddStringPositionalArg( + { + .name = "CLANG-ARG", + .help = R"""( +Additional Clang arguments. See help for `--clang-arg` for details. +)""", + }, + [&](auto& arg_b) { arg_b.Append(&options->clang_args); }); + + b.AddOneOfOption( + { + .name = "optimize", + .help = R"""( +Selects the amount of optimization to perform. +)""", + }, + [&](auto& arg_b) { + arg_b.SetOneOf( + { + // We intentionally don't expose O2 and Os. The difference + // between these levels tends to reflect what achieves the + // best speed for a specific application, as they all + // largely optimize for speed as the primary factor. + // + // Instead of controlling this with more nuanced flags, we + // plan to support profile and in-source hints to the + // optimizer to adjust its strategy in the specific places + // where the default doesn't have the desired results. + arg_b.OneOfValue("none", Lower::OptimizationLevel::None), + arg_b.OneOfValue("debug", Lower::OptimizationLevel::Debug), + arg_b.OneOfValue("speed", Lower::OptimizationLevel::Speed), + arg_b.OneOfValue("size", Lower::OptimizationLevel::Size), + }, + &options->opt_level); + }); + + // Include the common code generation options at this point to render it + // after the more common options above, but before the more unusual options + // below. + options->codegen_options->Build(b); + + b.AddFlag( + { + .name = "debug-info", + .help = R"""( +Whether to emit DWARF debug information. +)""", + }, + [&](auto& arg_b) { + arg_b.Default(true); + arg_b.Set(&options->include_debug_info); + }); + b.AddFlag( + { + .name = "verify-llvm-ir", + .help = R"""( +Whether to run the LLVM verifier on modules. +)""", + }, + [&](auto& arg_b) { + arg_b.Default(true); + arg_b.Set(&options->run_llvm_verifier); + }); + b.AddFlag( + { + .name = "prelude-import", + .help = R"""( +Whether to use the implicit prelude import. Enabled by default. +)""", + }, + [&](auto& arg_b) { + arg_b.Default(true); + arg_b.Set(&options->prelude_import); + }); +} + +} // namespace + +auto CompileOptions::BuildForCompileSubcommand(CommandLine::CommandBuilder& b) + -> void { + BuildSharedOptions(b, this); + + b.AddOneOfOption( + { + .name = "phase", + .help = R"""( +Selects the compilation phase to run. These phases are always run in sequence, +so every phase before the one selected will also be run. The default is to +compile to machine code. +)""", + }, + [&](auto& arg_b) { + arg_b.SetOneOf( + { + arg_b.OneOfValue("lex", Phase::Lex), + arg_b.OneOfValue("parse", Phase::Parse), + arg_b.OneOfValue("check", Phase::Check), + arg_b.OneOfValue("lower", Phase::Lower), + arg_b.OneOfValue("optimize", Phase::Optimize), + arg_b.OneOfValue("codegen", Phase::CodeGen).Default(true), + }, + &phase); + }); + + // TODO: Rearrange the code setting this option and two related ones to + // allow them to reference each other instead of hard-coding their names. + b.AddStringOption( + { + .name = "output", + .value_name = "FILE", + .help = R"""( +The output filename for codegen. + +When this is a file name, either textual assembly or a binary object will be +written to it based on the flag `--asm-output`. The default is to write a binary +object file. + +Passing `--output=-` will write the output to stdout. In that case, the flag +`--asm-output` is ignored and the output defaults to textual assembly. Binary +object output can be forced by enabling `--force-obj-output`. +)""", + }, + [&](auto& arg_b) { arg_b.Set(&output_filename); }); + b.AddFlag( + { + .name = "asm-output", + .help = R"""( +Write textual assembly rather than a binary object file to the code generation +output. + +This flag only applies when writing to a file. When writing to stdout, the +default is textual assembly and this flag is ignored. +)""", + }, + [&](auto& arg_b) { arg_b.Set(&asm_output); }); + + b.AddFlag( + { + .name = "force-obj-output", + .help = R"""( +Force binary object output, even with `--output=-`. + +When `--output=-` is set, the default is textual assembly; this forces printing +of a binary object file instead. Ignored for other `--output` values. +)""", + }, + [&](auto& arg_b) { arg_b.Set(&force_obj_output); }); + + b.AddFlag( + { + .name = "stream-errors", + .help = R"""( +Stream error messages to stderr as they are generated rather than sorting them +and displaying them in source order. +)""", + }, + [&](auto& arg_b) { arg_b.Set(&stream_errors); }); + + b.AddFlag( + { + .name = "dump-shared-values", + .help = R"""( +Dumps shared values. These aren't owned by any particular file or phase. +)""", + }, + [&](auto& arg_b) { arg_b.Set(&dump_shared_values); }); + b.AddFlag( + { + .name = "dump-tokens", + .help = R"""( +Dump the tokens to stdout when lexed. +)""", + }, + [&](auto& arg_b) { arg_b.Set(&dump_tokens); }); + + b.AddFlag( + { + .name = "omit-file-boundary-tokens", + .help = R"""( +For `--dump-tokens`, omit file start and end boundary tokens. +)""", + }, + [&](auto& arg_b) { arg_b.Set(&omit_file_boundary_tokens); }); + + b.AddFlag( + { + .name = "dump-parse-tree", + .help = R"""( +Dump the parse tree to stdout when parsed. +)""", + }, + [&](auto& arg_b) { arg_b.Set(&dump_parse_tree); }); + b.AddFlag( + { + .name = "preorder-parse-tree", + .help = R"""( +When dumping the parse tree, reorder it so that it is in preorder rather than +postorder. +)""", + }, + [&](auto& arg_b) { arg_b.Set(&preorder_parse_tree); }); + b.AddFlag( + { + .name = "dump-raw-sem-ir", + .help = R"""( +Dump the raw JSON structure of SemIR to stdout when built. +)""", + }, + [&](auto& arg_b) { arg_b.Set(&dump_raw_sem_ir); }); + b.AddFlag( + { + .name = "dump-sem-ir", + .help = R"""( +Dump the full SemIR to stdout when built. +)""", + }, + [&](auto& arg_b) { arg_b.Set(&dump_sem_ir); }); + b.AddFlag( + { + .name = "dump-cpp-ast", + .help = R"""( +Dump the full C++ AST to stdout when built. +)""", + }, + [&](auto& arg_b) { arg_b.Set(&dump_cpp_ast); }); + + b.AddOneOfOption( + { + .name = "dump-sem-ir-ranges", + .help = R"""( +Selects handling of `//@dump-sem-ir-[begin|end]` markers when dumping SemIR. +By default, `if-present` prints ranges for files that have them, and full SemIR +for files that don't. `only` skips files with no ranges, and `ignore` always +prints full SemIR. +)""", + }, + [&](auto& arg_b) { + using DumpSemIRRanges = Check::CheckParseTreesOptions::DumpSemIRRanges; + arg_b.SetOneOf( + { + arg_b.OneOfValue("if-present", DumpSemIRRanges::IfPresent) + .Default(true), + arg_b.OneOfValue("only", DumpSemIRRanges::Only), + arg_b.OneOfValue("ignore", DumpSemIRRanges::Ignore), + }, + &dump_sem_ir_ranges); + }); + + b.AddFlag( + { + .name = "builtin-sem-ir", + .help = R"""( +Include the SemIR for builtins when dumping it. +)""", + }, + [&](auto& arg_b) { arg_b.Set(&builtin_sem_ir); }); + b.AddFlag( + { + .name = "dump-llvm-ir", + .help = R"""( +Dump the LLVM IR to stdout after lowering. +)""", + }, + [&](auto& arg_b) { arg_b.Set(&dump_llvm_ir); }); + b.AddFlag( + { + .name = "dump-asm", + .help = R"""( +Dump the generated assembly to stdout after codegen. +)""", + }, + [&](auto& arg_b) { arg_b.Set(&dump_asm); }); + b.AddFlag( + { + .name = "dump-mem-usage", + .help = R"""( +Dumps the amount of memory used. +)""", + }, + [&](auto& arg_b) { arg_b.Set(&dump_mem_usage); }); + b.AddFlag( + { + .name = "dump-timings", + .help = R"""( +Dumps the duration of each phase for each compilation unit. +)""", + }, + [&](auto& arg_b) { arg_b.Set(&dump_timings); }); + b.AddFlag( + { + .name = "custom-core", + .value_name = "CUSTOM_CORE", + .help = R"""( +Whether to use a custom Core package, the files for which must all be included +in the compile command line. + +The prelude library in the Core package is imported automatically. By default, +the Core package shipped with the toolchain is used, and its files do not need +to be specified in the compile command line. +)""", + }, + [&](auto& arg_b) { + arg_b.Default(false); + arg_b.Set(&custom_core); + }); + b.AddStringOption( + { + .name = "exclude-dump-file-prefix", + .value_name = "PREFIX", + .help = R"""( +Excludes files with the given prefix from dumps. +)""", + }, + [&](auto& arg_b) { arg_b.Append(&exclude_dump_file_prefixes); }); + b.AddFlag( + { + .name = "output-last-input-only", + .help = R"""( +Only write output for the last input file, ignoring all others. + +TODO: This is a temporary workaround and should be removed once separate +compilation is better implemented. +)""", + }, + [&](auto& arg_b) { arg_b.Set(&output_last_input_only); }); + b.AddStringOption( + { + .name = "sem-ir-crash-dump", + .value_name = "PATH", + .help = R"""( +Where to write a dump of the raw SemIR emitted so far, in the event of a crash +in the check phase. If empty, the dump is not written. +)""", + }, + [&](auto& arg_b) { arg_b.Set(&sem_ir_crash_dump); }); + b.AddFlag( + { + .name = "mangle-string-fingerprint", + .help = R"""( +Use the string form of the fingerprint from mangling instead of the hash form. +)""", + }, + [&](auto& arg_b) { arg_b.Set(&mangle_string_fingerprint); }); +} + +auto CompileOptions::BuildForBuildSubcommand(CommandLine::CommandBuilder& b) + -> void { + BuildSharedOptions(b, this); +} + +auto CompileOptions::ValidatePhase(Diagnostics::NoLocEmitter& emitter) const + -> bool { + CARBON_DIAGNOSTIC( + CompilePhaseFlagConflict, Error, + "requested dumping {0} but compile phase is limited to `{1}`", + std::string, std::string); + using Phase = CompileOptions::Phase; + switch (phase) { + case Phase::Lex: + if (dump_parse_tree) { + emitter.Emit(CompilePhaseFlagConflict, "parse tree", + CompileOptions::PhaseToString(phase)); + return false; + } + [[fallthrough]]; + case Phase::Parse: + if (dump_sem_ir) { + emitter.Emit(CompilePhaseFlagConflict, "SemIR", + CompileOptions::PhaseToString(phase)); + return false; + } + if (dump_cpp_ast) { + emitter.Emit(CompilePhaseFlagConflict, "C++ AST", + CompileOptions::PhaseToString(phase)); + return false; + } + [[fallthrough]]; + case Phase::Check: + if (dump_llvm_ir) { + emitter.Emit(CompilePhaseFlagConflict, "LLVM IR", + CompileOptions::PhaseToString(phase)); + return false; + } + [[fallthrough]]; + case Phase::Lower: + case Phase::Optimize: + case Phase::CodeGen: + // Everything can be dumped in these phases. + break; + } + return true; +} + +auto CompileOptions::ValidateTarget(Diagnostics::NoLocEmitter& emitter) + -> ErrorOr { + std::string target_error; + const llvm::Target* target = llvm::TargetRegistry::lookupTarget( + llvm::Triple(codegen_options->target), target_error); + if (!target) { + CARBON_DIAGNOSTIC(CompileTargetInvalid, Error, "invalid target: {0}", + std::string); + emitter.Emit(CompileTargetInvalid, target_error); + return ErrorBuilder() << "Invalid LLVM target: " << target_error; + } + + return target; +} + +auto CompileOptions::BuildClangInvocation(DriverEnv& driver_env) + -> ErrorOr> { + // TODO: Move this into `BuildClangInvocation` when it can accept an + // optimization level. + llvm::SmallVector all_clang_args = { + // Propagate our optimization level to Clang as a default. This can be + // overridden by Clang arguments, but doing so will only have an effect + // if those arguments affect Clang's IR, not its pass pipeline. + CompileOptions::GetClangOptimizationFlag(opt_level), + }; + all_clang_args.append(clang_args); + auto clang_invocation = Carbon::BuildClangInvocation( + driver_env.consumer, driver_env.fs, *driver_env.installation, + codegen_options->target, all_clang_args); + if (!clang_invocation) { + return ErrorBuilder() << "Failed to build a valid clang invocation."; + } + // We will run our own pass pipeline over the IR in the `Optimize` phase, so + // disable Clang's pipeline to avoid optimizing C++ code twice. + clang_invocation->getCodeGenOpts().DisableLLVMPasses = true; + return std::shared_ptr(clang_invocation.release()); +} + +// static +auto CompileOptions::GetLLVMOptimizationLevel( + Lower::OptimizationLevel opt_level) -> llvm::OptimizationLevel { + switch (opt_level) { + case Lower::OptimizationLevel::None: + return llvm::OptimizationLevel::O0; + case Lower::OptimizationLevel::Debug: + return llvm::OptimizationLevel::O1; + case Lower::OptimizationLevel::Size: + return llvm::OptimizationLevel::O2; + case Lower::OptimizationLevel::Speed: + return llvm::OptimizationLevel::O3; + } +} + +// static +auto CompileOptions::GetClangOptimizationFlag( + Lower::OptimizationLevel opt_level) -> llvm::StringLiteral { + switch (opt_level) { + case Lower::OptimizationLevel::None: + return "-O0"; + case Lower::OptimizationLevel::Debug: + return "-O1"; + case Lower::OptimizationLevel::Size: + return "-O2"; + case Lower::OptimizationLevel::Speed: + return "-O3"; + } +} + +// static +auto CompileOptions::PhaseToString(CompileOptions::Phase phase) -> std::string { + switch (phase) { + case CompileOptions::Phase::Lex: + return "lex"; + case CompileOptions::Phase::Parse: + return "parse"; + case CompileOptions::Phase::Check: + return "check"; + case CompileOptions::Phase::Lower: + return "lower"; + case CompileOptions::Phase::Optimize: + return "optimize"; + case CompileOptions::Phase::CodeGen: + return "codegen"; + } +} + +} // namespace Carbon diff --git a/toolchain/driver/compile_options.h b/toolchain/driver/compile_options.h new file mode 100644 index 000000000000..a36a8223089d --- /dev/null +++ b/toolchain/driver/compile_options.h @@ -0,0 +1,124 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef CARBON_TOOLCHAIN_DRIVER_COMPILE_OPTIONS_H_ +#define CARBON_TOOLCHAIN_DRIVER_COMPILE_OPTIONS_H_ + +#include + +#include "common/command_line.h" +#include "common/error.h" +#include "common/ostream.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/MC/TargetRegistry.h" +#include "llvm/Passes/OptimizationLevel.h" +#include "toolchain/check/check.h" +#include "toolchain/diagnostics/emitter.h" +#include "toolchain/driver/codegen_options.h" +#include "toolchain/driver/driver_env.h" +#include "toolchain/lower/lower.h" + +namespace Carbon { + +// Options for Carbon compilation. This struct is shared between the +// `build` and `compile` subcommands, supporting different flags +// for each subcomand. +// +// Note that the each subcommand supports its own `Build*()` function, +// supporting the different use cases for command-line control of compilation. +// +// Members are documented in their respective `Build` functions. +struct CompileOptions { + enum class Phase : int8_t { + Lex, + Parse, + Check, + Lower, + Optimize, + CodeGen, + }; + + friend auto operator<<(llvm::raw_ostream& out, Phase phase) + -> llvm::raw_ostream&; + + auto BuildForCompileSubcommand(CommandLine::CommandBuilder& b) -> void; + auto BuildForBuildSubcommand(CommandLine::CommandBuilder& b) -> void; + + // Validate that the compile options make sense for the compilation phase + // selected. + auto ValidatePhase(Diagnostics::NoLocEmitter& emitter) const -> bool; + + // Validate the target before passing to clang. + auto ValidateTarget(Diagnostics::NoLocEmitter& emitter) + -> ErrorOr; + + // Build a clang invocation. We do this regardless of whether we're running + // check, because this is essentially performing further option validation, + // and we generally validate all options even if we're not using them for the + // selected phases of compilation. We also use Clang's target option handling + // to configure our target, to ensure that we are using the same ABI for both + // the C++ and Carbon parts of the compilation. + // TODO: Share any arguments we specify here with the `carbon clang` + // subcommand. + auto BuildClangInvocation(DriverEnv& driver_env) + -> ErrorOr>; + + Lower::OptimizationLevel opt_level = Lower::OptimizationLevel::Debug; + std::shared_ptr codegen_options = + std::make_shared(); + + llvm::SmallVector input_filenames; + llvm::SmallVector clang_args; + + bool include_debug_info = true; + bool run_llvm_verifier = true; + + Phase phase = Phase::CodeGen; + Check::CheckParseTreesOptions::DumpSemIRRanges dump_sem_ir_ranges; + + llvm::StringRef output_filename; + + bool asm_output = false; + bool force_obj_output = false; + bool custom_core = false; + bool dump_shared_values = false; + bool dump_tokens = false; + bool omit_file_boundary_tokens = false; + bool dump_parse_tree = false; + bool dump_raw_sem_ir = false; + bool dump_sem_ir = false; + bool dump_cpp_ast = false; + bool dump_llvm_ir = false; + bool dump_asm = false; + bool dump_mem_usage = false; + bool dump_timings = false; + bool stream_errors = false; + bool preorder_parse_tree = false; + bool builtin_sem_ir = false; + bool prelude_import = false; + bool output_last_input_only = false; + + llvm::SmallVector exclude_dump_file_prefixes; + + llvm::StringRef sem_ir_crash_dump; + + bool mangle_string_fingerprint = false; + + // Get the LLVM optimization level corresponding to a Carbon optimization + // level. + static auto GetLLVMOptimizationLevel(Lower::OptimizationLevel opt_level) + -> llvm::OptimizationLevel; + + // Get the `-O` flag corresponding to an optimization level. + static auto GetClangOptimizationFlag(Lower::OptimizationLevel opt_level) + -> llvm::StringLiteral; + + // Returns a string for printing the phase in a diagnostic. + static auto PhaseToString(CompileOptions::Phase phase) -> std::string; +}; + +} // namespace Carbon + +#endif // CARBON_TOOLCHAIN_DRIVER_COMPILE_OPTIONS_H_ diff --git a/toolchain/driver/compile_subcommand.cpp b/toolchain/driver/compile_subcommand.cpp index 1b627d14676d..7a8fd6b96806 100644 --- a/toolchain/driver/compile_subcommand.cpp +++ b/toolchain/driver/compile_subcommand.cpp @@ -4,403 +4,10 @@ #include "toolchain/driver/compile_subcommand.h" -#include -#include -#include -#include -#include -#include - -#include "common/pretty_stack_trace_function.h" -#include "common/vlog.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/ScopeExit.h" -#include "llvm/MC/TargetRegistry.h" -#include "llvm/Passes/OptimizationLevel.h" -#include "llvm/Passes/PassBuilder.h" -#include "llvm/Passes/StandardInstrumentations.h" -#include "toolchain/base/clang_invocation.h" -#include "toolchain/base/timings.h" -#include "toolchain/check/check.h" -#include "toolchain/codegen/codegen.h" -#include "toolchain/diagnostics/emitter.h" -#include "toolchain/diagnostics/format_providers.h" -#include "toolchain/diagnostics/sorting_consumer.h" -#include "toolchain/lex/lex.h" -#include "toolchain/lower/lower.h" -#include "toolchain/parse/parse.h" -#include "toolchain/parse/tree_and_subtrees.h" -#include "toolchain/sem_ir/ids.h" -#include "toolchain/source/source_buffer.h" +#include "toolchain/driver/compile_driver.h" namespace Carbon { -auto CompileOptions::Build(CommandLine::CommandBuilder& b) -> void { - b.AddStringPositionalArg( - { - .name = "FILE", - .help = R"""( -The input Carbon source file to compile. -)""", - }, - [&](auto& arg_b) { - arg_b.Required(true); - arg_b.Append(&input_filenames); - }); - - b.AddOneOfOption( - { - .name = "phase", - .help = R"""( -Selects the compilation phase to run. These phases are always run in sequence, -so every phase before the one selected will also be run. The default is to -compile to machine code. -)""", - }, - [&](auto& arg_b) { - arg_b.SetOneOf( - { - arg_b.OneOfValue("lex", Phase::Lex), - arg_b.OneOfValue("parse", Phase::Parse), - arg_b.OneOfValue("check", Phase::Check), - arg_b.OneOfValue("lower", Phase::Lower), - arg_b.OneOfValue("optimize", Phase::Optimize), - arg_b.OneOfValue("codegen", Phase::CodeGen).Default(true), - }, - &phase); - }); - - b.AddStringOption( - { - .name = "clang-arg", - .value_name = "CLANG-ARG", - .help = R"""( -An argument to pass to the Clang compiler for use when compiling imported C++ -code. - -All flags that are accepted by the Clang driver are supported. However, you -cannot specify arguments that would result in additional compilations being -performed. Use `carbon clang` instead to compile additional source files. -)""", - }, - [&](auto& arg_b) { arg_b.Append(&clang_args); }); - - b.AddStringPositionalArg( - { - .name = "CLANG-ARG", - .help = R"""( -Additional Clang arguments. See help for `--clang-arg` for details. -)""", - }, - [&](auto& arg_b) { arg_b.Append(&clang_args); }); - - // TODO: Rearrange the code setting this option and two related ones to - // allow them to reference each other instead of hard-coding their names. - b.AddStringOption( - { - .name = "output", - .value_name = "FILE", - .help = R"""( -The output filename for codegen. - -When this is a file name, either textual assembly or a binary object will be -written to it based on the flag `--asm-output`. The default is to write a binary -object file. - -Passing `--output=-` will write the output to stdout. In that case, the flag -`--asm-output` is ignored and the output defaults to textual assembly. Binary -object output can be forced by enabling `--force-obj-output`. -)""", - }, - [&](auto& arg_b) { arg_b.Set(&output_filename); }); - - b.AddOneOfOption( - { - .name = "optimize", - .help = R"""( -Selects the amount of optimization to perform. -)""", - }, - [&](auto& arg_b) { - arg_b.SetOneOf( - { - // We intentionally don't expose O2 and Os. The difference - // between these levels tends to reflect what achieves the - // best speed for a specific application, as they all - // largely optimize for speed as the primary factor. - // - // Instead of controlling this with more nuanced flags, we - // plan to support profile and in-source hints to the - // optimizer to adjust its strategy in the specific places - // where the default doesn't have the desired results. - arg_b.OneOfValue("none", Lower::OptimizationLevel::None), - arg_b.OneOfValue("debug", Lower::OptimizationLevel::Debug), - arg_b.OneOfValue("speed", Lower::OptimizationLevel::Speed), - arg_b.OneOfValue("size", Lower::OptimizationLevel::Size), - }, - &opt_level); - }); - - // Include the common code generation options at this point to render it - // after the more common options above, but before the more unusual options - // below. - codegen_options.Build(b); - - b.AddFlag( - { - .name = "asm-output", - .help = R"""( -Write textual assembly rather than a binary object file to the code generation -output. - -This flag only applies when writing to a file. When writing to stdout, the -default is textual assembly and this flag is ignored. -)""", - }, - [&](auto& arg_b) { arg_b.Set(&asm_output); }); - - b.AddFlag( - { - .name = "force-obj-output", - .help = R"""( -Force binary object output, even with `--output=-`. - -When `--output=-` is set, the default is textual assembly; this forces printing -of a binary object file instead. Ignored for other `--output` values. -)""", - }, - [&](auto& arg_b) { arg_b.Set(&force_obj_output); }); - - b.AddFlag( - { - .name = "stream-errors", - .help = R"""( -Stream error messages to stderr as they are generated rather than sorting them -and displaying them in source order. -)""", - }, - [&](auto& arg_b) { arg_b.Set(&stream_errors); }); - - b.AddFlag( - { - .name = "dump-shared-values", - .help = R"""( -Dumps shared values. These aren't owned by any particular file or phase. -)""", - }, - [&](auto& arg_b) { arg_b.Set(&dump_shared_values); }); - b.AddFlag( - { - .name = "dump-tokens", - .help = R"""( -Dump the tokens to stdout when lexed. -)""", - }, - [&](auto& arg_b) { arg_b.Set(&dump_tokens); }); - - b.AddFlag( - { - .name = "omit-file-boundary-tokens", - .help = R"""( -For `--dump-tokens`, omit file start and end boundary tokens. -)""", - }, - [&](auto& arg_b) { arg_b.Set(&omit_file_boundary_tokens); }); - - b.AddFlag( - { - .name = "dump-parse-tree", - .help = R"""( -Dump the parse tree to stdout when parsed. -)""", - }, - [&](auto& arg_b) { arg_b.Set(&dump_parse_tree); }); - b.AddFlag( - { - .name = "preorder-parse-tree", - .help = R"""( -When dumping the parse tree, reorder it so that it is in preorder rather than -postorder. -)""", - }, - [&](auto& arg_b) { arg_b.Set(&preorder_parse_tree); }); - b.AddFlag( - { - .name = "dump-raw-sem-ir", - .help = R"""( -Dump the raw JSON structure of SemIR to stdout when built. -)""", - }, - [&](auto& arg_b) { arg_b.Set(&dump_raw_sem_ir); }); - b.AddFlag( - { - .name = "dump-sem-ir", - .help = R"""( -Dump the full SemIR to stdout when built. -)""", - }, - [&](auto& arg_b) { arg_b.Set(&dump_sem_ir); }); - b.AddFlag( - { - .name = "dump-cpp-ast", - .help = R"""( -Dump the full C++ AST to stdout when built. -)""", - }, - [&](auto& arg_b) { arg_b.Set(&dump_cpp_ast); }); - - b.AddOneOfOption( - { - .name = "dump-sem-ir-ranges", - .help = R"""( -Selects handling of `//@dump-sem-ir-[begin|end]` markers when dumping SemIR. -By default, `if-present` prints ranges for files that have them, and full SemIR -for files that don't. `only` skips files with no ranges, and `ignore` always -prints full SemIR. -)""", - }, - [&](auto& arg_b) { - using DumpSemIRRanges = Check::CheckParseTreesOptions::DumpSemIRRanges; - arg_b.SetOneOf( - { - arg_b.OneOfValue("if-present", DumpSemIRRanges::IfPresent) - .Default(true), - arg_b.OneOfValue("only", DumpSemIRRanges::Only), - arg_b.OneOfValue("ignore", DumpSemIRRanges::Ignore), - }, - &dump_sem_ir_ranges); - }); - - b.AddFlag( - { - .name = "builtin-sem-ir", - .help = R"""( -Include the SemIR for builtins when dumping it. -)""", - }, - [&](auto& arg_b) { arg_b.Set(&builtin_sem_ir); }); - b.AddFlag( - { - .name = "dump-llvm-ir", - .help = R"""( -Dump the LLVM IR to stdout after lowering. -)""", - }, - [&](auto& arg_b) { arg_b.Set(&dump_llvm_ir); }); - b.AddFlag( - { - .name = "dump-asm", - .help = R"""( -Dump the generated assembly to stdout after codegen. -)""", - }, - [&](auto& arg_b) { arg_b.Set(&dump_asm); }); - b.AddFlag( - { - .name = "dump-mem-usage", - .help = R"""( -Dumps the amount of memory used. -)""", - }, - [&](auto& arg_b) { arg_b.Set(&dump_mem_usage); }); - b.AddFlag( - { - .name = "dump-timings", - .help = R"""( -Dumps the duration of each phase for each compilation unit. -)""", - }, - [&](auto& arg_b) { arg_b.Set(&dump_timings); }); - b.AddFlag( - { - .name = "prelude-import", - .help = R"""( -Whether to use the implicit prelude import. Enabled by default. -)""", - }, - [&](auto& arg_b) { - arg_b.Default(true); - arg_b.Set(&prelude_import); - }); - b.AddFlag( - { - .name = "custom-core", - .value_name = "CUSTOM_CORE", - .help = R"""( -Whether to use a custom Core package, the files for which must all be included -in the compile command line. - -The prelude library in the Core package is imported automatically. By default, -the Core package shipped with the toolchain is used, and its files do not need -to be specified in the compile command line. -)""", - }, - [&](auto& arg_b) { - arg_b.Default(false); - arg_b.Set(&custom_core); - }); - b.AddStringOption( - { - .name = "exclude-dump-file-prefix", - .value_name = "PREFIX", - .help = R"""( -Excludes files with the given prefix from dumps. -)""", - }, - [&](auto& arg_b) { arg_b.Append(&exclude_dump_file_prefixes); }); - b.AddFlag( - { - .name = "debug-info", - .help = R"""( -Whether to emit DWARF debug information. -)""", - }, - [&](auto& arg_b) { - arg_b.Default(true); - arg_b.Set(&include_debug_info); - }); - b.AddFlag( - { - .name = "output-last-input-only", - .help = R"""( -Only write output for the last input file, ignoring all others. - -TODO: This is a temporary workaround and should be removed once separate -compilation is better implemented. -)""", - }, - [&](auto& arg_b) { arg_b.Set(&output_last_input_only); }); - b.AddFlag( - { - .name = "verify-llvm-ir", - .help = R"""( -Whether to run the LLVM verifier on modules. -)""", - }, - [&](auto& arg_b) { - arg_b.Default(true); - arg_b.Set(&run_llvm_verifier); - }); - b.AddStringOption( - { - .name = "sem-ir-crash-dump", - .value_name = "PATH", - .help = R"""( -Where to write a dump of the raw SemIR emitted so far, in the event of a crash -in the check phase. If empty, the dump is not written. -)""", - }, - [&](auto& arg_b) { arg_b.Set(&sem_ir_crash_dump); }); - b.AddFlag( - { - .name = "mangle-string-fingerprint", - .help = R"""( -Use the string form of the fingerprint from mangling instead of the hash form. -)""", - }, - [&](auto& arg_b) { arg_b.Set(&mangle_string_fingerprint); }); -} - static constexpr CommandLine::CommandInfo SubcommandInfo = { .name = "compile", .help = R"""( @@ -418,938 +25,24 @@ can be written to standard output as these phases progress. CompileSubcommand::CompileSubcommand() : DriverSubcommand(SubcommandInfo) {} -// Returns a string for printing the phase in a diagnostic. -static auto PhaseToString(CompileOptions::Phase phase) -> std::string { - switch (phase) { - case CompileOptions::Phase::Lex: - return "lex"; - case CompileOptions::Phase::Parse: - return "parse"; - case CompileOptions::Phase::Check: - return "check"; - case CompileOptions::Phase::Lower: - return "lower"; - case CompileOptions::Phase::Optimize: - return "optimize"; - case CompileOptions::Phase::CodeGen: - return "codegen"; - } -} - -auto CompileSubcommand::ValidateOptions( - Diagnostics::NoLocEmitter& emitter) const -> bool { - CARBON_DIAGNOSTIC( - CompilePhaseFlagConflict, Error, - "requested dumping {0} but compile phase is limited to `{1}`", - std::string, std::string); - using Phase = CompileOptions::Phase; - switch (options_.phase) { - case Phase::Lex: - if (options_.dump_parse_tree) { - emitter.Emit(CompilePhaseFlagConflict, "parse tree", - PhaseToString(options_.phase)); - return false; - } - [[fallthrough]]; - case Phase::Parse: - if (options_.dump_sem_ir) { - emitter.Emit(CompilePhaseFlagConflict, "SemIR", - PhaseToString(options_.phase)); - return false; - } - if (options_.dump_cpp_ast) { - emitter.Emit(CompilePhaseFlagConflict, "C++ AST", - PhaseToString(options_.phase)); - return false; - } - [[fallthrough]]; - case Phase::Check: - if (options_.dump_llvm_ir) { - emitter.Emit(CompilePhaseFlagConflict, "LLVM IR", - PhaseToString(options_.phase)); - return false; - } - [[fallthrough]]; - case Phase::Lower: - case Phase::Optimize: - case Phase::CodeGen: - // Everything can be dumped in these phases. - break; - } - return true; -} - -namespace { - -class MultiUnitCache; - -// Ties together information for a file being compiled. -class CompilationUnit { - public: - // `driver_env`, `options`, `consumer`, and `target` must be non-null. - explicit CompilationUnit(SemIR::CheckIRId check_ir_id, int total_ir_count, - DriverEnv* driver_env, const CompileOptions* options, - Diagnostics::Consumer* consumer, - llvm::StringRef input_filename, - const llvm::Target* target); - - // Sets the multi-unit cache and initializes dependent member state. - auto SetMultiUnitCache(MultiUnitCache* cache) -> void; - - // Loads source and lexes it. Returns true on success. - auto RunLex() -> void; - - // Parses tokens. Returns true on success. - auto RunParse() -> void; - - // Returns information needed to check this unit. - auto GetCheckUnit() -> Check::Unit; - - // Runs post-check logic. Returns true if checking succeeded for the IR. - auto PostCheck() -> void; - - // Lower SemIR to LLVM IR. - auto RunLower() -> void; - - // Runs the optimization pipeline. - auto RunOptimize(const clang::CompilerInvocation& clang_invocation) -> void; - - // Runs post-lowering-to-LLVM-IR logic. This is always called if we do any - // lowering work, after we've finished building the IR in RunLower() and, - // optionally, RunOptimize(). - auto PostLower() -> void; - - auto RunCodeGen() -> void; - - // Runs post-compile logic. This is always called, and called after all other - // actions on the CompilationUnit. - auto PostCompile() -> void; - - // Flushes diagnostics, specifically as part of generating stack trace - // information. - auto FlushForStackTrace() -> void { consumer_->Flush(); } - - auto input_filename() -> llvm::StringRef { return input_filename_; } - auto has_include_in_dumps() -> bool { - return tokens_ && tokens_->has_include_in_dumps(); - } - auto success() -> bool { return success_; } - auto has_source() -> bool { return source_.has_value(); } - auto get_trees_and_subtrees() -> Parse::GetTreeAndSubtreesFn { - return *tree_and_subtrees_getter_; - } - - private: - // Do codegen. Returns true on success. - auto RunCodeGenHelper() -> bool; - - // The TreeAndSubtrees is mainly used for debugging and diagnostics, and has - // significant overhead. Avoid constructing it when unused. - auto GetParseTreeAndSubtrees() -> const Parse::TreeAndSubtrees&; - - // Wraps a call with log statements to indicate start and end. Typically logs - // with the actual function name, but marks timings with the appropriate - // phase. - auto LogCall(llvm::StringLiteral logging_label, - llvm::StringLiteral timing_label, - llvm::function_refvoid> fn) -> void; - - // Returns true if the current file should be included in debug dumps. - auto IncludeInDumps() -> bool; - - // Builds the LLVM target machine. - auto MakeTargetMachine(const clang::CompilerInvocation& clang_invocation) - -> void; - - // The index of the unit amongst all units. - SemIR::CheckIRId check_ir_id_; - // The number of units in total. - int total_ir_count_; - - DriverEnv* driver_env_; - const CompileOptions* options_; - const llvm::Target* target_; - - SharedValueStores value_stores_; - - // The input filename from the command line. For most diagnostics, we - // typically use `source_->filename()`, which includes a `-` -> `` - // translation. However, logging and some diagnostics use the command line - // argument. - std::string input_filename_; - - // Copied from driver_ for CARBON_VLOG. - llvm::raw_pwrite_stream* vlog_stream_; - - // Diagnostics are sent to consumer_, with optional sorting. - std::optional sorting_consumer_; - Diagnostics::Consumer* consumer_; - - bool success_ = true; - - // Initialized by `SetMultiUnitCache`. - MultiUnitCache* cache_ = nullptr; - // Tracks memory usage of the compile. - std::optional mem_usage_; - // Tracks timings of the compile. - std::optional timings_; - - // These are initialized as steps are run. - std::optional source_; - std::optional tokens_; - std::optional parse_tree_; - std::optional parse_tree_and_subtrees_; - std::optionalconst Parse::TreeAndSubtrees&>> - tree_and_subtrees_getter_; - std::unique_ptr llvm_context_; - std::optional sem_ir_; - std::unique_ptr module_; - std::unique_ptr target_machine_; -}; - -// Caches lists that are shared cross-unit. Accessors do lazy caching because -// they may not be used. -class MultiUnitCache { - public: - using IncludeInDumpsStore = FixedSizeValueStore; - using TreeAndSubtreesGettersStore = Parse::GetTreeAndSubtreesStore; - - // This relies on construction after `units` are all initialized, which is - // reflected by the `ArrayRef` here. - explicit MultiUnitCache( - const CompileOptions* options, - llvm::ArrayRef> units) - : options_(options), units_(units) {} - - // If `include_in_dumps` is in use, we need to apply per-file include - // settings. - auto ApplyPerFileIncludeInDumps() -> void { - if (!include_in_dumps_) { - // No cached value to update. - return; - } - for (const auto& [i, unit] : llvm::enumerate(units_)) { - if (unit->has_include_in_dumps()) { - include_in_dumps_->Set(SemIR::CheckIRId(i), true); - } - } - } - - auto include_in_dumps() -> const IncludeInDumpsStore& { - if (!include_in_dumps_) { - include_in_dumps_.emplace( - IncludeInDumpsStore::MakeWithExplicitSize(units_.size(), false)); - for (const auto& [i, unit] : llvm::enumerate(units_)) { - // If this is first accessed after lexing is complete, we need to apply - // per-file includes. Otherwise, this is based only on the exclude - // option. - bool include = - unit->has_include_in_dumps() || - llvm::none_of(options_->exclude_dump_file_prefixes, - [&](auto prefix) { - return unit->input_filename().starts_with(prefix); - }); - include_in_dumps_->Set(SemIR::CheckIRId(i), include); - } - } - return *include_in_dumps_; - } - - auto tree_and_subtrees_getters() -> const TreeAndSubtreesGettersStore& { - if (!tree_and_subtrees_getters_) { - tree_and_subtrees_getters_.emplace( - TreeAndSubtreesGettersStore::MakeWithExplicitSize(units_.size(), - nullptr)); - for (const auto& [i, unit] : llvm::enumerate(units_)) { - if (unit->has_source()) { - tree_and_subtrees_getters_->Set(SemIR::CheckIRId(i), - unit->get_trees_and_subtrees()); - } - } - } - return *tree_and_subtrees_getters_; - } - - private: - const CompileOptions* options_; - - // The units being compiled. - llvm::ArrayRef> units_; - - // For each unit, whether it's included in dumps. Used cross-phase. - std::optional include_in_dumps_; - - // For each unit, the `TreeAndSubtrees` getter. Used by lowering. - std::optional tree_and_subtrees_getters_; -}; - -} // namespace - -CompilationUnit::CompilationUnit(SemIR::CheckIRId check_ir_id, - int total_ir_count, DriverEnv* driver_env, - const CompileOptions* options, - Diagnostics::Consumer* consumer, - llvm::StringRef input_filename, - const llvm::Target* target) - : check_ir_id_(check_ir_id), - total_ir_count_(total_ir_count), - driver_env_(driver_env), - options_(options), - target_(target), - input_filename_(input_filename), - vlog_stream_(driver_env_->vlog_stream) { - if (vlog_stream_ != nullptr || options_->stream_errors) { - consumer_ = consumer; - } else { - sorting_consumer_ = Diagnostics::SortingConsumer(*consumer); - consumer_ = &*sorting_consumer_; - } -} - -auto CompilationUnit::IncludeInDumps() -> bool { - return cache_->include_in_dumps().Get(check_ir_id_); -} - -auto CompilationUnit::SetMultiUnitCache(MultiUnitCache* cache) -> void { - CARBON_CHECK(!cache_, "Called SetMultiUnitCache twice"); - cache_ = cache; - - if (options_->dump_mem_usage && IncludeInDumps()) { - CARBON_CHECK(!mem_usage_); - mem_usage_ = MemUsage(); - } - if (options_->dump_timings && IncludeInDumps()) { - CARBON_CHECK(!timings_); - timings_ = Timings(); - } -} - -auto CompilationUnit::RunLex() -> void { - CARBON_CHECK(cache_, "Must call SetMultiUnitCache first"); - CARBON_CHECK(!tokens_, "Called RunLex twice"); - - LogCall("SourceBuffer::MakeFromFileOrStdin", "source", [&] { - source_ = SourceBuffer::MakeFromFileOrStdin(*driver_env_->fs, - input_filename_, *consumer_); - }); - - if (!source_) { - success_ = false; - return; - } - - if (mem_usage_) { - mem_usage_->Add("source_", source_->text().size(), source_->text().size()); - } - - CARBON_VLOG("*** SourceBuffer ***\n```\n{0}\n```\n", source_->text()); - - LogCall("Lex::Lex", "lex", [&] { - Lex::LexOptions options; - options.consumer = consumer_; - options.vlog_stream = vlog_stream_; - if (options_->dump_tokens && IncludeInDumps()) { - options.dump_stream = driver_env_->output_stream; - options.omit_file_boundary_tokens = options_->omit_file_boundary_tokens; - } - tokens_ = Lex::Lex(value_stores_, *source_, options); - }); - if (mem_usage_) { - mem_usage_->Collect("tokens_", *tokens_); - } - if (tokens_->has_errors()) { - success_ = false; - } -} - -auto CompilationUnit::RunParse() -> void { - LogCall("Parse::Parse", "parse", [&] { - Parse::ParseOptions options; - options.consumer = consumer_; - options.vlog_stream = vlog_stream_; - if (options_->dump_parse_tree && IncludeInDumps()) { - options.dump_stream = driver_env_->output_stream; - options.dump_preorder_parse_tree = options_->preorder_parse_tree; - } - parse_tree_ = Parse::Parse(*tokens_, options); - }); - if (mem_usage_) { - mem_usage_->Collect("parse_tree_", *parse_tree_); - } - if (parse_tree_->has_errors()) { - success_ = false; - } -} - -auto CompilationUnit::GetCheckUnit() -> Check::Unit { - CARBON_CHECK(parse_tree_, "Must call RunParse first"); - CARBON_CHECK(!sem_ir_, "Called GetCheckUnit twice"); - - tree_and_subtrees_getter_ = [this]() -> const Parse::TreeAndSubtrees& { - return this->GetParseTreeAndSubtrees(); - }; - sem_ir_.emplace(&*parse_tree_, check_ir_id_, parse_tree_->packaging_decl(), - value_stores_, input_filename_); - if (!llvm_context_) { - llvm_context_ = std::make_unique(); - } - return {.consumer = consumer_, - .value_stores = &value_stores_, - .timings = timings_ ? &*timings_ : nullptr, - .sem_ir = &*sem_ir_, - .llvm_context = llvm_context_.get(), - .total_ir_count = total_ir_count_}; -} - -auto CompilationUnit::PostCheck() -> void { - CARBON_CHECK(sem_ir_, "Must call GetCheckUnit first"); - - // We've finished all steps that can produce diagnostics. Emit the - // diagnostics now, so that the developer sees them sooner and doesn't need - // to wait for code generation. - consumer_->Flush(); - - if (mem_usage_) { - mem_usage_->Collect("sem_ir_", *sem_ir_); - } - - if (sem_ir_->has_errors()) { - success_ = false; - } -} - -auto CompilationUnit::RunLower() -> void { - LogCall("Lower::LowerToLLVM", "lower", [&] { - if (!llvm_context_) { - llvm_context_ = std::make_unique(); - } - Lower::LowerToLLVMOptions options; - options.llvm_verifier_stream = - options_->run_llvm_verifier ? driver_env_->error_stream : nullptr; - options.want_debug_info = options_->include_debug_info; - options.vlog_stream = vlog_stream_; - options.opt_level = options_->opt_level; - options.mangle_string_fingerprint = options_->mangle_string_fingerprint; - module_ = Lower::LowerToLLVM(*llvm_context_, driver_env_->fs, - cache_->tree_and_subtrees_getters(), *sem_ir_, - total_ir_count_, options); - }); -} - -auto CompilationUnit::MakeTargetMachine( - const clang::CompilerInvocation& clang_invocation) -> void { - CARBON_CHECK(module_, "Must call RunLower first"); - CARBON_CHECK(!target_machine_, "Should not call this multiple times"); - - // Set the target on the module. - // TODO: We should do this earlier. Lower should be passed the target triple - // so it can create the module with this already set. - llvm::Triple target_triple(options_->codegen_options.target); - module_->setTargetTriple(target_triple); - - // TODO: Provide flags to control these. - constexpr llvm::StringLiteral CPU = "generic"; - constexpr llvm::StringLiteral Features = ""; - - const auto& codegen_opts = clang_invocation.getCodeGenOpts(); - - // TODO: Make the code in Clang's BackendUtil.cpp externally accessible and - // call it from here. This is doing a subset of the same work to translate - // Clang code generation options into target options. - llvm::TargetOptions target_opts; - target_opts.UseInitArray = codegen_opts.UseInitArray; - target_opts.FunctionSections = codegen_opts.FunctionSections; - target_opts.DataSections = codegen_opts.DataSections; - target_opts.UniqueSectionNames = codegen_opts.UniqueSectionNames; - target_machine_.reset(target_->createTargetMachine( - target_triple, CPU, Features, target_opts, llvm::Reloc::PIC_)); -} - -// Get the LLVM optimization level corresponding to a Carbon optimization level. -static auto GetLLVMOptimizationLevel(Lower::OptimizationLevel opt_level) - -> llvm::OptimizationLevel { - switch (opt_level) { - case Lower::OptimizationLevel::None: - return llvm::OptimizationLevel::O0; - case Lower::OptimizationLevel::Debug: - return llvm::OptimizationLevel::O1; - case Lower::OptimizationLevel::Size: - return llvm::OptimizationLevel::O2; - case Lower::OptimizationLevel::Speed: - return llvm::OptimizationLevel::O3; - } -} - -// Get the `-O` flag corresponding to an optimization level. -static auto GetClangOptimizationFlag(Lower::OptimizationLevel opt_level) - -> llvm::StringLiteral { - switch (opt_level) { - case Lower::OptimizationLevel::None: - return "-O0"; - case Lower::OptimizationLevel::Debug: - return "-O1"; - case Lower::OptimizationLevel::Size: - return "-O2"; - case Lower::OptimizationLevel::Speed: - return "-O3"; - } -} - -auto CompilationUnit::RunOptimize( - const clang::CompilerInvocation& clang_invocation) -> void { - CARBON_CHECK(module_, "Must call RunLower first"); - - // TODO: A lot of the work done here duplicates work done by Clang setting up - // its pass manager. Moreover, we probably want to pick up Clang's - // customizations and make use of its flags for controlling LLVM passes. We - // should consider whether we would be better off running Clang's pass - // pipeline rather than building one of our own, or factoring out enough of - // Clang's pipeline builder that we can reuse and further customize it. - - MakeTargetMachine(clang_invocation); - - // TODO: There's no way to set these automatically from an - // llvm::OptimizationLevel. Add such a mechanism to LLVM and use it from - // here. For now we reconstruct what Clang does by default. - llvm::PipelineTuningOptions pto; - bool opt_for_speed = options_->opt_level == Lower::OptimizationLevel::Speed; - bool opt_for_size_or_speed = - opt_for_speed || options_->opt_level == Lower::OptimizationLevel::Size; - // Loop unrolling is enabled by `--optimize=size` but isn't actually performed - // because we add `optsize` attributes to the function definitions we emit. - pto.LoopUnrolling = opt_for_size_or_speed; - pto.LoopInterleaving = opt_for_size_or_speed; - pto.LoopVectorization = opt_for_speed; - pto.SLPVectorization = opt_for_size_or_speed; - - llvm::LoopAnalysisManager lam; - llvm::FunctionAnalysisManager fam; - llvm::CGSCCAnalysisManager cgam; - llvm::ModuleAnalysisManager mam; - - llvm::PassInstrumentationCallbacks pic; - - // Register standard pass instrumentations. This adds support for things like - // `-print-after-all`. - llvm::StandardInstrumentations si(module_->getContext(), - /*DebugLogging=*/false); - si.registerCallbacks(pic); - - llvm::PassBuilder builder(target_machine_.get(), pto, - /*PGOOpt=*/std::nullopt, &pic); - - // TODO: Add an AssignmentTrackingPass for at least `--optimize=debug`. - - // Set up target library information and add an analysis pass to supply it. - std::unique_ptr tlii(llvm::driver::createTLII( - module_->getTargetTriple(), llvm::driver::VectorLibrary::NoLibrary)); - fam.registerPass([&] { return llvm::TargetLibraryAnalysis(*tlii); }); - - builder.registerModuleAnalyses(mam); - builder.registerCGSCCAnalyses(cgam); - builder.registerFunctionAnalyses(fam); - builder.registerLoopAnalyses(lam); - builder.crossRegisterProxies(lam, fam, cgam, mam); - - llvm::ModulePassManager pass_manager = builder.buildPerModuleDefaultPipeline( - GetLLVMOptimizationLevel(options_->opt_level)); - - if (vlog_stream_) { - CARBON_VLOG("*** Running pass pipeline: "); - pass_manager.printPipeline( - *vlog_stream_, [&pic](llvm::StringRef class_name) { - auto pass_name = pic.getPassNameForClassName(class_name); - return pass_name.empty() ? class_name : pass_name; - }); - CARBON_VLOG(" ***\n"); - } - - LogCall("ModulePassManager::run", "optimize", - [&] { pass_manager.run(*module_, mam); }); - - if (vlog_stream_) { - CARBON_VLOG("*** Optimized llvm::Module ***\n"); - module_->print(*vlog_stream_, /*AAW=*/nullptr, - /*ShouldPreserveUseListOrder=*/false, - /*IsForDebug=*/true); - } -} - -auto CompilationUnit::PostLower() -> void { - CARBON_CHECK(module_, "Must call RunLower first"); - if (options_->dump_llvm_ir && IncludeInDumps()) { - module_->print(*driver_env_->output_stream, /*AAW=*/nullptr, - /*ShouldPreserveUseListOrder=*/true); - } -} - -auto CompilationUnit::RunCodeGen() -> void { - CARBON_CHECK(module_, "Must call RunLower first"); - LogCall("CodeGen", "codegen", [&] { success_ = RunCodeGenHelper(); }); -} - -auto CompilationUnit::PostCompile() -> void { - if (options_->dump_shared_values && IncludeInDumps()) { - Yaml::Print(*driver_env_->output_stream, - value_stores_.OutputYaml(input_filename_)); - } - if (mem_usage_) { - mem_usage_->Collect("value_stores_", value_stores_); - Yaml::Print(*driver_env_->output_stream, - mem_usage_->OutputYaml(input_filename_)); - } - if (timings_) { - Yaml::Print(*driver_env_->output_stream, - timings_->OutputYaml(input_filename_)); - } - - // The diagnostics consumer must be flushed before compilation artifacts are - // destructed, because diagnostics can refer to their state. - consumer_->Flush(); -} - -auto CompilationUnit::RunCodeGenHelper() -> bool { - CARBON_CHECK(module_, "Must call RunLower first"); - CARBON_CHECK(target_machine_, "Must call MakeTargetMachine first"); - - CodeGen codegen(module_.get(), target_machine_.get(), consumer_); - if (vlog_stream_) { - CARBON_VLOG("*** Assembly ***\n"); - codegen.EmitAssembly(*vlog_stream_); - } - - if (options_->output_filename == "-") { - // TODO: The output file name, forcing object output, and requesting - // textual assembly output are all somewhat linked flags. We should add - // some validation that they are used correctly. - if (options_->force_obj_output) { - if (!codegen.EmitObject(*driver_env_->output_stream)) { - return false; - } - } else { - if (!codegen.EmitAssembly(*driver_env_->output_stream)) { - return false; - } - } - } else { - llvm::SmallString<256> output_filename = options_->output_filename; - if (output_filename.empty()) { - if (!source_->is_regular_file()) { - // Don't invent file names like `-.o` or `/dev/stdin.o`. - // TODO: Consider rephrasing the diagnostic to use the file as the - // `Emit` location. - CARBON_DIAGNOSTIC(CompileInputNotRegularFile, Error, - "output file name must be specified for input `{0}` " - "that is not a regular file", - std::string); - driver_env_->emitter.Emit(CompileInputNotRegularFile, input_filename_); - return false; - } - output_filename = input_filename_; - llvm::sys::path::replace_extension(output_filename, - options_->asm_output ? ".s" : ".o"); - } - CARBON_VLOG("Writing output to: {0}\n", output_filename); - - std::error_code ec; - llvm::raw_fd_ostream output_file(output_filename, ec, - llvm::sys::fs::OF_None); - if (ec) { - // TODO: Consider rephrasing the diagnostic to use the file as the `Emit` - // location. - CARBON_DIAGNOSTIC(CompileOutputFileOpenError, Error, - "could not open output file `{0}`: {1}", std::string, - std::string); - driver_env_->emitter.Emit(CompileOutputFileOpenError, - output_filename.str().str(), ec.message()); - return false; - } - if (options_->asm_output) { - if (!codegen.EmitAssembly(output_file)) { - return false; - } - } else { - if (!codegen.EmitObject(output_file)) { - return false; - } - } - } - return true; -} - -auto CompilationUnit::GetParseTreeAndSubtrees() - -> const Parse::TreeAndSubtrees& { - if (!parse_tree_and_subtrees_) { - parse_tree_and_subtrees_ = Parse::TreeAndSubtrees(*tokens_, *parse_tree_); - if (mem_usage_) { - mem_usage_->Collect("parse_tree_and_subtrees_", - *parse_tree_and_subtrees_); - } - } - return *parse_tree_and_subtrees_; -} - -auto CompilationUnit::LogCall(llvm::StringLiteral logging_label, - llvm::StringLiteral timing_label, - llvm::function_refvoid> fn) -> void { - PrettyStackTraceFunction trace_file([&](llvm::raw_ostream& out) { - out << "Filename: " << input_filename_ << "\n"; - }); - CARBON_VLOG("*** {0}: {1} ***\n", logging_label, input_filename_); - Timings::ScopedTiming timing(timings_ ? &*timings_ : nullptr, timing_label); - fn(); - CARBON_VLOG("*** {0} done ***\n", logging_label); -} - auto CompileSubcommand::Run(DriverEnv& driver_env) -> DriverResult { - if (!ValidateOptions(driver_env.emitter)) { + if (driver_env.fuzzing && !options_.clang_args.empty()) { + // Parsing specific Clang arguments can reach deep into + // external libraries that aren't fuzz clean. + TestAndDiagnoseIfFuzzingExternalLibraries(driver_env, "compile"); return {.success = false}; } - // Validate the target before passing it to Clang. - std::string target_error; - const llvm::Target* target = llvm::TargetRegistry::lookupTarget( - llvm::Triple(options_.codegen_options.target), target_error); - if (!target) { - CARBON_DIAGNOSTIC(CompileTargetInvalid, Error, "invalid target: {0}", - std::string); - driver_env.emitter.Emit(CompileTargetInvalid, target_error); + auto compile_driver = CompileDriver(&options_); + + if (!compile_driver.Initialize(driver_env, + [&](llvm::StringRef) -> std::string { + return options_.output_filename.str(); + })) { return {.success = false}; } - std::shared_ptr clang_invocation; - // Build a clang invocation. We do this regardless of whether we're running - // check, because this is essentially performing further option validation, - // and we generally validate all options even if we're not using them for the - // selected phases of compilation. We also use Clang's target option handling - // to configure our target, to ensure that we are using the same ABI for both - // the C++ and Carbon parts of the compilation. - // TODO: Share any arguments we specify here with the `carbon clang` - // subcommand. - { - if (driver_env.fuzzing && !options_.clang_args.empty()) { - // Parsing specific Clang arguments can reach deep into - // external libraries that aren't fuzz clean. - TestAndDiagnoseIfFuzzingExternalLibraries(driver_env, "compile"); - return {.success = false}; - } - - // TODO: Move this into `BuildClangInvocation` when it can accept an - // optimization level. - llvm::SmallVector clang_args = { - // Propagate our optimization level to Clang as a default. This can be - // overridden by Clang arguments, but doing so will only have an effect - // if those arguments affect Clang's IR, not its pass pipeline. - GetClangOptimizationFlag(options_.opt_level), - }; - clang_args.append(options_.clang_args); - clang_invocation = BuildClangInvocation( - driver_env.consumer, driver_env.fs, *driver_env.installation, - options_.codegen_options.target, clang_args); - if (!clang_invocation) { - return {.success = false}; - } - // We will run our own pass pipeline over the IR in the `Optimize` phase, so - // disable Clang's pipeline to avoid optimizing C++ code twice. - clang_invocation->getCodeGenOpts().DisableLLVMPasses = true; - } - - // Find the files comprising the prelude if we are importing it. - // TODO: Replace this with a search for library api files in a - // package-specific search path based on the library name. - llvm::SmallVector prelude; - if (options_.prelude_import && !options_.custom_core && - options_.phase >= CompileOptions::Phase::Check) { - if (auto find = driver_env.installation->ReadPreludeManifest(); find.ok()) { - prelude = std::move(*find); - } else { - // TODO: Change ReadPreludeManifest to produce diagnostics. - CARBON_DIAGNOSTIC(CompilePreludeManifestError, Error, "{0}", std::string); - driver_env.emitter.Emit(CompilePreludeManifestError, - PrintToString(find.error())); - return {.success = false}; - } - } - - // Prepare CompilationUnits before building scope exit handlers. - llvm::SmallVector> units; - int unit_index = -1; - int total_unit_count = prelude.size() + options_.input_filenames.size(); - auto unit_builder = [&](llvm::StringRef filename) { - ++unit_index; - return std::make_unique( - SemIR::CheckIRId(unit_index), total_unit_count, &driver_env, &options_, - &driver_env.consumer, filename, target); - }; - llvm::append_range(units, llvm::map_range(prelude, unit_builder)); - llvm::append_range(units, - llvm::map_range(options_.input_filenames, unit_builder)); - CARBON_CHECK(units.size() == static_cast(total_unit_count)); - - // Add the cache to all units. This must be done after all units are created. - MultiUnitCache cache(&options_, units); - for (auto& unit : units) { - unit->SetMultiUnitCache(&cache); - } - - auto on_exit = llvm::scope_exit([&]() { - // Finish compilation units. This flushes their diagnostics in the order in - // which they were specified on the command line. - for (auto& unit : units) { - unit->PostCompile(); - } - - driver_env.consumer.Flush(); - }); - - PrettyStackTraceFunction flush_on_crash([&](llvm::raw_ostream& out) { - // When crashing, flush diagnostics. If sorting diagnostics, they can be - // redirected to the crash stream; if streaming, the original stream is - // flushed. - // TODO: Eventually we'll want to limit the count. - if (options_.stream_errors) { - out << "Flushing diagnostics\n"; - } else { - out << "Pending diagnostics:\n"; - driver_env.consumer.set_stream(&out); - } - - for (auto& unit : units) { - unit->FlushForStackTrace(); - } - driver_env.consumer.Flush(); - driver_env.consumer.set_stream(driver_env.error_stream); - }); - - // Returns a DriverResult object. Called whenever Compile returns. - auto make_result = [&]() { - DriverResult result = {.success = true}; - for (const auto& unit : units) { - result.success &= unit->success(); - result.per_file_success.push_back( - {unit->input_filename().str(), unit->success()}); - } - return result; - }; - - // Lex. - for (auto& unit : units) { - unit->RunLex(); - } - if (options_.phase == CompileOptions::Phase::Lex) { - return make_result(); - } - cache.ApplyPerFileIncludeInDumps(); - // Parse and check phases examine `has_source` because they want to proceed if - // lex failed, but not if source doesn't exist. Later steps are skipped if - // anything failed, so don't need this. - - // Parse. - for (auto& unit : units) { - if (unit->has_source()) { - unit->RunParse(); - } - } - if (options_.phase == CompileOptions::Phase::Parse) { - return make_result(); - } - - // Gather Check::Units. - llvm::SmallVector check_units; - check_units.reserve(units.size()); - for (auto& unit : units) { - if (unit->has_source()) { - check_units.push_back(unit->GetCheckUnit()); - } - } - - // Execute the actual checking. - CARBON_VLOG_TO(driver_env.vlog_stream, "*** Check::CheckParseTrees ***\n"); - Check::CheckParseTreesOptions options; - options.prelude_import = options_.prelude_import; - options.vlog_stream = driver_env.vlog_stream; - options.fuzzing = driver_env.fuzzing; - options.mangle_string_fingerprint = options_.mangle_string_fingerprint; - if (options.vlog_stream || options_.dump_sem_ir || options_.dump_cpp_ast || - options_.dump_raw_sem_ir) { - options.include_in_dumps = &cache.include_in_dumps(); - if (options_.dump_sem_ir) { - options.dump_stream = driver_env.output_stream; - } - if (options_.dump_cpp_ast) { - options.dump_cpp_ast_stream = driver_env.output_stream; - } - if (options.vlog_stream || options_.dump_sem_ir) { - options.dump_sem_ir_ranges = options_.dump_sem_ir_ranges; - } - if (options_.dump_raw_sem_ir) { - options.raw_dump_stream = driver_env.output_stream; - options.dump_raw_sem_ir_builtins = options_.builtin_sem_ir; - } - options.sem_ir_crash_dump = options_.sem_ir_crash_dump; - } - Check::CheckParseTrees(check_units, cache.tree_and_subtrees_getters(), - driver_env.fs, options, clang_invocation); - CARBON_VLOG_TO(driver_env.vlog_stream, - "*** Check::CheckParseTrees done ***\n"); - for (auto& unit : units) { - if (unit->has_source()) { - unit->PostCheck(); - } - } - if (options_.phase == CompileOptions::Phase::Check) { - return make_result(); - } - - // Unlike previous steps, errors block further progress. - if (llvm::any_of(units, [&](const auto& unit) { return !unit->success(); })) { - CARBON_VLOG_TO(driver_env.vlog_stream, - "*** Stopping before lowering due to errors ***\n"); - return make_result(); - } - - // Lower and optimize. - for (const auto& unit : units) { - unit->RunLower(); - - if (options_.phase != CompileOptions::Phase::Lower) { - unit->RunOptimize(*clang_invocation); - } - - unit->PostLower(); - } - if (options_.phase == CompileOptions::Phase::Lower || - options_.phase == CompileOptions::Phase::Optimize) { - return make_result(); - } - CARBON_CHECK(options_.phase == CompileOptions::Phase::CodeGen, - "CodeGen should be the last stage"); - - bool output_last_input_only = options_.output_last_input_only; - if (!output_last_input_only && units.size() > 1 && - !options_.output_filename.empty() && options_.output_filename != "-") { - // TODO: Command line structure should change to make this implicit (passing - // non-compiling inputs differently), and the warning should be removed. - CARBON_DIAGNOSTIC( - CompileMultipleInputsWithOutput, Warning, - "only outputting {0} to {1}, skipping output of {2} input " - "file{2:s}; pass `--output-last-input-only` to silence this warning", - std::string, std::string, Diagnostics::IntAsSelect); - driver_env.emitter.Emit(CompileMultipleInputsWithOutput, - units.back()->input_filename().str(), - options_.output_filename.str(), units.size() - 1); - output_last_input_only = true; - } - - // Codegen. - if (output_last_input_only) { - units.back()->RunCodeGen(); - } else { - for (const auto& unit : units) { - unit->RunCodeGen(); - } - } - return make_result(); + return compile_driver.Compile(driver_env); } } // namespace Carbon diff --git a/toolchain/driver/compile_subcommand.h b/toolchain/driver/compile_subcommand.h index c1042c990e4d..50f1260bbd9a 100644 --- a/toolchain/driver/compile_subcommand.h +++ b/toolchain/driver/compile_subcommand.h @@ -6,92 +6,24 @@ #define CARBON_TOOLCHAIN_DRIVER_COMPILE_SUBCOMMAND_H_ #include "common/command_line.h" -#include "common/error.h" -#include "common/ostream.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/StringRef.h" -#include "toolchain/check/check.h" -#include "toolchain/driver/codegen_options.h" +#include "toolchain/driver/compile_options.h" #include "toolchain/driver/driver_env.h" #include "toolchain/driver/driver_subcommand.h" -#include "toolchain/lower/options.h" namespace Carbon { -// Options for the compile subcommand. -// -// See the implementation of `Build` for documentation on members. -struct CompileOptions { - enum class Phase : int8_t { - Lex, - Parse, - Check, - Lower, - Optimize, - CodeGen, - }; - - friend auto operator<<(llvm::raw_ostream& out, Phase phase) - -> llvm::raw_ostream&; - - auto Build(CommandLine::CommandBuilder& b) -> void; - - Lower::OptimizationLevel opt_level = Lower::OptimizationLevel::Debug; - CodegenOptions codegen_options; - - Phase phase; - Check::CheckParseTreesOptions::DumpSemIRRanges dump_sem_ir_ranges; - - llvm::StringRef output_filename; - llvm::SmallVector input_filenames; - llvm::SmallVector clang_args; - - bool asm_output = false; - bool force_obj_output = false; - bool custom_core = false; - bool dump_shared_values = false; - bool dump_tokens = false; - bool omit_file_boundary_tokens = false; - bool dump_parse_tree = false; - bool dump_raw_sem_ir = false; - bool dump_sem_ir = false; - bool dump_cpp_ast = false; - bool dump_llvm_ir = false; - bool dump_asm = false; - bool dump_mem_usage = false; - bool dump_timings = false; - bool stream_errors = false; - bool preorder_parse_tree = false; - bool builtin_sem_ir = false; - bool prelude_import = false; - bool include_debug_info = true; - bool output_last_input_only = false; - bool run_llvm_verifier = true; - - llvm::SmallVector exclude_dump_file_prefixes; - - llvm::StringRef sem_ir_crash_dump; - - bool mangle_string_fingerprint = false; -}; - // Implements the compile subcommand of the driver. class CompileSubcommand : public DriverSubcommand { public: explicit CompileSubcommand(); auto BuildOptions(CommandLine::CommandBuilder& b) -> void override { - options_.Build(b); + options_.BuildForCompileSubcommand(b); } auto Run(DriverEnv& driver_env) -> DriverResult override; private: - // Does custom validation of the compile-subcommand options structure beyond - // what the command line parsing library supports. Diagnoses and returns false - // on failure. - auto ValidateOptions(Diagnostics::NoLocEmitter& emitter) const -> bool; - CompileOptions options_; }; diff --git a/toolchain/driver/driver.cpp b/toolchain/driver/driver.cpp index 16e348e2399d..f47df68a711e 100644 --- a/toolchain/driver/driver.cpp +++ b/toolchain/driver/driver.cpp @@ -13,6 +13,7 @@ #include "common/pretty_stack_trace_function.h" #include "common/version.h" #include "toolchain/driver/build_runtimes_subcommand.h" +#include "toolchain/driver/build_subcommand.h" #include "toolchain/driver/clang_subcommand.h" #include "toolchain/driver/compile_subcommand.h" #include "toolchain/driver/config_subcommand.h" @@ -40,6 +41,7 @@ struct Options { llvm::StringRef prebuilt_runtimes_path; BuildRuntimesSubcommand runtimes; + BuildSubcommand build; ClangSubcommand clang; CompileSubcommand compile; ConfigSubcommand config; @@ -169,6 +171,7 @@ when there are errors or other output. }); runtimes.AddTo(b, &selected_subcommand); + build.AddTo(b, &selected_subcommand); clang.AddTo(b, &selected_subcommand); compile.AddTo(b, &selected_subcommand); config.AddTo(b, &selected_subcommand); diff --git a/toolchain/driver/driver_test.cpp b/toolchain/driver/driver_test.cpp index 94d72eea8cb9..32cd8a18b68c 100644 --- a/toolchain/driver/driver_test.cpp +++ b/toolchain/driver/driver_test.cpp @@ -208,7 +208,7 @@ TEST_F(DriverTest, StdoutOutput) { EXPECT_TRUE(result->get()->isObject()); } -TEST_F(DriverTest, FileOutput) { +TEST_F(DriverTest, LinkFileOutput) { auto scope = ScopedTempWorkingDir(); // Use explicit filenames as the default output filename is computed from @@ -336,5 +336,35 @@ TEST_F(DriverTest, ConfigJson) { EXPECT_THAT(Filesystem::Cwd().OpenDir(clang_sysroot->str()), IsSuccess(_)); } +TEST_F(DriverTest, BuildFileOutput) { + auto scope = ScopedTempWorkingDir(); + + MakeTestFile(R"""( +import Core library "io"; + +fn Run() { + Core.PrintStr("Hello world!\n"); +} +)""", + "hello_world.carbon"); + + // File should compile to a `hello_world` binary without error. + EXPECT_TRUE( + driver_ + .RunCommand({"--no-build-runtimes", "build", "--no-use-temp-dir", + "hello_world.carbon", "--", "--", "-lc"}) + .success); + EXPECT_THAT(test_error_stream_.TakeStr(), StrEq("")); + + // Binary should read as valid to LLVM. + auto result = llvm::object::createBinary("hello_world"); + if (auto error = result.takeError()) { + FAIL() << toString(std::move(error)); + } + + // Executables are also classified as object files. + EXPECT_TRUE(result->getBinary()->isObject()); +} + } // namespace } // namespace Carbon diff --git a/toolchain/driver/link_driver.cpp b/toolchain/driver/link_driver.cpp new file mode 100644 index 000000000000..97cb1f027020 --- /dev/null +++ b/toolchain/driver/link_driver.cpp @@ -0,0 +1,85 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "toolchain/driver/link_driver.h" + +namespace Carbon { + +LinkDriver::LinkDriver(LinkOptions* options) : options_(options) {} + +auto LinkDriver::Link(DriverEnv& driver_env) -> DriverResult { + // TODO: Currently we use the Clang driver to link. This works well on Unix + // OSes but we likely need to directly build logic to invoke `link.exe` on + // Windows where `cl.exe` doesn't typically cover that logic. + + // Use a reasonably large small vector here to minimize allocations. We expect + // to link reasonably large numbers of object files. + llvm::SmallVector clang_args; + + // We link using a C++ mode of the driver. + clang_args.push_back("--driver-mode=g++"); + + // Pass the target down to Clang to pick up the correct defaults. + std::string target_arg = + llvm::formatv("--target={0}", options_->codegen_options->target).str(); + clang_args.push_back(target_arg); + + if (!options_->output_filename.empty()) { + clang_args.push_back("-o"); + clang_args.push_back(options_->output_filename); + } else if (options_->extra_clang_args.empty()) { + CARBON_DIAGNOSTIC(LinkOutputOptionMissing, Error, + "no output specified to a link command and no extra " + "Clang options that can provide an output"); + driver_env.emitter.Emit(LinkOutputOptionMissing); + return {.success = false}; + } + + if (options_->object_filenames.empty() && + options_->extra_clang_args.empty()) { + CARBON_DIAGNOSTIC(LinkObjectFilesMissing, Error, + "no object files provided to link command and no extra " + "Clang options that could provide them"); + driver_env.emitter.Emit(LinkObjectFilesMissing); + return {.success = false}; + } + + // Note that we append any extra Clang args before our object filenames. This + // allows us to propagate object filenames that collide with Clang flags using + // `--` before the filenames. While in theory, this could create a problem in + // the presence of mixtures of object files in the two lists and the order + // being dependent, we don't expect that in practice. + clang_args.append(options_->extra_clang_args.begin(), + options_->extra_clang_args.end()); + clang_args.push_back("--"); + clang_args.append(options_->object_filenames.begin(), + options_->object_filenames.end()); + + ClangRunner runner(driver_env.installation, driver_env.fs, + driver_env.vlog_stream); + ErrorOr run_result = + driver_env.prebuilt_runtimes + ? runner.RunWithPrebuiltRuntimes(clang_args, + *driver_env.prebuilt_runtimes, + driver_env.enable_leaking) + : driver_env.build_runtimes_on_demand + ? runner.Run(clang_args, driver_env.runtimes_cache, + *driver_env.thread_pool, driver_env.enable_leaking) + : runner.RunWithNoRuntimes(clang_args, driver_env.enable_leaking); + + if (!run_result.ok()) { + // This is not a Clang failure, but a failure to even run Clang, so we need + // to diagnose it here. + CARBON_DIAGNOSTIC(FailureRunningClangToLink, Error, + "failure running `clang` to perform linking: {0}", + std::string); + driver_env.emitter.Emit(FailureRunningClangToLink, + run_result.error().message()); + return {.success = false}; + } + // Successfully ran Clang to perform the link, return its result. + return {.success = *run_result}; +} + +} // namespace Carbon diff --git a/toolchain/driver/link_driver.h b/toolchain/driver/link_driver.h new file mode 100644 index 000000000000..e2dd24e1172c --- /dev/null +++ b/toolchain/driver/link_driver.h @@ -0,0 +1,31 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef CARBON_TOOLCHAIN_DRIVER_LINK_DRIVER_H_ +#define CARBON_TOOLCHAIN_DRIVER_LINK_DRIVER_H_ + +#include "llvm/TargetParser/Triple.h" +#include "toolchain/driver/clang_runner.h" +#include "toolchain/driver/driver_env.h" +#include "toolchain/driver/link_options.h" + +namespace Carbon { + +// Helper class to link object files into an output binary using `clang`. Used +// by the `build` and `link` subcommands. +class LinkDriver { + public: + explicit LinkDriver(LinkOptions* options); + + // Link the input binaries to the output binary with the configuration + // specified in the `LinkOptions` provided at construction time. + auto Link(DriverEnv& driver_env) -> DriverResult; + + private: + LinkOptions* options_; +}; + +} // namespace Carbon + +#endif // CARBON_TOOLCHAIN_DRIVER_LINK_DRIVER_H_ diff --git a/toolchain/driver/link_options.cpp b/toolchain/driver/link_options.cpp new file mode 100644 index 000000000000..49265b12880d --- /dev/null +++ b/toolchain/driver/link_options.cpp @@ -0,0 +1,82 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "toolchain/driver/link_options.h" + +namespace Carbon { + +namespace { + +auto BuildSharedOptions(CommandLine::CommandBuilder& b, LinkOptions* options) + -> void { + b.AddStringPositionalArg( + { + .name = "EXTRA_CLANG_LINK_ARGS", + .help = R"""( +Extra arguments to pass to Clang when forming the link command. This is +primarily useful for expanding `LDFLAGS` or other baseline linking flags in a +build system. + +These can also be used to pass object files to the link in the event your build +system mixes object files and linker flags. +)""", + }, + [&](auto& arg_b) { arg_b.Append(&options->extra_clang_args); }); +} + +} // namespace + +auto LinkOptions::BuildForLinkSubcommand(CommandLine::CommandBuilder& b) + -> void { + b.AddStringPositionalArg( + { + .name = "OBJECT_FILE", + .help = R"""( +The input object files. + +If empty, there must be extra Clang link arguments that provide object files +intermingled with linking flags. +)""", + }, + [&](auto& arg_b) { arg_b.Append(&object_filenames); }); + + b.AddStringOption( + { + .name = "output", + .value_name = "FILE", + .help = R"""( +The linked file name. The output is always a linked binary. + +If not provided, there must be extra Clang link arguments that include +specifying the output of the link. This allows supporting build systems that +intermingle the output flag with arbitrary other linker flags that need to use +legacy parsing logic. +)""", + }, + [&](auto& arg_b) { arg_b.Set(&output_filename); }); + + BuildSharedOptions(b, this); + + codegen_options = std::make_shared(); + codegen_options->Build(b); +} + +auto LinkOptions::BuildForBuildSubcommand(CommandLine::CommandBuilder& b) + -> void { + b.AddStringOption( + { + .name = "output", + .short_name = "o", + .value_name = "FILE", + .help = R"""( +The file name for the output binary. If none is specified, `build` will use the +name of the first provided input file. +)""", + }, + [&](auto& arg_b) { arg_b.Set(&output_filename); }); + + BuildSharedOptions(b, this); +} + +} // namespace Carbon diff --git a/toolchain/driver/link_options.h b/toolchain/driver/link_options.h new file mode 100644 index 000000000000..0c442fce618e --- /dev/null +++ b/toolchain/driver/link_options.h @@ -0,0 +1,35 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef CARBON_TOOLCHAIN_DRIVER_LINK_OPTIONS_H_ +#define CARBON_TOOLCHAIN_DRIVER_LINK_OPTIONS_H_ + +#include + +#include "common/command_line.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "toolchain/driver/codegen_options.h" +#include "toolchain/driver/driver_env.h" +#include "toolchain/driver/driver_subcommand.h" + +namespace Carbon { + +// Options for the link subcommand. +// +// See the implementation of `link` for documentation on members. +struct LinkOptions { + auto BuildForLinkSubcommand(CommandLine::CommandBuilder& b) -> void; + auto BuildForBuildSubcommand(CommandLine::CommandBuilder& b) -> void; + + std::shared_ptr codegen_options; + llvm::StringRef output_filename; + llvm::SmallVector object_filenames; + + llvm::SmallVector extra_clang_args; +}; + +} // namespace Carbon + +#endif // CARBON_TOOLCHAIN_DRIVER_LINK_OPTIONS_H_ diff --git a/toolchain/driver/link_subcommand.cpp b/toolchain/driver/link_subcommand.cpp index 99f4a4d5ef36..85badf19f4f8 100644 --- a/toolchain/driver/link_subcommand.cpp +++ b/toolchain/driver/link_subcommand.cpp @@ -4,56 +4,10 @@ #include "toolchain/driver/link_subcommand.h" -#include "llvm/TargetParser/Triple.h" -#include "toolchain/driver/clang_runner.h" +#include "toolchain/driver/link_driver.h" namespace Carbon { -auto LinkOptions::Build(CommandLine::CommandBuilder& b) -> void { - b.AddStringPositionalArg( - { - .name = "OBJECT_FILE", - .help = R"""( -The input object files. - -If empty, there must be extra Clang link arguments that provide object files -intermingled with linking flags. -)""", - }, - [&](auto& arg_b) { arg_b.Append(&object_filenames); }); - - b.AddStringOption( - { - .name = "output", - .value_name = "FILE", - .help = R"""( -The linked file name. The output is always a linked binary. - -If not provided, there must be extra Clang link arguments that include -specifying the output of the link. This allows supporting build systems that -intermingle the output flag with arbitrary other linker flags that need to use -legacy parsing logic. -)""", - }, - [&](auto& arg_b) { arg_b.Set(&output_filename); }); - - b.AddStringPositionalArg( - { - .name = "EXTRA_CLANG_LINK_ARGS", - .help = R"""( -Extra arguments to pass to Clang when forming the link command. This is -primarily useful for expanding `LDFLAGS` or other baseline linking flags in a -build system. - -These can also be used to pass object files to the link in the event your build -system mixes object files and linker flags. -)""", - }, - [&](auto& arg_b) { arg_b.Append(&extra_clang_args); }); - - codegen_options.Build(b); -} - static constexpr CommandLine::CommandInfo SubcommandInfo = { .name = "link", .help = R"""( @@ -62,88 +16,20 @@ Link Carbon executables. This subcommand links Carbon executables by combining object files. TODO: Support linking binary libraries, both archives and shared libraries. -TODO: Support linking against binary libraries. )""", }; LinkSubcommand::LinkSubcommand() : DriverSubcommand(SubcommandInfo) {} auto LinkSubcommand::Run(DriverEnv& driver_env) -> DriverResult { - // TODO: Currently we use the Clang driver to link. This works well on Unix - // OSes but we likely need to directly build logic to invoke `link.exe` on - // Windows where `cl.exe` doesn't typically cover that logic. - - // Use a reasonably large small vector here to minimize allocations. We expect - // to link reasonably large numbers of object files. - llvm::SmallVector clang_args; - - // We link using a C++ mode of the driver. - clang_args.push_back("--driver-mode=g++"); - - // Pass the target down to Clang to pick up the correct defaults. - std::string target_arg = - llvm::formatv("--target={0}", options_.codegen_options.target).str(); - clang_args.push_back(target_arg); - - if (!options_.output_filename.empty()) { - clang_args.push_back("-o"); - clang_args.push_back(options_.output_filename); - } else if (options_.extra_clang_args.empty()) { - CARBON_DIAGNOSTIC(LinkOutputOptionMissing, Error, - "no output specified to a link command and no extra " - "Clang options that can provide an output"); - driver_env.emitter.Emit(LinkOutputOptionMissing); - return {.success = false}; - } - - if (options_.object_filenames.empty() && options_.extra_clang_args.empty()) { - CARBON_DIAGNOSTIC(LinkObjectFilesMissing, Error, - "no object files provided to link command and no extra " - "Clang options that could provide them"); - driver_env.emitter.Emit(LinkObjectFilesMissing); - return {.success = false}; - } - - // Note that we append any extra Clang args before our object filenames. This - // allows us to propagate object filenames that collide with Clang flags using - // `--` before the filenames. While in theory, this could create a problem in - // the presence of mixtures of object files in the two lists and the order - // being dependent, we don't expect that in practice. - clang_args.append(options_.extra_clang_args.begin(), - options_.extra_clang_args.end()); - clang_args.push_back("--"); - clang_args.append(options_.object_filenames.begin(), - options_.object_filenames.end()); - - ClangRunner runner(driver_env.installation, driver_env.fs, - driver_env.vlog_stream); // Don't run Clang when fuzzing, it is known to not be reliable under fuzzing // due to many unfixed issues. if (TestAndDiagnoseIfFuzzingExternalLibraries(driver_env, "clang")) { return {.success = false}; } - ErrorOr run_result = - driver_env.prebuilt_runtimes - ? runner.RunWithPrebuiltRuntimes(clang_args, - *driver_env.prebuilt_runtimes, - driver_env.enable_leaking) - : driver_env.build_runtimes_on_demand - ? runner.Run(clang_args, driver_env.runtimes_cache, - *driver_env.thread_pool, driver_env.enable_leaking) - : runner.RunWithNoRuntimes(clang_args, driver_env.enable_leaking); - if (!run_result.ok()) { - // This is not a Clang failure, but a failure to even run Clang, so we need - // to diagnose it here. - CARBON_DIAGNOSTIC(FailureRunningClangToLink, Error, - "failure running `clang` to perform linking: {0}", - std::string); - driver_env.emitter.Emit(FailureRunningClangToLink, - run_result.error().message()); - return {.success = false}; - } - // Successfully ran Clang to perform the link, return its result. - return {.success = *run_result}; + LinkDriver driver(&options_); + return driver.Link(driver_env); } } // namespace Carbon diff --git a/toolchain/driver/link_subcommand.h b/toolchain/driver/link_subcommand.h index dcb0765e0d25..f074998a8e69 100644 --- a/toolchain/driver/link_subcommand.h +++ b/toolchain/driver/link_subcommand.h @@ -11,29 +11,17 @@ #include "toolchain/driver/codegen_options.h" #include "toolchain/driver/driver_env.h" #include "toolchain/driver/driver_subcommand.h" +#include "toolchain/driver/link_options.h" namespace Carbon { -// Options for the link subcommand. -// -// See the implementation of `Build` for documentation on members. -struct LinkOptions { - auto Build(CommandLine::CommandBuilder& b) -> void; - - CodegenOptions codegen_options; - llvm::StringRef output_filename; - llvm::SmallVector object_filenames; - - llvm::SmallVector extra_clang_args; -}; - // Implements the link subcommand of the driver. class LinkSubcommand : public DriverSubcommand { public: explicit LinkSubcommand(); auto BuildOptions(CommandLine::CommandBuilder& b) -> void override { - options_.Build(b); + options_.BuildForLinkSubcommand(b); } auto Run(DriverEnv& driver_env) -> DriverResult override;