From 8d1d491ad0c32edb6338382d8482ff5a6bbeb245 Mon Sep 17 00:00:00 2001 From: Chandler Carruth Date: Thu, 20 Feb 2025 20:26:45 -0800 Subject: [PATCH] Add LLD subcommand and busybox support (#4973) This removes the separately built and installed LLD binary. The symlinks used by Clang when directly invoking LLD now point back to the main `carbon-busybox` binary and dispatch through the newly added subcommand. With this change we're down to shipping a single busybox binary in the toolchain, removing duplicate installed copies of LLD and all its LLVM dependencies. =] The LLD subcommand works a bit differently from the `clang` subcommand because the CLI for LLD is specific to which platform flavor of linker is being invoked. As part of this, I've extracted some of the common functionality in the Clang runner into a base class that can be re-used. I expect to use this again in a follow-up change to add subcommands to run other LLVM tools. --------- Co-authored-by: Jon Ross-Perkins --- toolchain/diagnostics/diagnostic_kind.def | 1 + toolchain/driver/BUILD | 62 +++++- toolchain/driver/clang_runner.cpp | 50 +---- toolchain/driver/clang_runner.h | 6 +- toolchain/driver/driver.cpp | 3 + toolchain/driver/lld_runner.cpp | 59 +++++ toolchain/driver/lld_runner.h | 36 +++ toolchain/driver/lld_runner_test.cpp | 210 ++++++++++++++++++ toolchain/driver/lld_subcommand.cpp | 113 ++++++++++ toolchain/driver/lld_subcommand.h | 54 +++++ .../driver/testdata/fail_lld_fuzzing.carbon | 15 ++ toolchain/driver/tool_runner_base.cpp | 73 ++++++ toolchain/driver/tool_runner_base.h | 65 ++++++ toolchain/install/BUILD | 15 +- toolchain/install/busybox_main.cpp | 15 +- toolchain/install/install_paths.cpp | 24 ++ toolchain/install/install_paths.h | 5 + toolchain/install/install_paths_test.cpp | 3 +- 18 files changed, 745 insertions(+), 64 deletions(-) create mode 100644 toolchain/driver/lld_runner.cpp create mode 100644 toolchain/driver/lld_runner.h create mode 100644 toolchain/driver/lld_runner_test.cpp create mode 100644 toolchain/driver/lld_subcommand.cpp create mode 100644 toolchain/driver/lld_subcommand.h create mode 100644 toolchain/driver/testdata/fail_lld_fuzzing.carbon create mode 100644 toolchain/driver/tool_runner_base.cpp create mode 100644 toolchain/driver/tool_runner_base.h diff --git a/toolchain/diagnostics/diagnostic_kind.def b/toolchain/diagnostics/diagnostic_kind.def index 9a58df5bf90e..d803021abed3 100644 --- a/toolchain/diagnostics/diagnostic_kind.def +++ b/toolchain/diagnostics/diagnostic_kind.def @@ -29,6 +29,7 @@ CARBON_DIAGNOSTIC_KIND(CompilePreludeManifestError) CARBON_DIAGNOSTIC_KIND(CompileInputNotRegularFile) CARBON_DIAGNOSTIC_KIND(CompileOutputFileOpenError) CARBON_DIAGNOSTIC_KIND(FormatMultipleFilesToOneOutput) +CARBON_DIAGNOSTIC_KIND(LLDFuzzingDisallowed) // ============================================================================ // SourceBuffer diagnostics diff --git a/toolchain/driver/BUILD b/toolchain/driver/BUILD index 798b5a345561..d845a46a186f 100644 --- a/toolchain/driver/BUILD +++ b/toolchain/driver/BUILD @@ -20,11 +20,8 @@ cc_library( name = "clang_runner", srcs = ["clang_runner.cpp"], hdrs = ["clang_runner.h"], - data = [ - "//toolchain/install:install_data.no_driver", - ], deps = [ - "//common:command_line", + ":tool_runner_base", "//common:ostream", "//common:vlog", "//toolchain/install:install_paths", @@ -104,6 +101,8 @@ cc_library( "language_server_subcommand.h", "link_subcommand.cpp", "link_subcommand.h", + "lld_subcommand.cpp", + "lld_subcommand.h", ], hdrs = [ "driver.h", @@ -115,6 +114,7 @@ cc_library( textual_hdrs = ["flags.def"], deps = [ ":clang_runner", + ":lld_runner", "//common:command_line", "//common:error", "//common:ostream", @@ -182,3 +182,57 @@ cc_fuzz_test( "@llvm-project//llvm:Support", ], ) + +cc_library( + name = "lld_runner", + srcs = ["lld_runner.cpp"], + hdrs = ["lld_runner.h"], + deps = [ + ":tool_runner_base", + "//common:ostream", + "//common:vlog", + "//toolchain/install:install_paths", + "@llvm-project//lld:Common", + "@llvm-project//lld:ELF", + "@llvm-project//lld:MachO", + "@llvm-project//llvm:Support", + ], +) + +cc_test( + name = "lld_runner_test", + size = "small", + srcs = ["lld_runner_test.cpp"], + env = cc_env(), + deps = [ + ":clang_runner", + ":lld_runner", + "//common:all_llvm_targets", + "//common:check", + "//common:ostream", + "//common:raw_string_ostream", + "//testing/base:capture_std_streams", + "//testing/base:file_helpers", + "//testing/base:global_exe_path", + "//testing/base:gtest_main", + "@googletest//:gtest", + "@llvm-project//llvm:Object", + "@llvm-project//llvm:Support", + "@llvm-project//llvm:TargetParser", + ], +) + +cc_library( + name = "tool_runner_base", + srcs = ["tool_runner_base.cpp"], + hdrs = ["tool_runner_base.h"], + data = [ + "//toolchain/install:install_data.no_driver", + ], + deps = [ + "//common:ostream", + "//common:vlog", + "//toolchain/install:install_paths", + "@llvm-project//llvm:Support", + ], +) diff --git a/toolchain/driver/clang_runner.cpp b/toolchain/driver/clang_runner.cpp index 2d569c743218..ca05bd530744 100644 --- a/toolchain/driver/clang_runner.cpp +++ b/toolchain/driver/clang_runner.cpp @@ -15,7 +15,6 @@ #include "clang/Driver/Driver.h" #include "clang/Frontend/CompilerInvocation.h" #include "clang/Frontend/TextDiagnosticPrinter.h" -#include "common/command_line.h" #include "common/vlog.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/ScopeExit.h" @@ -44,59 +43,20 @@ ClangRunner::ClangRunner(const InstallPaths* install_paths, llvm::StringRef target, llvm::IntrusiveRefCntPtr fs, llvm::raw_ostream* vlog_stream) - : installation_(install_paths), + : ToolRunnerBase(install_paths, vlog_stream), target_(target), fs_(std::move(fs)), - vlog_stream_(vlog_stream), diagnostic_ids_(new clang::DiagnosticIDs()) {} auto ClangRunner::Run(llvm::ArrayRef args) -> bool { // TODO: Maybe handle response file expansion similar to the Clang CLI? - // If we have a verbose logging stream, and that stream is the same as - // `llvm::errs`, then add the `-v` flag so that the driver also prints verbose - // information. - bool inject_v_arg = vlog_stream_ == &llvm::errs(); - std::array v_arg_storage; - llvm::ArrayRef maybe_v_arg; - if (inject_v_arg) { - v_arg_storage[0] = "-v"; - maybe_v_arg = v_arg_storage; - } - - CARBON_VLOG("Running Clang driver with arguments: \n"); - - // Render the arguments into null-terminated C-strings for use by the Clang - // driver. Command lines can get quite long in build systems so this tries to - // minimize the memory allocation overhead. - - // Provide the wrapped `clang` path in order to support subprocessing. We also - // set the install directory below. std::string clang_path = installation_->clang_path(); - std::array exe_arg = {clang_path}; - auto args_range = - llvm::concat(exe_arg, maybe_v_arg, args); - int total_size = 0; - for (llvm::StringRef arg : args_range) { - // Accumulate both the string size and a null terminator byte. - total_size += arg.size() + 1; - } - // Allocate one chunk of storage for the actual C-strings and a vector of - // pointers into the storage. - llvm::OwningArrayRef cstr_arg_storage(total_size); - llvm::SmallVector cstr_args; - cstr_args.reserve(args.size() + inject_v_arg + 1); - for (ssize_t i = 0; llvm::StringRef arg : args_range) { - cstr_args.push_back(&cstr_arg_storage[i]); - memcpy(&cstr_arg_storage[i], arg.data(), arg.size()); - i += arg.size(); - cstr_arg_storage[i] = '\0'; - ++i; - } - for (const char* cstr_arg : llvm::ArrayRef(cstr_args)) { - CARBON_VLOG(" '{0}'\n", cstr_arg); - } + // Rebuild the args as C-string args. + llvm::OwningArrayRef cstr_arg_storage; + llvm::SmallVector cstr_args = + BuildCStrArgs("Clang", clang_path, "-v", args, cstr_arg_storage); if (!args.empty() && args[0].starts_with("-cc1")) { CARBON_VLOG("Calling clang_main for cc1..."); diff --git a/toolchain/driver/clang_runner.h b/toolchain/driver/clang_runner.h index 4d93031b865b..854c5bb85c22 100644 --- a/toolchain/driver/clang_runner.h +++ b/toolchain/driver/clang_runner.h @@ -10,6 +10,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/VirtualFileSystem.h" +#include "toolchain/driver/tool_runner_base.h" #include "toolchain/install/install_paths.h" namespace Carbon { @@ -36,7 +37,7 @@ namespace Carbon { // standard output and standard error, and otherwise can only read and write // files based on their names described in the arguments. It doesn't provide any // higher-level abstraction such as streams for inputs or outputs. -class ClangRunner { +class ClangRunner : ToolRunnerBase { public: // Build a Clang runner that uses the provided `exe_name` and `err_stream`. // @@ -61,11 +62,8 @@ class ClangRunner { auto EnableLeakingMemory() -> void { enable_leaking_ = true; } private: - const InstallPaths* installation_; - llvm::StringRef target_; llvm::IntrusiveRefCntPtr fs_; - llvm::raw_ostream* vlog_stream_; llvm::IntrusiveRefCntPtr diagnostic_ids_; diff --git a/toolchain/driver/driver.cpp b/toolchain/driver/driver.cpp index e0bd0381e0eb..850c029c09bc 100644 --- a/toolchain/driver/driver.cpp +++ b/toolchain/driver/driver.cpp @@ -15,6 +15,7 @@ #include "toolchain/driver/format_subcommand.h" #include "toolchain/driver/language_server_subcommand.h" #include "toolchain/driver/link_subcommand.h" +#include "toolchain/driver/lld_subcommand.h" namespace Carbon { @@ -33,6 +34,7 @@ struct Options { FormatSubcommand format; LanguageServerSubcommand language_server; LinkSubcommand link; + LldSubcommand lld; // On success, this is set to the subcommand to run. DriverSubcommand* selected_subcommand = nullptr; @@ -89,6 +91,7 @@ applies to each message that forms a diagnostic, not just the primary message. format.AddTo(b, &selected_subcommand); language_server.AddTo(b, &selected_subcommand); link.AddTo(b, &selected_subcommand); + lld.AddTo(b, &selected_subcommand); b.RequiresSubcommand(); } diff --git a/toolchain/driver/lld_runner.cpp b/toolchain/driver/lld_runner.cpp new file mode 100644 index 000000000000..8386cd516670 --- /dev/null +++ b/toolchain/driver/lld_runner.cpp @@ -0,0 +1,59 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "toolchain/driver/lld_runner.h" + +#include +#include +#include +#include + +#include "common/vlog.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" + +// Declare the supported driver flavor entry points. +// +// TODO: Currently, just ELF and MachO, but eventually we should support all of +// the LLD platforms. +// +// NOLINTBEGIN(readability-identifier-naming): External library name. +LLD_HAS_DRIVER(elf) +LLD_HAS_DRIVER(macho) +// NOLINTEND(readability-identifier-naming) + +namespace Carbon { + +auto LldRunner::LinkHelper(llvm::StringLiteral label, + llvm::ArrayRef args, + const std::string& path, lld::DriverDef driver_def) + -> bool { + // Allocate one chunk of storage for the actual C-strings and a vector of + // pointers into the storage. + llvm::OwningArrayRef cstr_arg_storage; + llvm::SmallVector cstr_args = + BuildCStrArgs("LLD", path, "-v", args, cstr_arg_storage); + + CARBON_VLOG("Running LLD {0}-platform link...\n", label); + lld::Result result = + lld::lldMain(cstr_args, llvm::outs(), llvm::errs(), {driver_def}); + + // Check for an unrecoverable error. + CARBON_CHECK(result.canRunAgain, "LLD encountered an unrecoverable error!"); + + // TODO: Should this be forwarding the full exit code? + return result.retCode == 0; +} + +auto LldRunner::ElfLink(llvm::ArrayRef args) -> bool { + return LinkHelper("GNU", args, installation_->ld_lld_path(), + {.f = lld::Gnu, .d = &lld::elf::link}); +} + +auto LldRunner::MachOLink(llvm::ArrayRef args) -> bool { + return LinkHelper("Darwin", args, installation_->ld64_lld_path(), + {.f = lld::Darwin, .d = &lld::macho::link}); +} + +} // namespace Carbon diff --git a/toolchain/driver/lld_runner.h b/toolchain/driver/lld_runner.h new file mode 100644 index 000000000000..9143bdf040b1 --- /dev/null +++ b/toolchain/driver/lld_runner.h @@ -0,0 +1,36 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef CARBON_TOOLCHAIN_DRIVER_LLD_RUNNER_H_ +#define CARBON_TOOLCHAIN_DRIVER_LLD_RUNNER_H_ + +#include "common/ostream.h" +#include "lld/Common/Driver.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" +#include "toolchain/driver/tool_runner_base.h" +#include "toolchain/install/install_paths.h" + +namespace Carbon { + +// Runs LLD in a manner similar to invoking it with the provided arguments. +class LldRunner : ToolRunnerBase { + public: + using ToolRunnerBase::ToolRunnerBase; + + // Run LLD as a GNU-style linker with the provided arguments. + auto ElfLink(llvm::ArrayRef args) -> bool; + + // Run LLD as a Darwin-style linker with the provided arguments. + auto MachOLink(llvm::ArrayRef args) -> bool; + + private: + auto LinkHelper(llvm::StringLiteral label, + llvm::ArrayRef args, const std::string& path, + lld::DriverDef driver_def) -> bool; +}; + +} // namespace Carbon + +#endif // CARBON_TOOLCHAIN_DRIVER_LLD_RUNNER_H_ diff --git a/toolchain/driver/lld_runner_test.cpp b/toolchain/driver/lld_runner_test.cpp new file mode 100644 index 000000000000..fc26f5a17235 --- /dev/null +++ b/toolchain/driver/lld_runner_test.cpp @@ -0,0 +1,210 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "toolchain/driver/lld_runner.h" + +#include +#include + +#include +#include +#include + +#include "common/check.h" +#include "common/ostream.h" +#include "common/raw_string_ostream.h" +#include "llvm/ADT/ScopeExit.h" +#include "llvm/Object/Binary.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/Program.h" +#include "llvm/TargetParser/Host.h" +#include "testing/base/capture_std_streams.h" +#include "testing/base/file_helpers.h" +#include "testing/base/global_exe_path.h" +#include "toolchain/driver/clang_runner.h" + +namespace Carbon { +namespace { + +using ::testing::HasSubstr; +using ::testing::Not; +using ::testing::StrEq; + +TEST(LldRunnerTest, Version) { + RawStringOstream test_os; + const auto install_paths = + InstallPaths::MakeForBazelRunfiles(Testing::GetExePath()); + LldRunner runner(&install_paths, &test_os); + + std::string out; + std::string err; + EXPECT_TRUE(Testing::CallWithCapturedOutput( + out, err, [&] { return runner.ElfLink({"--version"}); })); + + // The arguments to LLD should be part of the verbose log. + EXPECT_THAT(test_os.TakeStr(), HasSubstr("--version")); + + // Nothing should print to stderr here. + EXPECT_THAT(err, StrEq("")); + + // We don't care about any particular version, just that it is printed. + EXPECT_THAT(out, HasSubstr("LLD")); + // Check that it was in fact the GNU linker. + EXPECT_THAT(out, HasSubstr("compatible with GNU linkers")); + + // Try the Darwin linker. + EXPECT_TRUE(Testing::CallWithCapturedOutput( + out, err, [&] { return runner.MachOLink({"--version"}); })); + + // Again, the arguments to LLD should be part of the verbose log. + EXPECT_THAT(test_os.TakeStr(), HasSubstr("--version")); + + // Nothing should print to stderr. + EXPECT_THAT(err, StrEq("")); + + // We don't care about any particular version. + EXPECT_THAT(out, HasSubstr("LLD")); + // The Darwin link code path doesn't print anything distinct, so instead check + // that the GNU output isn't repeated. + EXPECT_THAT(out, Not(HasSubstr("GNU"))); +} + +static auto CompileTwoSources(const InstallPaths& install_paths, + llvm::StringRef target) + -> std::pair { + std::filesystem::path test_a_file = + *Testing::WriteTestFile("test_a.cpp", "int test_a() { return 0; }"); + std::filesystem::path test_b_file = *Testing::WriteTestFile( + "test_b.cpp", "int test_a();\nint main() { return test_a(); }"); + std::filesystem::path test_a_output = *Testing::WriteTestFile("test_a.o", ""); + std::filesystem::path test_b_output = *Testing::WriteTestFile("test_b.o", ""); + + // First compile the two source files to `.o` files with Clang. + RawStringOstream verbose_out; + auto vfs = llvm::vfs::getRealFileSystem(); + ClangRunner clang(&install_paths, target, vfs, &verbose_out); + std::string target_arg = llvm::formatv("--target={0}", target).str(); + std::string out; + std::string err; + CARBON_CHECK( + Testing::CallWithCapturedOutput( + out, err, + [&] { + return clang.Run({target_arg, "-fPIE", "-c", test_a_file.string(), + "-o", test_a_output.string()}); + }), + "Verbose output from runner:\n{0}\nStderr:\n{1}\n", verbose_out.TakeStr(), + err); + verbose_out.clear(); + + CARBON_CHECK( + Testing::CallWithCapturedOutput( + out, err, + [&] { + return clang.Run({target_arg, "-fPIE", "-c", test_b_file.string(), + "-o", test_b_output.string()}); + }), + "Verbose output from runner:\n{0}\nStderr:\n{1}\n", verbose_out.TakeStr(), + err); + verbose_out.clear(); + + return {test_a_output, test_b_output}; +} + +TEST(LldRunnerTest, ElfLinkTest) { + const auto install_paths = + InstallPaths::MakeForBazelRunfiles(Testing::GetExePath()); + + std::filesystem::path test_a_output; + std::filesystem::path test_b_output; + std::tie(test_a_output, test_b_output) = + CompileTwoSources(install_paths, "aarch64-unknown-linux"); + + std::filesystem::path test_output = *Testing::WriteTestFile("test.o", ""); + + RawStringOstream verbose_out; + std::string out; + std::string err; + + LldRunner lld(&install_paths, &verbose_out); + + // Link the two object files together. + // + // TODO: Currently, this uses a relocatable link, but it would be better to do + // a full link to an executable. For that to work, we need at least the + // C-runtime built artifacts available in the toolchain. We should revisit + // this once we have those in place. This also prevents us from testing a + // failed link easily. + EXPECT_TRUE(Testing::CallWithCapturedOutput( + out, err, + [&] { + return lld.ElfLink({"-m", "aarch64linux", "--relocatable", "-o", + test_output.string(), test_a_output.string(), + test_b_output.string()}); + })) + << "Verbose output from runner:\n" + << verbose_out.TakeStr() << "\n"; + verbose_out.clear(); + + // No output should be produced. + EXPECT_THAT(out, StrEq("")); + EXPECT_THAT(err, StrEq("")); +} + +TEST(LldRunnerTest, MachOLinkTest) { + const auto install_paths = + InstallPaths::MakeForBazelRunfiles(Testing::GetExePath()); + + std::filesystem::path test_a_output; + std::filesystem::path test_b_output; + std::tie(test_a_output, test_b_output) = + CompileTwoSources(install_paths, "arm64-unknown-macosx10.4.0"); + + std::filesystem::path test_output = *Testing::WriteTestFile("test.o", ""); + + RawStringOstream verbose_out; + std::string out; + std::string err; + + // Link the two object files together. + // + // This is a somewhat arbitrary command line, and is missing the C-runtimes, + // but seems to succeed currently. The goal isn't to test any *particular* + // link, but just than an actual link occurs successfully. + LldRunner lld(&install_paths, &verbose_out); + EXPECT_TRUE(Testing::CallWithCapturedOutput( + out, err, + [&] { + return lld.MachOLink({"-arch", "arm64", "-platform_version", "macos", + "10.4.0", "10.4.0", "-o", test_output.string(), + test_a_output.string(), test_b_output.string()}); + })) + << "Verbose output from runner:\n" + << verbose_out.TakeStr() << "\n"; + verbose_out.clear(); + + // No output should be produced. + EXPECT_THAT(out, StrEq("")); + EXPECT_THAT(err, StrEq("")); + + // Re-do the link, but with only one of the inputs. This should fail due to an + // unresolved symbol. + EXPECT_FALSE(Testing::CallWithCapturedOutput( + out, err, + [&] { + return lld.MachOLink({"-arch", "arm64", "-platform_version", "macos", + "10.4.0", "10.4.0", "-o", test_output.string(), + test_b_output.string()}); + })) + << "Verbose output from runner:\n" + << verbose_out.TakeStr() << "\n"; + verbose_out.clear(); + + // The missing symbol should be diagnosed on `stderr`. + EXPECT_THAT(out, StrEq("")); + EXPECT_THAT(err, HasSubstr("undefined symbol: __Z6test_av")); +} + +} // namespace +} // namespace Carbon diff --git a/toolchain/driver/lld_subcommand.cpp b/toolchain/driver/lld_subcommand.cpp new file mode 100644 index 000000000000..3c082fbdf31b --- /dev/null +++ b/toolchain/driver/lld_subcommand.cpp @@ -0,0 +1,113 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "toolchain/driver/lld_subcommand.h" + +#include "llvm/TargetParser/Host.h" +#include "llvm/TargetParser/Triple.h" +#include "toolchain/driver/lld_runner.h" + +namespace Carbon { + +auto LldOptions::Build(CommandLine::CommandBuilder& b) -> void { + // We want to select a default platform based on the default target. Since + // that requires some dynamic inspection of the target, do that here. + std::string default_target = llvm::sys::getDefaultTargetTriple(); + llvm::Triple default_triple(default_target); + switch (default_triple.getObjectFormat()) { + case llvm::Triple::MachO: + platform = Platform::MachO; + break; + + // We default to the GNU or Unix platform as ELF is a plausible default + // and LLD doesn't support any generic invocations. + default: + case llvm::Triple::ELF: + platform = Platform::Elf; + break; + } + + b.AddOneOfOption( + { + .name = "platform", + .help = R"""( +Platform linking style to use. The default is selected to match the default +target's platform. +)""", + }, + [&](auto& arg_b) { + arg_b.SetOneOf( + { + arg_b.OneOfValue("elf", Platform::Elf), + // Some of LLD documentation uses "Unix" or "GNU", so + // include an alias here. + arg_b.OneOfValue("gnu", Platform::Elf), + arg_b.OneOfValue("unix", Platform::Elf), + + arg_b.OneOfValue("macho", Platform::MachO), + // Darwin is also sometimes used, include it as an alias here. + arg_b.OneOfValue("darwin", Platform::MachO), + }, + &platform); + }); + b.AddStringPositionalArg( + { + .name = "ARG", + .help = R"""( +Arguments passed to LLD. +)""", + }, + [&](auto& arg_b) { arg_b.Append(&args); }); +} + +static constexpr CommandLine::CommandInfo SubcommandInfo = { + .name = "lld", + .help = R"""( +Runs LLD with the provided arguments. + +Note that a specific LLD platform must be selected, and it is actually that +particular platform's LLD-driver that is run with the arguments. There is no +generic LLD command line. + +For a given platform, this is equivalent to running that platform's LLD alias +directly, and provides the full command line interface. + +Use `carbon lld --platform=elf -- ARGS` to separate the `ARGS` forwarded to LLD +from the flags passed to the Carbon subcommand. + +Note that typically it is better to use a higher level command to link code, +such as invoking `carbon link` with the relevant flags. However, this subcommand +supports when you already have a specific invocation using existing command line +syntaxes, as well as testing and debugging of the underlying tool. +)""", +}; + +LldSubcommand::LldSubcommand() : DriverSubcommand(SubcommandInfo) {} + +// TODO: This lacks a lot of features from the main driver code. We may need to +// add more. +// https://github.com/llvm/llvm-project/blob/main/clang/tools/driver/driver.cpp +auto LldSubcommand::Run(DriverEnv& driver_env) -> DriverResult { + LldRunner runner(driver_env.installation, driver_env.vlog_stream); + + // Don't run LLD when fuzzing, as we're not currently in a good position to + // debug and fix fuzzer-found bugs within LLD. + if (driver_env.fuzzing) { + CARBON_DIAGNOSTIC( + LLDFuzzingDisallowed, Error, + "preventing fuzzing of `lld` subcommand due to external library"); + driver_env.emitter.Emit(LLDFuzzingDisallowed); + return {.success = false}; + } + + switch (options_.platform) { + case LldOptions::Platform::Elf: + return {.success = runner.ElfLink(options_.args)}; + case LldOptions::Platform::MachO: + return {.success = runner.MachOLink(options_.args)}; + } + CARBON_FATAL("Failed to find and run a valid LLD platform link!"); +} + +} // namespace Carbon diff --git a/toolchain/driver/lld_subcommand.h b/toolchain/driver/lld_subcommand.h new file mode 100644 index 000000000000..24569cae695d --- /dev/null +++ b/toolchain/driver/lld_subcommand.h @@ -0,0 +1,54 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef CARBON_TOOLCHAIN_DRIVER_LLD_SUBCOMMAND_H_ +#define CARBON_TOOLCHAIN_DRIVER_LLD_SUBCOMMAND_H_ + +#include "common/command_line.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "toolchain/driver/driver_env.h" +#include "toolchain/driver/driver_subcommand.h" + +namespace Carbon { + +// Options for the LLD subcommand, which is just a thin wrapper. +// +// See the implementation of `Build` for documentation on members. +struct LldOptions { + // Supported linking platforms. + // + // Note that these are similar to the object formats in an LLVM triple, but we + // use a distinct enum because we only include the platforms supported by our + // subcommand which is a subset of those recognized by the LLVM triple + // infrastructure. + enum class Platform { + Elf, + MachO, + }; + + auto Build(CommandLine::CommandBuilder& b) -> void; + + Platform platform; + llvm::SmallVector args; +}; + +// Implements the LLD subcommand of the driver. +class LldSubcommand : public DriverSubcommand { + public: + explicit LldSubcommand(); + + auto BuildOptions(CommandLine::CommandBuilder& b) -> void override { + options_.Build(b); + } + + auto Run(DriverEnv& driver_env) -> DriverResult override; + + private: + LldOptions options_; +}; + +} // namespace Carbon + +#endif // CARBON_TOOLCHAIN_DRIVER_LLD_SUBCOMMAND_H_ diff --git a/toolchain/driver/testdata/fail_lld_fuzzing.carbon b/toolchain/driver/testdata/fail_lld_fuzzing.carbon new file mode 100644 index 000000000000..b1e144c75344 --- /dev/null +++ b/toolchain/driver/testdata/fail_lld_fuzzing.carbon @@ -0,0 +1,15 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// ARGS: --include-diagnostic-kind --fuzzing lld -- -o foo foo.o bar.o +// +// SET-CAPTURE-CONSOLE-OUTPUT +// clang-format off +// AUTOUPDATE +// TIP: To test this file alone, run: +// TIP: bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/driver/testdata/fail_lld_fuzzing.carbon +// TIP: To dump output, run: +// TIP: bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/driver/testdata/fail_lld_fuzzing.carbon +// CHECK:STDERR: error: preventing fuzzing of `lld` subcommand due to external library [LLDFuzzingDisallowed] +// CHECK:STDERR: diff --git a/toolchain/driver/tool_runner_base.cpp b/toolchain/driver/tool_runner_base.cpp new file mode 100644 index 000000000000..a3455ed546bb --- /dev/null +++ b/toolchain/driver/tool_runner_base.cpp @@ -0,0 +1,73 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "toolchain/driver/tool_runner_base.h" + +#include + +#include "common/vlog.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" + +namespace Carbon { + +ToolRunnerBase::ToolRunnerBase(const InstallPaths* install_paths, + llvm::raw_ostream* vlog_stream) + : installation_(install_paths), vlog_stream_(vlog_stream) {} + +auto ToolRunnerBase::BuildCStrArgs(llvm::StringRef tool_name, + llvm::StringRef tool_path, + std::optional verbose_flag, + llvm::ArrayRef args, + llvm::OwningArrayRef& cstr_arg_storage) + -> llvm::SmallVector { + // TODO: Maybe handle response file expansion similar to the Clang CLI? + + // If we have a verbose logging stream, and that stream is the same as + // `llvm::errs`, then add the `-v` flag so that the driver also prints verbose + // information. + bool inject_v_arg = verbose_flag.has_value() && vlog_stream_ == &llvm::errs(); + std::array v_arg_storage; + llvm::ArrayRef maybe_v_arg; + if (inject_v_arg) { + v_arg_storage[0] = *verbose_flag; + maybe_v_arg = v_arg_storage; + } + + CARBON_VLOG("Running {} driver with arguments:\n", tool_name); + + // Render the arguments into null-terminated C-strings. Command lines can get + // quite long in build systems so this tries to minimize the memory allocation + // overhead. + + // Provide the wrapped tool path as the synthetic `argv[0]`. + std::array exe_arg = {tool_path}; + auto args_range = + llvm::concat(exe_arg, maybe_v_arg, args); + int total_size = 0; + for (llvm::StringRef arg : args_range) { + // Accumulate both the string size and a null terminator byte. + total_size += arg.size() + 1; + } + + // Allocate one chunk of storage for the actual C-strings and a vector of + // pointers into the storage. + cstr_arg_storage = llvm::OwningArrayRef(total_size); + llvm::SmallVector cstr_args; + cstr_args.reserve(args.size() + inject_v_arg + 1); + for (ssize_t i = 0; llvm::StringRef arg : args_range) { + cstr_args.push_back(&cstr_arg_storage[i]); + memcpy(&cstr_arg_storage[i], arg.data(), arg.size()); + i += arg.size(); + cstr_arg_storage[i] = '\0'; + ++i; + } + for (const char* cstr_arg : llvm::ArrayRef(cstr_args)) { + CARBON_VLOG(" '{0}'\n", cstr_arg); + } + + return cstr_args; +} + +} // namespace Carbon diff --git a/toolchain/driver/tool_runner_base.h b/toolchain/driver/tool_runner_base.h new file mode 100644 index 000000000000..5fc4c2b33a3d --- /dev/null +++ b/toolchain/driver/tool_runner_base.h @@ -0,0 +1,65 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef CARBON_TOOLCHAIN_DRIVER_TOOL_RUNNER_BASE_H_ +#define CARBON_TOOLCHAIN_DRIVER_TOOL_RUNNER_BASE_H_ + +#include + +#include "common/ostream.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" +#include "toolchain/install/install_paths.h" + +namespace Carbon { + +// Base that factors out common utilities needed when implementing a runner of +// an external tool, especially tools part of the LLVM and Clang C++ toolchain +// that Carbon will end up wrapping. +// +// Note that this struct just collects common data and helper methods, and does +// not itself impose any invariants or form a meaningful API. It should be used +// as an implementation detail only. +class ToolRunnerBase { + public: + // Construct the tool runner bas. + // + // If `vlog_stream` is provided, it will be used for `CARBON_VLOG`s. If it is + // also equal to `&llvm::errs()`, and so tied to stderr, that will be used by + // verbose flag injection helpers in this class. + explicit ToolRunnerBase(const InstallPaths* install_paths, + llvm::raw_ostream* vlog_stream = nullptr); + + protected: + // Translates `args` into C-string arguments for tool APIs based on `main`. + // + // Accepts a `tool_name` for logging, and a `tool_path` that will be used as + // the first C-string argument to simulate and `argv[0]` entry. + // + // Accepts a `cstr_arg_storage` that will provide the underlying storage for + // the C-strings, and returns a small vector of the C-string pointers. The + // returned small vector uses a large small size to allow most common command + // lines to avoid extra allocations and growth passes. + // + // Lastly accepts an optional `verbose_flag`. If provided, and if + // `vlog_stream_` is bound to stderr for this instance, the verbose flag will + // be injected at the start of the argument list. + auto BuildCStrArgs(llvm::StringRef tool_name, llvm::StringRef tool_path, + std::optional verbose_flag, + llvm::ArrayRef args, + llvm::OwningArrayRef& cstr_arg_storage) + -> llvm::SmallVector; + + // We use protected members as this base is just factoring out common + // implementation details of other runners. + // + // NOLINTBEGIN(misc-non-private-member-variables-in-classes) + const InstallPaths* installation_; + llvm::raw_ostream* vlog_stream_; + // NOLINTEND(misc-non-private-member-variables-in-classes) +}; + +} // namespace Carbon + +#endif // CARBON_TOOLCHAIN_DRIVER_TOOL_RUNNER_BASE_H_ diff --git a/toolchain/install/BUILD b/toolchain/install/BUILD index 6c520478f6fb..61c967455abe 100644 --- a/toolchain/install/BUILD +++ b/toolchain/install/BUILD @@ -116,11 +116,11 @@ cc_binary( ], ) +# TODO: Add remaining aliases of LLD for Windows and WASM when we have support +# for them wired up through the busybox. lld_aliases = [ "ld.lld", "ld64.lld", - "lld-link", - "wasm-ld", ] filegroup( @@ -156,17 +156,16 @@ install_dirs = { install_filegroup("core", "//core:prelude"), ], "lib/carbon/llvm/bin": [ - install_target( - "lld", - "@llvm-project//lld:lld", - executable = True, - ), install_symlink( "clang", "../../carbon-busybox", is_driver = True, ), - ] + [install_symlink(name, "lld") for name in lld_aliases], + ] + [install_symlink( + name, + "../../carbon-busybox", + is_driver = True, + ) for name in lld_aliases], "lib/carbon/llvm/lib/clang/" + LLVM_VERSION_MAJOR: [ install_filegroup("include", ":clang_headers", "staging/include/"), ], diff --git a/toolchain/install/busybox_main.cpp b/toolchain/install/busybox_main.cpp index c4a87635a839..bfc93971e1c7 100644 --- a/toolchain/install/busybox_main.cpp +++ b/toolchain/install/busybox_main.cpp @@ -42,7 +42,20 @@ static auto Main(int argc, char** argv) -> ErrorOr { llvm::SmallVector args; args.reserve(argc + 1); if (busybox_info.mode) { - args.append({*busybox_info.mode, "--"}); + // Map busybox modes to the relevant subcommands with any flags needed to + // emulate the requested command. Typically, our busyboxed binaries redirect + // to a specific subcommand with some flags set and then pass the remaining + // busybox arguments as positional arguments to that subcommand. + // + // TODO: Add relevant flags to the `clang` subcommand and add `clang`-based + // symlinks to this like `clang++`. + auto subcommand_args = + llvm::StringSwitch>( + *busybox_info.mode) + .Case("ld.lld", {"lld", "--platform=gnu", "--"}) + .Case("ld64.lld", {"lld", "--platform=darwin", "--"}) + .Default({*busybox_info.mode, "--"}); + args.append(subcommand_args); } args.append(argv + 1, argv + argc); diff --git a/toolchain/install/install_paths.cpp b/toolchain/install/install_paths.cpp index eff97b6d80dd..cf7d6ccf5f9f 100644 --- a/toolchain/install/install_paths.cpp +++ b/toolchain/install/install_paths.cpp @@ -171,4 +171,28 @@ auto InstallPaths::clang_path() const -> std::string { return path.str().str(); } +auto InstallPaths::lld_path() const -> std::string { + llvm::SmallString<256> path(prefix_); + // TODO: Adjust this to work equally well on Windows. + llvm::sys::path::append(path, llvm::sys::path::Style::posix, + "lib/carbon/llvm/bin/lld"); + return path.str().str(); +} + +auto InstallPaths::ld_lld_path() const -> std::string { + llvm::SmallString<256> path(prefix_); + // TODO: Adjust this to work equally well on Windows. + llvm::sys::path::append(path, llvm::sys::path::Style::posix, + "lib/carbon/llvm/bin/ld.lld"); + return path.str().str(); +} + +auto InstallPaths::ld64_lld_path() const -> std::string { + llvm::SmallString<256> path(prefix_); + // TODO: Adjust this to work equally well on Windows. + llvm::sys::path::append(path, llvm::sys::path::Style::posix, + "lib/carbon/llvm/bin/ld64.lld"); + return path.str().str(); +} + } // namespace Carbon diff --git a/toolchain/install/install_paths.h b/toolchain/install/install_paths.h index 745d451f06a2..b28d677cffea 100644 --- a/toolchain/install/install_paths.h +++ b/toolchain/install/install_paths.h @@ -87,6 +87,11 @@ class InstallPaths { // The path to `clang`. auto clang_path() const -> std::string; + // The path to `lld' and various aliases of `lld`. + auto lld_path() const -> std::string; + auto ld_lld_path() const -> std::string; + auto ld64_lld_path() const -> std::string; + private: friend class InstallPathsTestPeer; diff --git a/toolchain/install/install_paths_test.cpp b/toolchain/install/install_paths_test.cpp index 1ba4abfc6eb9..cedcf2019ca5 100644 --- a/toolchain/install/install_paths_test.cpp +++ b/toolchain/install/install_paths_test.cpp @@ -75,8 +75,7 @@ class InstallPathsTest : public ::testing::Test { EXPECT_TRUE(llvm::sys::fs::is_directory(llvm_bin_path)) << "path: " << llvm_bin_path; - for (llvm::StringRef llvm_bin : - {"lld", "ld.lld", "ld64.lld", "lld-link", "wasm-ld"}) { + for (llvm::StringRef llvm_bin : {"ld.lld", "ld64.lld"}) { llvm::SmallString<128> bin_path; bin_path.assign(llvm_bin_path); llvm::sys::path::append(bin_path, llvm_bin);