diff --git a/BUILD b/BUILD index eb40ce2548cf..529f97e42cce 100644 --- a/BUILD +++ b/BUILD @@ -2,7 +2,7 @@ # Exceptions. See /LICENSE for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -load("@bazel_skylib//rules:common_settings.bzl", "string_list_setting") +load("@bazel_skylib//rules:common_settings.bzl", "bool_setting", "int_setting") filegroup( name = "clang_tidy_config", @@ -16,7 +16,14 @@ alias( actual = "@wolfd_bazel_compile_commands//:generate_compile_commands", ) -string_list_setting( - name = "original_platforms", - build_setting_default = [], +bool_setting( + name = "runtimes_build", + build_setting_default = False, + visibility = ["//visibility:public"], +) + +int_setting( + name = "bootstrap_stage", + build_setting_default = 0, + visibility = ["//visibility:public"], ) diff --git a/bazel/cc_toolchains/BUILD b/bazel/cc_toolchains/BUILD index f605a721df7e..552a35299f05 100644 --- a/bazel/cc_toolchains/BUILD +++ b/bazel/cc_toolchains/BUILD @@ -6,25 +6,7 @@ load("@bazel_skylib//lib:selects.bzl", "selects") package(default_visibility = ["//visibility:public"]) -constraint_setting( - name = "bootstrap_stage", - default_constraint_value = ":stage0", -) - -constraint_value( - name = "stage0", - constraint_setting = ":bootstrap_stage", -) - -constraint_value( - name = "stage1", - constraint_setting = ":bootstrap_stage", -) - -constraint_value( - name = "stage2", - constraint_setting = ":bootstrap_stage", -) +exports_files(["carbon_cc_toolchain_config.bzl"]) # For use by defs.bzl. # Matches when asan is enabled on a macOS platform. @@ -70,7 +52,6 @@ config_setting( filegroup( name = "installed_cc_toolchain_starlark", srcs = [ - "carbon_cc_toolchain_config.bzl", "cc_toolchain_actions.bzl", "cc_toolchain_base_features.bzl", "cc_toolchain_config_features.bzl", diff --git a/bazel/cc_toolchains/carbon_cc_toolchain_config.bzl b/bazel/cc_toolchains/carbon_cc_toolchain_config.bzl index b32ef88c450e..870329a9f461 100644 --- a/bazel/cc_toolchains/carbon_cc_toolchain_config.bzl +++ b/bazel/cc_toolchains/carbon_cc_toolchain_config.bzl @@ -18,6 +18,7 @@ load( "cc_toolchain", ) load("@rules_cc//cc/common:cc_common.bzl", "cc_common") +load("//toolchain/runtimes:carbon_runtimes.bzl", "carbon_runtimes_build") load( "carbon_clang_variables.bzl", "clang_include_dirs", @@ -174,7 +175,7 @@ def _carbon_cc_toolchain_config_impl(ctx): "runtimes/libunwind/include", "runtimes/libcxx/include", "runtimes/libcxxabi/include", - clang_resource_dir + "/include", + "{}/include".format(clang_resource_dir), "runtimes/clang_resource_dir/include", ] + _compute_clang_system_include_dirs() + sysroot_include_search, builtin_sysroot = builtin_sysroot, @@ -200,61 +201,47 @@ carbon_cc_toolchain_config = rule( provides = [CcToolchainConfigInfo], ) -def _set_platform_transition_impl(settings, attr): - original_platforms = settings["//:original_platforms"] - - # If the requested platform is the special value of the setting where we - # store the original platforms on an initial transition, set the platform to - # the saved list and clear it. Otherwise, we will set the platform to the - # requested one. - if not attr.platform: - return { - "//:original_platforms": [], - "//command_line_option:platforms": original_platforms, - } - - if original_platforms: - # If there is already a saved original platforms list, preserve it. - original_platforms = [str(label) for label in original_platforms] - else: - # If there is no saved original platforms list, save the current one. - current_platforms = settings["//command_line_option:platforms"] - original_platforms = [str(label) for label in current_platforms] - +def _transition_with_stage_impl(_, attr): return { - "//:original_platforms": original_platforms, - "//command_line_option:platforms": [str(attr.platform)], + "//:bootstrap_stage": attr.stage, + "//:runtimes_build": attr.enable_runtimes_build, } -set_platform_transition = transition( - inputs = [ - "//command_line_option:platforms", - "//:original_platforms", - ], +_transition_with_stage = transition( + inputs = [], outputs = [ - "//command_line_option:platforms", - "//:original_platforms", + "//:bootstrap_stage", + "//:runtimes_build", ], - implementation = _set_platform_transition_impl, + implementation = _transition_with_stage_impl, ) -def _set_platform_filegroup_impl(ctx): +def _filegroup_with_stage_impl(ctx): return [DefaultInfo(files = depset(ctx.files.srcs))] -set_platform_filegroup = rule( - implementation = _set_platform_filegroup_impl, +filegroup_with_stage = rule( + implementation = _filegroup_with_stage_impl, attrs = { - # The platform to use when building the runtimes. - "platform": attr.label(mandatory = False), + # Whether to enable runtimes building for the sources of this filegroup. + "enable_runtimes_build": attr.bool(default = False), # Mark that our dependencies are built through a transition. - "srcs": attr.label_list(mandatory = True, cfg = set_platform_transition), + "srcs": attr.label_list(mandatory = True, cfg = _transition_with_stage), + + # The bootstrap stage that the sources of this filegroup should be built + # with. + "stage": attr.int(mandatory = True), # Enable transitions in this rule. "_allowlist_function_transition": attr.label( default = "@bazel_tools//tools/allowlists/function_transition_allowlist", ), }, + doc = """ + A filegroup whose sources are built using a specific toolchain stage, and + which provides an interface to build those sources with or without enabling + runtimes building. + """, ) def carbon_cc_toolchain_suite( @@ -263,11 +250,17 @@ def carbon_cc_toolchain_suite( base_files, clang_hdrs, platforms, - runtimes, - build_stage = None, - base_stage = None, + runtimes_cfg, + build_stage = 1, + base_stage = 0, tags = []): - """Create a toolchain suite that uses the local Clang/LLVM install. + """Create a Carbon `cc_toolchain` for the current target platform. + + This provides the final toolchain for Carbon, but also all of the + infrastructure for supporting on-demand built runtimes in this toolchain. + + There is also support for bootstrapping, where one `build_stage` toolchain + builds on top of another `base_stage`. Args: name: @@ -279,74 +272,50 @@ def carbon_cc_toolchain_suite( base_stage: The stage to use for the base files. clang_hdrs: A list of header files to include in the toolchain. platforms: An array of (os, cpu) pairs to support in the toolchain. - runtimes: A list of runtimes to include in the toolchain. + runtimes_cfg: The runtimes configuration to use in the toolchain. tags: Tags to apply to the toolchain. """ - def _platform_name(os, cpu, name_suffix = ""): - return "{}{}_{}_{}_platform".format(name, name_suffix, os, cpu) + # First, declare file groups that are explicitly built using the base stage, + # and not in the runtimes build. These allow us to form the inputs to both + # the runtimes toolchain and the main toolchain of this stage that are built + # entirely by the base stage toolchain. + filegroup_with_stage( + name = "{}_clang_hdrs".format(name), + srcs = clang_hdrs, + stage = base_stage, + tags = tags, + ) - # Define platforms for each supported OS/CPU pair. - for os, cpus in platforms.items(): - for cpu in cpus: - constraint_values = [ - "@platforms//os:" + os, - "@platforms//cpu:" + cpu, - ] - if base_stage: - native.platform( - name = _platform_name(os, cpu, "_base"), - constraint_values = constraint_values + [base_stage], - ) - if build_stage: - constraint_values.append(build_stage) - native.platform( - name = _platform_name(os, cpu), - constraint_values = constraint_values, - ) - native.platform( - name = _platform_name(os, cpu, "_runtimes"), - constraint_values = constraint_values + [":is_runtimes_build"], - ) - - base_platform_select = None - if base_stage: - base_platform_select = select({ - ":is_{}_{}".format(os, cpu): ":" + _platform_name(os, cpu, "_base") - for os, cpus in platforms.items() - for cpu in cpus - }) - - runtimes_platform_select = select({ - ":is_{}_{}".format(os, cpu): ":" + _platform_name(os, cpu, "_runtimes") - for os, cpus in platforms.items() - for cpu in cpus - }) - - set_platform_filegroup( - name = name + "_base_files", + filegroup_with_stage( + name = "{}_base_files".format(name), srcs = base_files, - platform = base_platform_select, + stage = base_stage, tags = tags, ) - set_platform_filegroup( - name = name + "_runtimes_compile_files", - srcs = [":" + name + "_base_files"] + clang_hdrs, - platform = base_platform_select, + filegroup_with_stage( + name = "{}_runtimes_compile_files".format(name), + srcs = [ + ":{}_base_files".format(name), + ":{}_clang_hdrs".format(name), + ], + stage = base_stage, tags = tags, ) - set_platform_filegroup( - name = name + "_compile_files", - srcs = [":" + name + "_base_files"] + all_hdrs, - platform = base_platform_select, + filegroup_with_stage( + name = "{}_compile_files".format(name), + srcs = [":{}_base_files".format(name)] + all_hdrs, + stage = base_stage, tags = tags, ) + # Now build a configuration and toolchain that is configured to work + # _without_ runtimes, and be used to _build_ the runtimes on-demand. carbon_cc_toolchain_config( - name = name + "_runtimes_toolchain_config", - identifier_prefix = name + "_runtimes", + name = "{}_runtimes_toolchain_config".format(name), + identifier_prefix = "{}_runtimes".format(name), target_cpu = select({ # Note that we need to select on both OS and CPU so that we end up # spelling the CPU in the correct OS-specific ways. @@ -355,26 +324,26 @@ def carbon_cc_toolchain_suite( for cpu in cpus }), target_os = select({ - "@platforms//os:" + os: os + "@platforms//os:{}".format(os): os for os in platforms.keys() }), - bins = ":" + name + "_base_files", + bins = ":{}_base_files".format(name), tags = tags, ) cc_toolchain( - name = name + "_runtimes_cc_toolchain", - all_files = ":" + name + "_runtimes_compile_files", - ar_files = ":" + name + "_base_files", - as_files = ":" + name + "_runtimes_compile_files", - compiler_files = ":" + name + "_runtimes_compile_files", - dwp_files = ":" + name + "_base_files", - linker_files = ":" + name + "_base_files", - objcopy_files = ":" + name + "_base_files", - strip_files = ":" + name + "_base_files", - toolchain_config = ":" + name + "_runtimes_toolchain_config", + name = "{}_runtimes_cc_toolchain".format(name), + all_files = ":{}_runtimes_compile_files".format(name), + ar_files = ":{}_base_files".format(name), + as_files = ":{}_runtimes_compile_files".format(name), + compiler_files = ":{}_runtimes_compile_files".format(name), + dwp_files = ":{}_base_files".format(name), + linker_files = ":{}_base_files".format(name), + objcopy_files = ":{}_base_files".format(name), + strip_files = ":{}_base_files".format(name), + toolchain_config = ":{}_runtimes_toolchain_config".format(name), toolchain_identifier = select({ - ":is_{}_{}".format(os, cpu): _platform_name(os, cpu, "_runtimes") + ":is_{}_{}".format(os, cpu): "{}_{}_{}_runtimes_toolchain".format(name, os, cpu) for os, cpus in platforms.items() for cpu in cpus }), @@ -382,22 +351,45 @@ def carbon_cc_toolchain_suite( ) native.toolchain( - name = name + "_runtimes_toolchain", - target_compatible_with = [":is_runtimes_build"], - toolchain = ":" + name + "_runtimes_cc_toolchain", + name = "{}_runtimes_toolchain".format(name), + target_settings = [ + ":is_bootstrap_stage_{}".format(build_stage), + ":is_runtimes_build", + ], + use_target_platform_constraints = True, + toolchain = ":{}_runtimes_cc_toolchain".format(name), toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", tags = tags, ) - set_platform_filegroup( - name = name + "_runtimes", - srcs = [runtimes], - platform = runtimes_platform_select, + # Now that we have a toolchain for building runtimes, actually do the build + # here using the runtimes config provided to us. This is important to do + # here because we need each runtimes build for a particular bootstrapping + # stage of the toolchain to be distinct. + carbon_runtimes_build( + name = "{}_runtimes_build".format(name), + config = runtimes_cfg, + clang_hdrs = [":{}_clang_hdrs".format(name)], tags = tags, ) + # Wrap the built runtimes for this stage in a filegroup that ensures they + # are built at this stage, but with the runtimes build enabled. This will + # select the runtimes build toolchain above that doesn't yet provide any + # runtimes, avoiding a cycle when the main toolchain below depends on these + # runtimes. + filegroup_with_stage( + name = "{}_runtimes".format(name), + enable_runtimes_build = True, + srcs = ["{}_runtimes_build".format(name)], + stage = build_stage, + tags = tags, + ) + + # Now we can build the main toolchain configuration, filegroups including + # the on-demand built runtimes, and the final tolochain itself. carbon_cc_toolchain_config( - name = name + "_toolchain_config", + name = "{}_toolchain_config".format(name), identifier_prefix = name, target_cpu = select({ # Note that we need to select on both OS and CPU so that we end up @@ -407,35 +399,35 @@ def carbon_cc_toolchain_suite( for cpu in cpus }), target_os = select({ - "@platforms//os:" + os: os + "@platforms//os:{}".format(os): os for os in platforms.keys() }), - runtimes = ":" + name + "_runtimes", - bins = ":" + name + "_base_files", + runtimes = ":{}_runtimes".format(name), + bins = ":{}_base_files".format(name), tags = tags, ) native.filegroup( - name = name + "_linker_files", + name = "{}_linker_files".format(name), srcs = [ - ":" + name + "_base_files", - ":" + name + "_runtimes", + ":{}_base_files".format(name), + ":{}_runtimes".format(name), ], tags = tags, ) native.filegroup( - name = name + "_all_files", + name = "{}_all_files".format(name), srcs = [ - ":" + name + "_compile_files", - ":" + name + "_linker_files", + ":{}_compile_files".format(name), + ":{}_linker_files".format(name), ], tags = tags, ) cc_toolchain( - name = name + "_cc_toolchain", - all_files = ":" + name + "_all_files", + name = "{}_cc_toolchain".format(name), + all_files = ":{}_all_files".format(name), ar_files = ":" + name + "_base_files", as_files = ":" + name + "_compile_files", compiler_files = ":" + name + "_compile_files", @@ -445,7 +437,7 @@ def carbon_cc_toolchain_suite( strip_files = ":" + name + "_base_files", toolchain_config = ":" + name + "_toolchain_config", toolchain_identifier = select({ - ":is_{}_{}".format(os, cpu): _platform_name(os, cpu) + ":is_{}_{}".format(os, cpu): "{}_{}_{}_toolchain".format(name, os, cpu) for os, cpus in platforms.items() for cpu in cpus }), @@ -454,7 +446,8 @@ def carbon_cc_toolchain_suite( native.toolchain( name = name + "_toolchain", - target_compatible_with = [build_stage] if build_stage else [], + target_settings = [":is_bootstrap_stage_{}".format(build_stage), ":not_runtimes_build"], + use_target_platform_constraints = True, toolchain = ":" + name + "_cc_toolchain", toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", tags = tags, diff --git a/bazel/cc_toolchains/carbon_runtimes.bzl b/bazel/cc_toolchains/carbon_runtimes.bzl deleted file mode 100644 index 0394ab1e49a3..000000000000 --- a/bazel/cc_toolchains/carbon_runtimes.bzl +++ /dev/null @@ -1,154 +0,0 @@ -# Part of the Carbon Language project, under the Apache License v2.0 with LLVM -# Exceptions. See /LICENSE for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -"""Starlark rules for building a Carbon runtimes tree. - -TODO: Currently, this produces a complete, static Carbon runtimes tree that -mirrors the exact style of runtimes tree the Carbon toolchain would build on its -own. However, it would be preferable to preserve the builtins, libc++, and -libunwind `cc_library` rules as "normal" library rules (if behind a transition) -and automatically depend on them. This would allow things like LTO and such to -include these. However, this requires support in `@rules_cc` for this kind of -dependency to be added. -""" - -load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain") -load("@rules_cc//cc/common:cc_common.bzl", "cc_common") - -def _build_crt_file(ctx, cc_toolchain, feature_configuration, crt_file): - _, compilation_outputs = cc_common.compile( - name = ctx.label.name + ".compile_" + crt_file.basename, - actions = ctx.actions, - feature_configuration = feature_configuration, - cc_toolchain = cc_toolchain, - srcs = [crt_file], - user_compile_flags = ctx.attr.crt_copts, - ) - - # Extract the PIC object file and make sure we built one. - obj = compilation_outputs.pic_objects[0] - if not obj: - fail("The toolchain failed to produce a PIC object file. Ensure your " + - "toolchain supports PIC.") - - return obj - -def _removeprefix_or_fail(s, prefix): - new_s = s.removeprefix(prefix) - if new_s == s: - fail("Unable to remove prefix '{0}' from '{1}'".format(prefix, s)) - return new_s - -def _carbon_runtimes_impl(ctx): - outputs = [] - prefix = ctx.attr.name - - # Create a marker file in the runtimes root first. We'll use this to locate - # the runtimes for the toolchain. - root_out = ctx.actions.declare_file("{0}/runtimes_root".format(prefix)) - ctx.actions.write(output = root_out, content = "") - outputs.append(root_out) - - # Setup the C++ toolchain and configuration. We also force the `pic` feature - # to be enabled for these actions as we always want PIC generated code -- - # this avoids the need to build two versions of the runtimes and doesn't - # create problems with modern code generation when linking statically. This - # also simplifies extracting the outputs as we only need to look at - # `pic_objects`. - cc_toolchain = find_cpp_toolchain(ctx) - feature_configuration = cc_common.configure_features( - ctx = ctx, - cc_toolchain = cc_toolchain, - requested_features = ctx.features + ["pic"], - unsupported_features = ctx.disabled_features, - ) - - if ctx.attr.target_triple != "": - builtins_lib_path = "clang_resource_dir/lib/{0}".format(ctx.attr.target_triple) - else: - builtins_lib_path = "clang_resource_dir/lib" - - for filename, src in [ - ("crtbegin", ctx.files.crtbegin_src), - ("crtend", ctx.files.crtend_src), - ]: - if not src: - continue - src = src[0] - crt_obj = _build_crt_file(ctx, cc_toolchain, feature_configuration, src) - crt_out = ctx.actions.declare_file("{0}/{1}/clang_rt.{2}.o".format( - prefix, - builtins_lib_path, - filename, - )) - ctx.actions.symlink(output = crt_out, target_file = crt_obj) - outputs.append(crt_out) - - for runtime_dir, archive_name, archive in [ - (builtins_lib_path, "libclang_rt.builtins.a", ctx.files.builtins_archive[0]), - ("libcxx/lib", "libc++.a", ctx.files.libcxx_archive[0]), - ("libunwind/lib", "libunwind.a", ctx.files.libunwind_archive[0]), - ]: - runtime_out = ctx.actions.declare_file("{0}/{1}/{2}".format( - prefix, - runtime_dir, - archive_name, - )) - ctx.actions.symlink(output = runtime_out, target_file = archive) - outputs.append(runtime_out) - - for hdr in ctx.files.clang_hdrs: - # Incrementally remove prefixes of the paths to find the `include` - # directory we want to symlink into the output tree. - rel_path = hdr.path - if hdr.root.path != "": - rel_path = _removeprefix_or_fail(rel_path, hdr.root.path + "/") - if hdr.owner.workspace_root != "": - rel_path = _removeprefix_or_fail(rel_path, hdr.owner.workspace_root + "/") - if hdr.owner.package != "": - rel_path = _removeprefix_or_fail(rel_path, hdr.owner.package + "/") - rel_path = _removeprefix_or_fail(rel_path, ctx.attr.clang_hdrs_prefix) - - out_hdr = ctx.actions.declare_file( - "{0}/clang_resource_dir/include/{1}".format(prefix, rel_path), - ) - ctx.actions.symlink(output = out_hdr, target_file = hdr) - outputs.append(out_hdr) - - return [DefaultInfo(files = depset(outputs))] - -carbon_runtimes = rule( - implementation = _carbon_runtimes_impl, - attrs = { - "builtins_archive": attr.label(mandatory = True, allow_files = [".a"]), - "clang_hdrs": attr.label_list(mandatory = True, allow_files = True), - "clang_hdrs_prefix": attr.string(default = "include/"), - "crt_copts": attr.string_list(default = []), - "crtbegin_src": attr.label(allow_files = [".c"]), - "crtend_src": attr.label(allow_files = [".c"]), - "libcxx_archive": attr.label(mandatory = True, allow_files = [".a"]), - "libunwind_archive": attr.label(mandatory = True, allow_files = [".a"]), - "target_triple": attr.string(mandatory = True), - "_cc_toolchain": attr.label( - default = Label("@bazel_tools//tools/cpp:current_cc_toolchain"), - ), - }, - toolchains = ["@bazel_tools//tools/cpp:toolchain_type"], - fragments = ["cpp"], - doc = """Builds a Carbon runtimes tree under the `name` directory. - - This rule works to replicate the behavior of the Carbon toolchain building a - runtimes tree directly in Bazel to allow it to fully benefit from Bazel's - orchestration, caching, and even remote execution. - - It produces a runtimes tree customized for the specific target platform, - similar to what the Carbon toolchain does on its own when invoked outside of - Bazel. - - The runtimes tree includes a complete Clang resource-dir, including CRT - begin/end objects, and the builtins library. It also includes a built libc++ - and libunwind library. These are arranged in the standard Carbon runtimes - layout so that they are correctly found by the toolchain. - """, -) diff --git a/bazel/cc_toolchains/clang_cc_toolchain_config.bzl b/bazel/cc_toolchains/clang_cc_toolchain_config.bzl index 59b78f1cfe5a..81481d4c44e0 100644 --- a/bazel/cc_toolchains/clang_cc_toolchain_config.bzl +++ b/bazel/cc_toolchains/clang_cc_toolchain_config.bzl @@ -92,7 +92,7 @@ def cc_local_toolchain_suite(name, configs): target_cpu = cpu, ) cc_toolchain( - name = config_name + "_tools", + name = config_name + "_toolchain", all_files = ":" + name + "_empty", ar_files = ":" + name + "_empty", as_files = ":" + name + "_empty", @@ -109,7 +109,11 @@ def cc_local_toolchain_suite(name, configs): native.toolchain( name = config_name, exec_compatible_with = compatible_with, - target_compatible_with = compatible_with + ["@carbon//bazel/cc_toolchains:stage0"], - toolchain = config_name + "_tools", + target_settings = [ + "@carbon//toolchain/install:is_bootstrap_stage_0", + "@carbon//toolchain/install:not_runtimes_build", + ], + target_compatible_with = compatible_with, + toolchain = config_name + "_toolchain", toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", ) diff --git a/toolchain/driver/clang_runtimes_test.cpp b/toolchain/driver/clang_runtimes_test.cpp index 30f33909fd4d..e34f2e4d059a 100644 --- a/toolchain/driver/clang_runtimes_test.cpp +++ b/toolchain/driver/clang_runtimes_test.cpp @@ -311,20 +311,20 @@ TEST_F(ClangRuntimesTest, DISABLED_Libcxx) { } TEST_F(ClangRuntimesTest, PrebuiltResourceDir) { - std::filesystem::path prebuilt_runtimes_path = - test_runfiles_->Rlocation("carbon/toolchain/install/runtimes"); + std::filesystem::path prebuilt_runtimes_path = test_runfiles_->Rlocation( + "carbon/toolchain/install/carbon_stage1_runtimes_build"); TestResourceDir(prebuilt_runtimes_path / "clang_resource_dir"); } TEST_F(ClangRuntimesTest, PrebuiltLibunwind) { - std::filesystem::path prebuilt_runtimes_path = - test_runfiles_->Rlocation("carbon/toolchain/install/runtimes"); + std::filesystem::path prebuilt_runtimes_path = test_runfiles_->Rlocation( + "carbon/toolchain/install/carbon_stage1_runtimes_build"); TestLibunwind(prebuilt_runtimes_path / "libunwind/lib/libunwind.a"); } TEST_F(ClangRuntimesTest, PrebuiltLibcxx) { - std::filesystem::path prebuilt_runtimes_path = - test_runfiles_->Rlocation("carbon/toolchain/install/runtimes"); + std::filesystem::path prebuilt_runtimes_path = test_runfiles_->Rlocation( + "carbon/toolchain/install/carbon_stage1_runtimes_build"); TestLibcxx(prebuilt_runtimes_path / "libcxx/lib/libc++.a"); } diff --git a/toolchain/install/BUILD b/toolchain/install/BUILD index 419c0e32d033..83186a31764b 100644 --- a/toolchain/install/BUILD +++ b/toolchain/install/BUILD @@ -38,14 +38,14 @@ load("//bazel/cc_rules:defs.bzl", "cc_binary", "cc_library", "cc_test") load( "//bazel/cc_toolchains:carbon_cc_toolchain_config.bzl", "carbon_cc_toolchain_suite", - "set_platform_filegroup", + "filegroup_with_stage", ) load("//bazel/manifest:defs.bzl", "manifest") load( "//toolchain/base:runtimes_build_info.bzl", "generate_runtimes_build_vars", ) -load("//toolchain/runtimes:carbon_runtimes.bzl", "carbon_runtimes") +load("//toolchain/runtimes:carbon_runtimes.bzl", "carbon_runtimes_config") load("bazel/make_include_copts.bzl", "make_include_copts") load( "install_filegroups.bzl", @@ -58,13 +58,29 @@ load("pkg_helpers.bzl", "pkg_naming_variables", "pkg_tar_and_test") package(default_visibility = ["//visibility:public"]) -constraint_setting( - name = "runtimes_build", +config_setting( + name = "is_runtimes_build", + flag_values = {"//:runtimes_build": "True"}, ) -constraint_value( - name = "is_runtimes_build", - constraint_setting = ":runtimes_build", +config_setting( + name = "not_runtimes_build", + flag_values = {"//:runtimes_build": "False"}, +) + +config_setting( + name = "is_bootstrap_stage_0", + flag_values = {"//:bootstrap_stage": "0"}, +) + +config_setting( + name = "is_bootstrap_stage_1", + flag_values = {"//:bootstrap_stage": "1"}, +) + +config_setting( + name = "is_bootstrap_stage_2", + flag_values = {"//:bootstrap_stage": "2"}, ) cc_library( @@ -242,7 +258,6 @@ toolchain_files( name = "libc_internal_libcxx_hdrs", srcs = ["@llvm-project//libc:libcxx_shared_headers_hdrs"], prefix = "runtimes/libc/internal/", - #remove_prefix = "src", ) toolchain_files( @@ -273,9 +288,25 @@ filegroup( ], ) +# The toolchain configuration starlark needs to switch from referencing the +# `carbon_runtimes.bzl` starlark in the toolchain layout to the `bazel` tree +# within the installation. +# +# TODO: Maybe we should move the `carbon_runtimes.bzl` to the same directory as +# the other starlark so we can use relative load lines and avoid this? +genrule( + name = "carbon_cc_toolchain_config_for_install", + srcs = ["//bazel/cc_toolchains:carbon_cc_toolchain_config.bzl"], + outs = ["carbon_cc_toolchain_config.bzl"], + cmd = "sed 's|//toolchain/runtimes:carbon_runtimes.bzl|//bazel:carbon_runtimes.bzl|' $< > $@", +) + toolchain_files( name = "bazel_common_srcs", - srcs = ["//bazel/cc_toolchains:installed_cc_toolchain_starlark"], + srcs = [ + ":carbon_cc_toolchain_config_for_install", + "//bazel/cc_toolchains:installed_cc_toolchain_starlark", + ], prefix = "bazel/", ) @@ -391,8 +422,10 @@ genrule( "install_digest_manifest.txt", ], outs = ["install_digest.txt"], - cmd = "$(location :make-installation-digest) " + - "$(location install_digest_manifest.txt) $@", + cmd = "{0} {1} $@".format( + "$(location :make-installation-digest)", + "$(location install_digest_manifest.txt)", + ), tools = [":make-installation-digest"], ) @@ -437,8 +470,7 @@ cc_library( ], hdrs_check = "strict", target_compatible_with = select({ - "//bazel/cc_toolchains:stage1": [], - "//bazel/cc_toolchains:stage2": [], + ":is_runtimes_build": [], "//conditions:default": ["@platforms//:incompatible"], }), deps = [":builtins_internal"], @@ -466,8 +498,7 @@ cc_library( includes = ["runtimes/libunwind/include"], linkstatic = 1, target_compatible_with = select({ - "//bazel/cc_toolchains:stage1": [], - "//bazel/cc_toolchains:stage2": [], + ":is_runtimes_build": [], "//conditions:default": ["@platforms//:incompatible"], }), ) @@ -482,8 +513,7 @@ cc_library( name = "libcxxabi_internal", hdrs_check = "strict", target_compatible_with = select({ - "//bazel/cc_toolchains:stage1": [], - "//bazel/cc_toolchains:stage2": [], + ":is_runtimes_build": [], "//conditions:default": ["@platforms//:incompatible"], }), textual_hdrs = [":libcxxabi_textual_srcs"], @@ -494,8 +524,7 @@ cc_library( hdrs = [":libc_internal_libcxx_hdrs"], hdrs_check = "strict", target_compatible_with = select({ - "//bazel/cc_toolchains:stage1": [], - "//bazel/cc_toolchains:stage2": [], + ":is_runtimes_build": [], "//conditions:default": ["@platforms//:incompatible"], }), ) @@ -528,8 +557,7 @@ cc_library( "runtimes/libcxxabi/include", ], target_compatible_with = select({ - "//bazel/cc_toolchains:stage1": [], - "//bazel/cc_toolchains:stage2": [], + ":is_runtimes_build": [], "//conditions:default": ["@platforms//:incompatible"], }), deps = [ @@ -544,10 +572,9 @@ filegroup( output_group = "archive", ) -carbon_runtimes( - name = "runtimes", +carbon_runtimes_config( + name = "runtimes_cfg", builtins_archive = ":builtins_archive", - clang_hdrs = [":clang_hdrs"], clang_hdrs_prefix = "llvm/lib/clang/{0}/include/".format(LLVM_VERSION_MAJOR), crt_copts = crt_copts + [ # Disable all sanitizers for CRT objects. @@ -570,11 +597,6 @@ carbon_runtimes( }), libcxx_archive = ":libcxx_archive", libunwind_archive = ":libunwind_archive", - target_compatible_with = select({ - "//bazel/cc_toolchains:stage1": [], - "//bazel/cc_toolchains:stage2": [], - "//conditions:default": ["@platforms//:incompatible"], - }), target_triple = select({ # TODO: Add other triples (and if needed, constraints) so that we can # build the correct Clang resource-dir structure for each. @@ -603,8 +625,8 @@ platforms = { config_setting( name = "is_{0}_{1}".format(os, cpu), constraint_values = [ - "@platforms//os:" + os, - "@platforms//cpu:" + cpu, + "@platforms//os:{}".format(os), + "@platforms//cpu:{}".format(cpu), ], ) for os, cpus in platforms.items() @@ -624,10 +646,10 @@ carbon_cc_toolchain_suite( ":carbon-busybox", ":llvm_bins", ], - build_stage = "//bazel/cc_toolchains:stage1", + build_stage = 1, clang_hdrs = [":clang_hdrs"], platforms = platforms, - runtimes = ":runtimes", + runtimes_cfg = ":runtimes_cfg", ) carbon_cc_toolchain_suite( @@ -643,11 +665,11 @@ carbon_cc_toolchain_suite( ":carbon-busybox", ":llvm_bins", ], - base_stage = "//bazel/cc_toolchains:stage1", - build_stage = "//bazel/cc_toolchains:stage2", + base_stage = 1, + build_stage = 2, clang_hdrs = [":clang_hdrs"], platforms = platforms, - runtimes = ":runtimes", + runtimes_cfg = ":runtimes_cfg", tags = ["manual"], ) @@ -680,28 +702,20 @@ pkg_tar_and_test( stamp = -1, # Allow `--stamp` builds to produce file timestamps. ) -set_platform_filegroup( +filegroup_with_stage( name = "stage2_pkg_files", srcs = [ ":all_data_files", ":gen_digest", ], - platform = select({ - ":is_{}_{}".format(os, cpu): ":carbon_stage2_base_{}_{}_platform".format(os, cpu) - for os, cpus in platforms.items() - for cpu in cpus - }), + stage = 1, tags = ["manual"], ) -set_platform_filegroup( +filegroup_with_stage( name = "stage2_busybox", srcs = [":carbon-busybox"], - platform = select({ - ":is_{}_{}".format(os, cpu): ":carbon_stage2_base_{}_{}_platform".format(os, cpu) - for os, cpus in platforms.items() - for cpu in cpus - }), + stage = 1, tags = ["manual"], ) diff --git a/toolchain/install/bazel/install.BUILD b/toolchain/install/bazel/install.BUILD index 2fc3bcdd42e8..3cb6b0137ef8 100644 --- a/toolchain/install/bazel/install.BUILD +++ b/toolchain/install/bazel/install.BUILD @@ -2,10 +2,10 @@ # Exceptions. See /LICENSE for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -load("@bazel_skylib//rules:common_settings.bzl", "string_list_setting") +load("@bazel_skylib//rules:common_settings.bzl", "bool_setting", "int_setting") load("@rules_cc//cc:defs.bzl", "cc_library") load("//bazel:carbon_cc_toolchain_config.bzl", "carbon_cc_toolchain_suite") -load("//bazel:carbon_runtimes.bzl", "carbon_runtimes") +load("//bazel:carbon_runtimes.bzl", "carbon_runtimes_build", "carbon_runtimes_config") load("//bazel:make_include_copts.bzl", "make_include_copts") load( "//bazel:runtimes_build_vars.bzl", @@ -41,11 +41,29 @@ _libcxx_hdrs = libcxx_hdrs + [ package(default_visibility = ["//visibility:public"]) -constraint_setting(name = "runtimes_build") +bool_setting( + name = "runtimes_build", + build_setting_default = False, +) -constraint_value( +config_setting( name = "is_runtimes_build", - constraint_setting = ":runtimes_build", + flag_values = {":runtimes_build": "True"}, +) + +config_setting( + name = "not_runtimes_build", + flag_values = {":runtimes_build": "False"}, +) + +int_setting( + name = "bootstrap_stage", + build_setting_default = 1, +) + +config_setting( + name = "is_bootstrap_stage_1", + flag_values = {":bootstrap_stage": "1"}, ) filegroup( @@ -185,10 +203,9 @@ filegroup( output_group = "archive", ) -carbon_runtimes( - name = "runtimes", +carbon_runtimes_config( + name = "runtimes_cfg", builtins_archive = ":builtins_archive", - clang_hdrs = [":clang_hdrs"], clang_hdrs_prefix = "llvm/lib/clang/{0}/include/".format(llvm_version_major), crt_copts = crt_copts, crtbegin_src = select({ @@ -224,6 +241,12 @@ carbon_runtimes( }), ) +carbon_runtimes_build( + name = "runtimes", + clang_hdrs = [":clang_hdrs"], + config = ":runtimes_cfg", +) + filegroup( name = "carbon_install_digest_file", srcs = ["install_digest.txt"], @@ -239,11 +262,6 @@ filegroup( srcs = ["carbon-busybox"], ) -string_list_setting( - name = "original_platforms", - build_setting_default = [], -) - platforms = { "freebsd": ["x86_64"], "linux": [ @@ -260,8 +278,8 @@ platforms = { config_setting( name = "is_{0}_{1}".format(os, cpu), constraint_values = [ - "@platforms//os:" + os, - "@platforms//cpu:" + cpu, + "@platforms//os:{}".format(os), + "@platforms//cpu:{}".format(cpu), ], ) for os, cpus in platforms.items() @@ -283,5 +301,5 @@ carbon_cc_toolchain_suite( ], clang_hdrs = [":clang_hdrs"], platforms = platforms, - runtimes = ":runtimes", + runtimes_cfg = ":runtimes_cfg", ) diff --git a/toolchain/install/install_test.py b/toolchain/install/install_test.py index d7d1e562a7b4..de1d3b9835f7 100644 --- a/toolchain/install/install_test.py +++ b/toolchain/install/install_test.py @@ -27,7 +27,7 @@ class InstallTest(unittest.TestCase): self.test_o_file.touch() self.runfiles = runfiles.Create() self.prebuilt_runtimes = self.runfiles.Rlocation( - "carbon/toolchain/install/runtimes" + "carbon/toolchain/install/carbon_stage1_runtimes_build" ) def get_link_cmd(self, clang: Path) -> list[str | Path]: diff --git a/toolchain/runtimes/carbon_runtimes.bzl b/toolchain/runtimes/carbon_runtimes.bzl index cc00737ebe0b..e870bb9440e1 100644 --- a/toolchain/runtimes/carbon_runtimes.bzl +++ b/toolchain/runtimes/carbon_runtimes.bzl @@ -16,14 +16,14 @@ dependency to be added. load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain") load("@rules_cc//cc/common:cc_common.bzl", "cc_common") -def _build_crt_file(ctx, cc_toolchain, feature_configuration, crt_file): +def _build_crt_file(ctx, cc_toolchain, feature_configuration, crt_file, crt_copts): _, compilation_outputs = cc_common.compile( - name = ctx.label.name + ".compile_" + crt_file.basename, + name = "{}.compile_{}".format(ctx.label.name, crt_file.basename), actions = ctx.actions, feature_configuration = feature_configuration, cc_toolchain = cc_toolchain, srcs = [crt_file], - user_compile_flags = ctx.attr.crt_copts, + user_compile_flags = crt_copts, ) # Extract the PIC object file and make sure we built one. @@ -40,7 +40,58 @@ def _removeprefix_or_fail(s, prefix): fail("Unable to remove prefix '{0}' from '{1}'".format(prefix, s)) return new_s -def _carbon_runtimes_impl(ctx): +CarbonRuntimesConfigInfo = provider( + doc = """Configuration for Carbon runtimes. + + This provider is used to collect all of the information that will be needed + to build a Carbon runtimes directory for a Bazel Carbon `cc_toolchain`. + """, + fields = [ + "builtins_archive", + "clang_hdrs_prefix", + "crt_copts", + "crtbegin_src", + "crtend_src", + "darwin_os_suffix", + "libcxx_archive", + "libunwind_archive", + "target_triple", + ], +) + +def _carbon_runtimes_config_impl(ctx): + return [ + CarbonRuntimesConfigInfo( + builtins_archive = ctx.files.builtins_archive[0], + clang_hdrs_prefix = ctx.attr.clang_hdrs_prefix, + crt_copts = ctx.attr.crt_copts, + crtbegin_src = ctx.files.crtbegin_src[0] if ctx.files.crtbegin_src else None, + crtend_src = ctx.files.crtend_src[0] if ctx.files.crtend_src else None, + darwin_os_suffix = ctx.attr.darwin_os_suffix, + libcxx_archive = ctx.files.libcxx_archive[0], + libunwind_archive = ctx.files.libunwind_archive[0], + target_triple = ctx.attr.target_triple, + ), + ] + +carbon_runtimes_config = rule( + implementation = _carbon_runtimes_config_impl, + attrs = { + "builtins_archive": attr.label(mandatory = True, allow_files = [".a"]), + "clang_hdrs_prefix": attr.string(default = "include/"), + "crt_copts": attr.string_list(default = []), + "crtbegin_src": attr.label(allow_files = [".c"]), + "crtend_src": attr.label(allow_files = [".c"]), + "darwin_os_suffix": attr.string(mandatory = False), + "libcxx_archive": attr.label(mandatory = True, allow_files = [".a"]), + "libunwind_archive": attr.label(mandatory = True, allow_files = [".a"]), + "target_triple": attr.string(mandatory = False), + }, + doc = "Collects configuration for building a Carbon runtimes tree.", +) + +def _carbon_runtimes_build_impl(ctx): + config = ctx.attr.config[CarbonRuntimesConfigInfo] outputs = [] prefix = ctx.attr.name @@ -67,20 +118,19 @@ def _carbon_runtimes_impl(ctx): builtins_lib_path = "clang_resource_dir/lib" builtins_archive_name = "libclang_rt.builtins.a" - if ctx.attr.target_triple != "": - builtins_lib_path = "clang_resource_dir/lib/{0}".format(ctx.attr.target_triple) - elif ctx.attr.darwin_os_suffix: + if config.target_triple != "": + builtins_lib_path = "clang_resource_dir/lib/{0}".format(config.target_triple) + elif config.darwin_os_suffix: builtins_lib_path = "clang_resource_dir/lib/darwin" - builtins_archive_name = "libclang_rt.{0}.a".format(ctx.attr.darwin_os_suffix) + builtins_archive_name = "libclang_rt.{0}.a".format(config.darwin_os_suffix) for filename, src in [ - ("crtbegin", ctx.files.crtbegin_src), - ("crtend", ctx.files.crtend_src), + ("crtbegin", config.crtbegin_src), + ("crtend", config.crtend_src), ]: if not src: continue - src = src[0] - crt_obj = _build_crt_file(ctx, cc_toolchain, feature_configuration, src) + crt_obj = _build_crt_file(ctx, cc_toolchain, feature_configuration, src, config.crt_copts) crt_out = ctx.actions.declare_file("{0}/{1}/clang_rt.{2}.o".format( prefix, builtins_lib_path, @@ -90,9 +140,9 @@ def _carbon_runtimes_impl(ctx): outputs.append(crt_out) for runtime_dir, archive_name, archive in [ - (builtins_lib_path, builtins_archive_name, ctx.files.builtins_archive[0]), - ("libcxx/lib", "libc++.a", ctx.files.libcxx_archive[0]), - ("libunwind/lib", "libunwind.a", ctx.files.libunwind_archive[0]), + (builtins_lib_path, builtins_archive_name, config.builtins_archive), + ("libcxx/lib", "libc++.a", config.libcxx_archive), + ("libunwind/lib", "libunwind.a", config.libunwind_archive), ]: runtime_out = ctx.actions.declare_file("{0}/{1}/{2}".format( prefix, @@ -107,12 +157,12 @@ def _carbon_runtimes_impl(ctx): # directory we want to symlink into the output tree. rel_path = hdr.path if hdr.root.path != "": - rel_path = _removeprefix_or_fail(rel_path, hdr.root.path + "/") + rel_path = _removeprefix_or_fail(rel_path, "{}/".format(hdr.root.path)) if hdr.owner.workspace_root != "": - rel_path = _removeprefix_or_fail(rel_path, hdr.owner.workspace_root + "/") + rel_path = _removeprefix_or_fail(rel_path, "{}/".format(hdr.owner.workspace_root)) if hdr.owner.package != "": - rel_path = _removeprefix_or_fail(rel_path, hdr.owner.package + "/") - rel_path = _removeprefix_or_fail(rel_path, ctx.attr.clang_hdrs_prefix) + rel_path = _removeprefix_or_fail(rel_path, "{}/".format(hdr.owner.package)) + rel_path = _removeprefix_or_fail(rel_path, config.clang_hdrs_prefix) out_hdr = ctx.actions.declare_file( "{0}/clang_resource_dir/include/{1}".format(prefix, rel_path), @@ -122,38 +172,27 @@ def _carbon_runtimes_impl(ctx): return [DefaultInfo(files = depset(outputs))] -carbon_runtimes = rule( - implementation = _carbon_runtimes_impl, +carbon_runtimes_build = rule( + implementation = _carbon_runtimes_build_impl, attrs = { - "builtins_archive": attr.label(mandatory = True, allow_files = [".a"]), - "clang_hdrs": attr.label_list(mandatory = True, allow_files = True), - "clang_hdrs_prefix": attr.string(default = "include/"), - "crt_copts": attr.string_list(default = []), - "crtbegin_src": attr.label(allow_files = [".c"]), - "crtend_src": attr.label(allow_files = [".c"]), - "darwin_os_suffix": attr.string(mandatory = False), - "libcxx_archive": attr.label(mandatory = True, allow_files = [".a"]), - "libunwind_archive": attr.label(mandatory = True, allow_files = [".a"]), - "target_triple": attr.string(mandatory = False), + "clang_hdrs": attr.label_list( + mandatory = True, + allow_files = True, + ), + "config": attr.label(mandatory = True, providers = [CarbonRuntimesConfigInfo]), "_cc_toolchain": attr.label( default = Label("@bazel_tools//tools/cpp:current_cc_toolchain"), ), }, toolchains = ["@bazel_tools//tools/cpp:toolchain_type"], fragments = ["cpp"], - doc = """Builds a Carbon runtimes tree under the `name` directory. + doc = """Builds a Carbon runtimes tree using a config rule and clang_hdrs. - This rule works to replicate the behavior of the Carbon toolchain building a - runtimes tree directly in Bazel to allow it to fully benefit from Bazel's - orchestration, caching, and even remote execution. + The configuration provides access to all of the targets that should be built + into the runtimes. - It produces a runtimes tree customized for the specific target platform, - similar to what the Carbon toolchain does on its own when invoked outside of - Bazel. - - The runtimes tree includes a complete Clang resource-dir, including CRT - begin/end objects, and the builtins library. It also includes a built libc++ - and libunwind library. These are arranged in the standard Carbon runtimes - layout so that they are correctly found by the toolchain. + Any files, such as `clang_hdrs`, that should be built _prior_ to the + runtimes build taking place are accepted as separate file groups so that + they can be properly handled when bootstrapping. """, )