Files
carbon-lang/bazel/cc_toolchains/clang_bootstrap.bzl
T
Chandler CarruthandMatthew Riley 13303159f5 Download prebuilt Clang instead of building locally (#299)
Tested on Ubuntu 20.04 and macOS (both Intel and M1 BigSur).

The bootstrap option is still available with `--config=force_local_bootstrap`.

Note that while this works on the ARM M1 for macOS, it is using the x86-based
toolchain and Rosetta, not doing an ARM build at the moment.

You can also point at a locally installed Clang and LLVM toolchain by using
the `--override_repository` flag to Bazel.

Co-authored-by: Matthew Riley <mdriley@gmail.com>
2021-02-25 18:36:41 -08:00

280 lines
11 KiB
Python

# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
# Exceptions. See /LICENSE for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
"""Starlark rules to bootstrap Clang (and LLVM).
These rules are loaded as part of the `WORKSPACE`, and used by
`clang_configuration.bzl`. The llvm-project submodule is used for the build.
"""
FORCE_LOCAL_BOOTSTRAP_ENV = "CARBON_FORCE_LOCAL_BOOTSTRAP_BUILD"
def _run(
repository_ctx,
cmd,
timeout = 10,
environment = {},
quiet = True):
"""Runs the provided `cmd`, checks for failure, and returns the result."""
exec_result = repository_ctx.execute(
cmd,
timeout = timeout,
environment = environment,
quiet = quiet,
)
if exec_result.return_code != 0:
fail("Unable to run command successfully: %s" % str(cmd))
return exec_result
def _detect_system_clang(repository_ctx):
"""Detects whether a system-provided clang can be used.
Returns a tuple of (is_clang, environment).
"""
# If the user provides an explicit `CC` environment variable, use that as
# the compiler.
cc = repository_ctx.os.environ.get("CC")
cxx = repository_ctx.os.environ.get("CXX")
if cc or cxx:
version_output = _run(repository_ctx, [cc, "--version"]).stdout
return "clang" in version_output, {}
# If we can build our Clang toolchain using a system-installed Clang, try
# to do so.
system_clang = repository_ctx.which("clang")
if system_clang:
return True, {
"CC": str(system_clang),
"CXX": str(system_clang) + "++",
}
return False, {}
def _get_cmake_defines(repository_ctx, is_clang):
"""Returns a long list of cmake defines for the bootstrap."""
modules_setting = "OFF"
if is_clang:
modules_setting = "ON"
static_link_cxx = "ON"
unstable_libcxx_abi = "ON"
if repository_ctx.os.name.lower().startswith("mac os"):
# macOS doesn't support the static C++ standard library linking. Turn
# it off here, and disable the unstable libc++ ABI as we will also be
# unable to use it later on.
static_link_cxx = "OFF"
unstable_libcxx_abi = "OFF"
return [
"-DLLVM_ENABLE_PROJECTS=clang;clang-tools-extra;lld;libcxx;libcxxabi;compiler-rt;libunwind",
"-DCMAKE_BUILD_TYPE=Release",
"-DLLVM_ENABLE_ASSERTIONS=OFF",
"-DLLVM_ENABLE_MODULES=" + modules_setting,
"-DLLVM_STATIC_LINK_CXX_STDLIB=" + static_link_cxx,
"-DLLVM_TARGETS_TO_BUILD=AArch64;X86",
"-DLIBCXX_ABI_UNSTABLE=" + unstable_libcxx_abi,
"-DLIBCXX_ENABLE_ASSERTIONS=OFF",
"-DLIBCXXABI_ENABLE_ASSERTIONS=OFF",
# Disable components of the build that we don't use while building Carbon.
"-DCLANG_ENABLE_ARCMT=OFF",
"-DCLANG_INCLUDE_TESTS=OFF",
"-DCLANG_TOOL_APINOTES_TEST_BUILD=OFF",
"-DCLANG_TOOL_ARCMT_TEST_BUILD=OFF",
"-DCLANG_TOOL_CLANG_CHECK_BUILD=OFF",
"-DCLANG_TOOL_CLANG_DIFF_BUILD=OFF",
"-DCLANG_TOOL_CLANG_EXTDEF_MAPPING_BUILD=OFF",
"-DCLANG_TOOL_CLANG_FUZZER_BUILD=OFF",
"-DCLANG_TOOL_CLANG_IMPORT_TEST_BUILD=OFF",
"-DCLANG_TOOL_CLANG_OFFLOAD_BUNDLER_BUILD=OFF",
"-DCLANG_TOOL_CLANG_OFFLOAD_WRAPPER_BUILD=OFF",
"-DCLANG_TOOL_CLANG_SCAN_DEPS_BUILD=OFF",
"-DCLANG_TOOL_CLANG_SHLIB_BUILD=OFF",
"-DCLANG_TOOL_C_ARCMT_TEST_BUILD=OFF",
"-DCLANG_TOOL_C_INDEX_TEST_BUILD=OFF",
"-DCLANG_TOOL_DIAGTOOL_BUILD=OFF",
"-DCLANG_TOOL_LIBCLANG_BUILD=OFF",
"-DCLANG_TOOL_SCAN_BUILD_BUILD=OFF",
"-DCLANG_TOOL_SCAN_VIEW_BUILD=OFF",
"-DLLVM_BUILD_UTILS=OFF",
"-DLLVM_ENABLE_BINDINGS=OFF",
"-DLLVM_ENABLE_LIBXML2=OFF",
"-DLLVM_ENABLE_OCAMLDOC=OFF",
"-DLLVM_INCLUDE_BENCHMARKS=OFF",
"-DLLVM_INCLUDE_DOCS=OFF",
"-DLLVM_INCLUDE_EXAMPLES=OFF",
"-DLLVM_INCLUDE_GO_TESTS=OFF",
"-DLLVM_INCLUDE_TESTS=OFF",
"-DLLVM_INCLUDE_UTILS=OFF",
"-DLLVM_TOOL_BUGPOINT_BUILD=OFF",
"-DLLVM_TOOL_BUGPOINT_PASSES_BUILD=OFF",
"-DLLVM_TOOL_DSYMUTIL_BUILD=OFF",
"-DLLVM_TOOL_GOLD_BUILD=OFF",
"-DLLVM_TOOL_LLC_BUILD=OFF",
"-DLLVM_TOOL_LLI_BUILD=OFF",
"-DLLVM_TOOL_LLVM_AS_BUILD=OFF",
"-DLLVM_TOOL_LLVM_BCANALYZER_BUILD=OFF",
"-DLLVM_TOOL_LLVM_CAT_BUILD=OFF",
"-DLLVM_TOOL_LLVM_CFI_VERIFY_BUILD=OFF",
"-DLLVM_TOOL_LLVM_CONFIG_BUILD=OFF",
"-DLLVM_TOOL_LLVM_CVTRES_BUILD=OFF",
"-DLLVM_TOOL_LLVM_CXXDUMP_BUILD=OFF",
"-DLLVM_TOOL_LLVM_CXXFILT_BUILD=OFF",
"-DLLVM_TOOL_LLVM_CXXMAP_BUILD=OFF",
"-DLLVM_TOOL_LLVM_C_TEST_BUILD=OFF",
"-DLLVM_TOOL_LLVM_DIFF_BUILD=OFF",
"-DLLVM_TOOL_LLVM_DWARFDUMP_BUILD=OFF",
"-DLLVM_TOOL_LLVM_ELFABI_BUILD=OFF",
"-DLLVM_TOOL_LLVM_EXEGESIS_BUILD=OFF",
"-DLLVM_TOOL_LLVM_EXTRACT_BUILD=OFF",
"-DLLVM_TOOL_LLVM_GO_BUILD=OFF",
"-DLLVM_TOOL_LLVM_GSYMUTIL_BUILD=OFF",
"-DLLVM_TOOL_LLVM_IFS_BUILD=OFF",
"-DLLVM_TOOL_LLVM_ISEL_FUZZER_BUILD=OFF",
"-DLLVM_TOOL_LLVM_ITANIUM_DEMANGLE_FUZZER_BUILD=OFF",
"-DLLVM_TOOL_LLVM_JITLINK_BUILD=OFF",
"-DLLVM_TOOL_LLVM_JITLISTENER_BUILD=OFF",
"-DLLVM_TOOL_LLVM_LIBTOOL_DARWIN_BUILD=OFF",
"-DLLVM_TOOL_LLVM_LINK_BUILD=OFF",
"-DLLVM_TOOL_LLVM_LIPO_BUILD=OFF",
"-DLLVM_TOOL_LLVM_LTO2_BUILD=OFF",
"-DLLVM_TOOL_LLVM_LTO_BUILD=OFF",
"-DLLVM_TOOL_LLVM_MCA_BUILD=OFF",
"-DLLVM_TOOL_LLVM_MC_BUILD=OFF",
"-DLLVM_TOOL_LLVM_ML_BUILD=OFF",
"-DLLVM_TOOL_LLVM_MICROSOFT_DEMANGLE_FUZZER_BUILD=OFF",
"-DLLVM_TOOL_LLVM_MT_BUILD=OFF",
"-DLLVM_TOOL_LLVM_OPT_FUZZER_BUILD=OFF",
"-DLLVM_TOOL_LLVM_PDBUTIL_BUILD=OFF",
"-DLLVM_TOOL_LLVM_PROFDATA_BUILD=OFF",
"-DLLVM_TOOL_LLVM_PROFGEN_BUILD=OFF",
"-DLLVM_TOOL_LLVM_RC_BUILD=OFF",
"-DLLVM_TOOL_LLVM_READOBJ_BUILD=OFF",
"-DLLVM_TOOL_LLVM_REDUCE_BUILD=OFF",
"-DLLVM_TOOL_LLVM_RTDYLD_BUILD=OFF",
"-DLLVM_TOOL_LLVM_SHLIB_BUILD=OFF",
"-DLLVM_TOOL_LLVM_SIZE_BUILD=OFF",
"-DLLVM_TOOL_LLVM_SPECIAL_CASE_LIST_FUZZER_BUILD=OFF",
"-DLLVM_TOOL_LLVM_SPLIT_BUILD=OFF",
"-DLLVM_TOOL_LLVM_STRESS_BUILD=OFF",
"-DLLVM_TOOL_LLVM_STRINGS_BUILD=OFF",
"-DLLVM_TOOL_LLVM_XRAY_BUILD=OFF",
"-DLLVM_TOOL_LLVM_YAML_NUMERIC_PARSER_FUZZER_BUILD=OFF",
"-DLLVM_TOOL_LLVM_YAML_PARSER_FUZZER_BUILD=OFF",
"-DLLVM_TOOL_LTO_BUILD=OFF",
"-DLLVM_TOOL_OBJ2YAML_BUILD=OFF",
"-DLLVM_TOOL_OPT_BUILD=OFF",
"-DLLVM_TOOL_OPT_VIEWER_BUILD=OFF",
"-DLLVM_TOOL_REMARKS_SHLIB_BUILD=OFF",
"-DLLVM_TOOL_SPLIT_FILE_BUILD=OFF",
"-DLLVM_TOOL_VERIFY_USELISTORDER_BUILD=OFF",
"-DLLVM_TOOL_YAML2OBJ_BUILD=OFF",
]
def _local_cmake_build_clang_toolchain(repository_ctx):
"""Locally build the LLVM toolchain with CMake and Ninja.
This is used as a fallback for when we can't download a prebuilt set of
binaries and libraries for a particular platform.
"""
repository_ctx.report_progress("Configuring Clang toolchain bootstrap...")
is_clang, environment = _detect_system_clang(repository_ctx)
cmake = repository_ctx.which("cmake")
if not cmake:
fail("`cmake` not found: is it installed?")
ninja = repository_ctx.which("ninja")
if not ninja:
fail("`ninja` not found: is it installed?")
workspace_dir = repository_ctx.path(repository_ctx.attr._workspace).dirname
llvm_dir = repository_ctx.path("%s/third_party/llvm-project/llvm" %
workspace_dir)
repository_ctx.report_progress(
"Running CMake for the Clang toolchain build...",
)
cmake_args = [cmake, "-G", "Ninja", str(llvm_dir)]
cmake_args += _get_cmake_defines(repository_ctx, is_clang)
_run(
repository_ctx,
cmake_args,
timeout = 600,
environment = environment,
# This is very slow, so print output as a form of progress.
quiet = False,
)
# Run ninja for the final build.
repository_ctx.report_progress("Building the Clang toolchain...")
_run(
repository_ctx,
[ninja],
timeout = 10800,
# This is very slow, so print output as a form of progress.
quiet = False,
)
def _download_prebuilt_toolchain(repository_ctx):
"""Downloads and extracts an LLVM build for the current platform.
Returns `True` when a toolchain can be successfully downloaded.
"""
repository_ctx.report_progress("Checking for a downloadable toolchain...")
os = repository_ctx.os.name
if os == "linux":
url = "https://github.com/mmdriley/llvm-builds/releases/download/r32/llvm-linux.tar.xz"
sha256 = "db9f2698aa84935efca3402bdebada127de16f6746adbe54d4cdb7e3b8fec5f3"
elif os == "mac os x":
url = "https://github.com/mmdriley/llvm-builds/releases/download/r32/llvm-macos.tar.xz"
sha256 = "937b81c235977ed2b265baf656f30b7a03c33b6299090d91beb72c2b41846673"
elif os.startswith("windows"):
url = "https://github.com/mmdriley/llvm-builds/releases/download/r32/llvm-windows.tar.xz"
sha256 = "b6b015f9f2fcfb79381004e6a3ae925df4fb827cf7e07f3d5b0b66210fddd172"
else:
print(("No prebuilt LLVM toolcahin to download for {}, falling back " +
"to a local build. This may be very slow!").format(os))
return False
repository_ctx.report_progress("Downloading and extracting a toolchain...")
repository_ctx.download_and_extract(url, sha256 = sha256)
return True
def _bootstrap_clang_toolchain_impl(repository_ctx):
"""Bootstrap a fresh Clang and LLVM toolchain for use.
This will first try to download a pre-built archive of the LLVM toolchain
if one is available.
Otherwise will locally build the toolchain using CMake out of the LLVM
submodule.
"""
force_local_build = False
if FORCE_LOCAL_BOOTSTRAP_ENV in repository_ctx.os.environ:
print("Forcing a local bootstrap build. This may be very slow!")
force_local_build = True
if force_local_build or not _download_prebuilt_toolchain(repository_ctx):
# Fallback to a local build.
_local_cmake_build_clang_toolchain(repository_ctx)
# Create an empty BUILD file to mark the package. The files are used without
# Bazel labels directly pointing at them.
repository_ctx.file("BUILD")
bootstrap_clang_toolchain = repository_rule(
implementation = _bootstrap_clang_toolchain_impl,
configure = True,
attrs = {
# We use a label pointing at the workspace file to compute the
# workspace directory.
"_workspace": attr.label(
default = Label("//:WORKSPACE"),
allow_single_file = True,
),
},
environ = ["CC", "CXX", FORCE_LOCAL_BOOTSTRAP_ENV],
)