From 0d37095ec62e5ce82a82d4146a072ea8417fc5a3 Mon Sep 17 00:00:00 2001 From: Chandler Carruth Date: Sat, 1 Jun 2024 06:23:45 +0200 Subject: [PATCH] Add very minimal support for packaging the toolchain. (#3994) This takes the installation layout and replicates it using `rules_pkg` to build either a tarball or a zip file of the toolchain. Correctly manages file permissions and symlinks, etc. There are some big remaining things here: - Figure out how we want to test this. We can add shell tests maybe? A bit awkward. Nicer would be to make the `//examples` tree build using this rather than the more native-bazel install data, however building these is quite slow and it seems bad to pay that cost constantly so dedicated testing is probably better. For now, I've tested these manually. - Need to add versions to the toolchain and then thread them through here so they install properly as a versioned release. But my primary goal for now is just to be able to validate that the install tree is working outside of Bazel and this does enough for that. The above will be longer-term things. --- MODULE.bazel | 1 + MODULE.bazel.lock | 129 ++++++++++++++++++++-------------------- toolchain/install/BUILD | 62 +++++++++++++++++++ 3 files changed, 127 insertions(+), 65 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index e00974b676ea..01b681e4eb8c 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -22,6 +22,7 @@ http_archive = use_repo_rule( ) bazel_dep(name = "bazel_skylib", version = "1.5.0") +bazel_dep(name = "rules_pkg", version = "0.10.1") bazel_dep(name = "abseil-cpp", version = "20240116.1") bazel_dep(name = "re2", version = "2024-03-01") bazel_dep(name = "googletest", version = "1.14.0.bcr.1") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index e5e038d0730d..dcc50b99c736 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1,6 +1,6 @@ { "lockFileVersion": 6, - "moduleFileHash": "e8ab8befa6e8b97ea295cf5ec5b694124bd41d0842fc98875746a48ee4108eaf", + "moduleFileHash": "439583f1efb19cfa4ef173e2ac4774602be6be5061bc14218f9856f4beb33e07", "flags": { "cmdRegistries": [ "https://bcr.bazel.build/" @@ -55,7 +55,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 65, + "line": 66, "column": 13 } }, @@ -80,7 +80,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 112, + "line": 113, "column": 13 } } @@ -94,7 +94,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 81, + "line": 82, "column": 35 }, "imports": { @@ -111,7 +111,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 126, + "line": 127, "column": 29 }, "imports": { @@ -128,7 +128,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 138, + "line": 139, "column": 23 }, "imports": { @@ -144,7 +144,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 139, + "line": 140, "column": 17 } } @@ -155,6 +155,7 @@ ], "deps": { "bazel_skylib": "bazel_skylib@1.5.0", + "rules_pkg": "rules_pkg@0.10.1", "abseil-cpp": "abseil-cpp@20240116.1", "re2": "re2@2024-03-01", "googletest": "googletest@1.14.0.bcr.1", @@ -206,6 +207,35 @@ } } }, + "rules_pkg@0.10.1": { + "name": "rules_pkg", + "version": "0.10.1", + "key": "rules_pkg@0.10.1", + "repoName": "rules_pkg", + "executionPlatformsToRegister": [], + "toolchainsToRegister": [], + "extensionUsages": [], + "deps": { + "rules_license": "rules_license@0.0.7", + "rules_python": "rules_python@0.31.0", + "bazel_skylib": "bazel_skylib@1.5.0", + "bazel_tools": "bazel_tools@_", + "local_config_platform": "local_config_platform@_" + }, + "repoSpec": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", + "attributes": { + "urls": [ + "https://github.com/bazelbuild/rules_pkg/releases/download/0.10.1/rules_pkg-0.10.1.tar.gz" + ], + "integrity": "sha256-0lCSSi7MUXaAj8TCXVz16eeeY0bXnVqxxJPiieci0dA=", + "strip_prefix": "", + "remote_patches": {}, + "remote_patch_strip": 0 + } + } + }, "abseil-cpp@20240116.1": { "name": "abseil-cpp", "version": "20240116.1", @@ -630,7 +660,7 @@ "rules_cc": "rules_cc@0.0.9", "rules_proto": "rules_proto@6.0.0-rc2", "rules_java": "rules_java@7.4.0", - "rules_pkg": "rules_pkg@0.7.0", + "rules_pkg": "rules_pkg@0.10.1", "com_google_abseil": "abseil-cpp@20240116.1", "zlib": "zlib@1.3.1.bcr.1", "upb": "upb@0.0.0-20220923-a547704", @@ -1109,6 +1139,32 @@ "bazel_tools": "bazel_tools@_" } }, + "rules_license@0.0.7": { + "name": "rules_license", + "version": "0.0.7", + "key": "rules_license@0.0.7", + "repoName": "rules_license", + "executionPlatformsToRegister": [], + "toolchainsToRegister": [], + "extensionUsages": [], + "deps": { + "bazel_tools": "bazel_tools@_", + "local_config_platform": "local_config_platform@_" + }, + "repoSpec": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", + "attributes": { + "urls": [ + "https://github.com/bazelbuild/rules_license/releases/download/0.0.7/rules_license-0.0.7.tar.gz" + ], + "integrity": "sha256-RTHezLkTY5ww5cdRKgVNXYdWmNrrddjPkPKEN1/nw2A=", + "strip_prefix": "", + "remote_patches": {}, + "remote_patch_strip": 0 + } + } + }, "apple_support@1.14.0": { "name": "apple_support", "version": "1.14.0", @@ -1276,32 +1332,6 @@ } } }, - "rules_license@0.0.7": { - "name": "rules_license", - "version": "0.0.7", - "key": "rules_license@0.0.7", - "repoName": "rules_license", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [], - "extensionUsages": [], - "deps": { - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/rules_license/releases/download/0.0.7/rules_license-0.0.7.tar.gz" - ], - "integrity": "sha256-RTHezLkTY5ww5cdRKgVNXYdWmNrrddjPkPKEN1/nw2A=", - "strip_prefix": "", - "remote_patches": {}, - "remote_patch_strip": 0 - } - } - }, "bazel_features@1.2.0": { "name": "bazel_features", "version": "1.2.0", @@ -1449,37 +1479,6 @@ } } }, - "rules_pkg@0.7.0": { - "name": "rules_pkg", - "version": "0.7.0", - "key": "rules_pkg@0.7.0", - "repoName": "rules_pkg", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [], - "extensionUsages": [], - "deps": { - "rules_python": "rules_python@0.31.0", - "bazel_skylib": "bazel_skylib@1.5.0", - "rules_license": "rules_license@0.0.7", - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/rules_pkg/releases/download/0.7.0/rules_pkg-0.7.0.tar.gz" - ], - "integrity": "sha256-iimOgydi7aGDBZfWT+fbWBeKqEzVkm121bdE1lWJQcI=", - "strip_prefix": "", - "remote_patches": { - "https://bcr.bazel.build/modules/rules_pkg/0.7.0/patches/module_dot_bazel.patch": "sha256-4OaEPZwYF6iC71ZTDg6MJ7LLqX7ZA0/kK4mT+4xKqiE=" - }, - "remote_patch_strip": 0 - } - } - }, "upb@0.0.0-20220923-a547704": { "name": "upb", "version": "0.0.0-20220923-a547704", diff --git a/toolchain/install/BUILD b/toolchain/install/BUILD index 5338f15016f5..67c400036e8a 100644 --- a/toolchain/install/BUILD +++ b/toolchain/install/BUILD @@ -5,6 +5,9 @@ load("@bazel_skylib//rules:write_file.bzl", "write_file") load("@llvm-project//llvm:binary_alias.bzl", "binary_alias") load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") +load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files", "pkg_mklink", "strip_prefix") +load("@rules_pkg//pkg:tar.bzl", "pkg_tar") +load("@rules_pkg//pkg:zip.bzl", "pkg_zip") load("symlink_filegroup.bzl", "symlink_filegroup") package(default_visibility = ["//visibility:public"]) @@ -139,3 +142,62 @@ cc_test( "@llvm-project//llvm:Support", ], ) + +# Build rules to construct packaged versions of the toolchain's install. + +pkg_files( + name = "packaging_exe_files", + # We break out the driver and LLD here because we can't easily use file + # groups that contain symlinks, so those are manually handled below. Other + # file groups should be directly included. + srcs = [ + "prefix_root/bin/carbon", + "prefix_root/lib/carbon/llvm/bin/lld", + ], + attributes = pkg_attributes(mode = "0755"), + strip_prefix = strip_prefix.from_pkg("prefix_root"), +) + +# Currently, `rules_pkg` can't replicate symlinks from the main tree. To an +# extent, this is reasonable because we want to be much more explicit about the +# symlink structure in the package where as for the filegroups we're comfortable +# with whatever "just works" for development and testing. +[ + pkg_mklink( + name = "packaging_link_lld_alias_" + bin_name, + link_name = "lib/carbon/llvm/bin/" + bin_name, + target = "lld", + ) + for bin_name in lld_bin_names +] + +pkg_filegroup( + name = "packaging_files", + srcs = [ + ":packaging_exe_files", + ] + [ + ":packaging_link_lld_alias_" + bin_name + for bin_name in lld_bin_names + ], +) + +# TODO: We should add support for injecting a version string into both the +# output filename and the package directory name. +pkg_tar( + name = "carbon_toolchain.tar.bz2.rule", + srcs = [":packaging_files"], + out = "carbon_toolchain.tar.bz2", + extension = "tar.bz2", + package_dir = "carbon_toolchain", + tags = ["manual"], # Slow, exclude from wildcard builds. +) + +# TODO: We should add support for injecting a version string into both the +# output filename and the package directory name. +pkg_zip( + name = "carbon_toolchain.zip.rule", + srcs = [":packaging_files"], + out = "carbon_toolchain.zip", + package_dir = "carbon_toolchain", + tags = ["manual"], # Slow, exclude from wildcard builds. +)