carbon-language/carbon-lang
1
0
Fork 0
mirror of https://github.com/carbon-language/carbon-lang.git synced 2026-10-06 08:54:47 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Try out hedronvision for compile_commands.json (#3533)

Browse Source 
Directories in `external/` now include versions of dependencies, so
continuing to make that work would require version-locking the
compile_flags.txt. We've previously seen issues with directories being
forgotten; this switches to letting bazel generate everything.

Setup follows instructions at
https://github.com/hedronvision/bazel-compile-commands-extractor?tab=readme-ov-file#first-add-this-tool-to-your-bazel-setup

Maintains `./scripts/create_compdb.py`. Per discussion, new files seem
to work okay -- I'm getting #include completion that clearly detects
files.

Note this creates an `external` directory:

```
╚╡ls -ld external
lrwxrwxrwx 1 jperkins primarygroup 27 Dec 19 14:02 external -> bazel-out/../../../external
```

I felt sort of weird checking that in though, at least alongside
`bazel-execroot` it feels like it should have a `bazel-` prefix.
However, I can't rename it. So omitting it, and leaving it to the script
to generate, felt like a reasonable compromise.

Running takes a few minutes (about 3 for me), and should be faster on
reruns. It does print a couple warnings, just tree_sitter missing
parser.c and _GNUC_PREREQ errors from m4.
This commit is contained in:
Jon Ross-Perkins
2023-12-21 00:35:17 +00:00
committed by GitHub
parent 1025e5db9d
commit 3d260b92f1
5 changed files with 87 additions and 253 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/create_compdb.py
+33 -130
View File
@@ -23,145 +23,48 @@ Exceptions. See /LICENSE for license information.
SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
"""
import json
import re
import subprocess
import sys
from pathlib import Path
import scripts_utils
scripts_utils.chdir_repo_root()
directory = Path.cwd()
# We use the `BAZEL` environment variable if present. If not, then we try to
# use `bazelisk` and then `bazel`.
bazel = scripts_utils.locate_bazel()
def _build_generated_files(bazel: str) -> None:
print("Building the generated files so that tools can find them...")
# Load compiler flags. We do this first in order to fail fast if not run from
# the workspace root.
print("Reading the arguments to use...")
try:
with open("compile_flags.txt") as flag_file:
arguments = [line.strip() for line in flag_file]
except FileNotFoundError:
sys.exit(Path(sys.argv[0]).name + " must be run from the project root")
# Prepend the `clang` executable path to the arguments that looks into our
# downloaded Clang toolchain.
arguments = ["clang++"] + arguments
print("Building compilation database...")
# Find all of the C++ source files that we expect to compile cleanly as
# stand-alone files. This is a bit simpler than scraping the actual compile
# actions and allows us to directly index header-only libraries easily and
# pro-actively index the specific headers in the project.
source_files_query = subprocess.check_output(
[
bazel,
"query",
"--keep_going",
"--output=location",
# Workaround for https://github.com/bazelbuild/bazel/issues/8900
"--incompatible_display_source_file_location",
'filter(".*\\.(h|cpp|cc|c|cxx)$", kind("source file", deps(//...)))',
],
stderr=subprocess.DEVNULL,
universal_newlines=True,
)
source_files = [
Path(line.split(":")[0]) for line in source_files_query.splitlines()
]
# Filter into the Carbon source files that we'll find directly in the
# workspace, and LLVM source files that need to be mapped through the merged
# LLVM tree in Bazel's execution root.
carbon_files = [
f.relative_to(directory)
for f in source_files
if f.parts[: len(directory.parts)] == directory.parts
]
llvm_files = [
Path("bazel-execroot/external").joinpath(
*f.parts[f.parts.index("llvm-project") :]
# Collect the generated file labels. Include some rules which generate
# files but aren't classified as "generated file".
kinds_query = (
"filter("
' ".*\\.(h|cpp|cc|c|cxx|def|inc)$",'
' kind("generated file", deps(//...))'
")"
" union "
'kind("cc_proto_library", deps(//...))'
)
for f in source_files
if "llvm-project" in f.parts
]
print(
"Found %d Carbon source files and %d LLVM source files..."
% (len(carbon_files), len(llvm_files))
)
generated_file_labels = subprocess.check_output(
[bazel, "query", "--keep_going", "--output=label", kinds_query],
stderr=subprocess.DEVNULL,
encoding="utf-8",
).splitlines()
print(f"Found {len(generated_file_labels)} generated files...")
# Now collect the generated file labels.
# cc_proto_library generates files, but they aren't seen with "generated file".
generated_file_labels = subprocess.check_output(
[
bazel,
"query",
"--keep_going",
"--output=label",
(
'filter(".*\\.(h|cpp|cc|c|cxx|def|inc)$",'
'kind("generated file", deps(//...)))'
" union "
'kind("cc_proto_library", deps(//...))'
),
],
stderr=subprocess.DEVNULL,
universal_newlines=True,
).splitlines()
print("Found %d generated files..." % (len(generated_file_labels),))
# Directly build these labels so that indexing can find them. Allow this to
# fail in case there are build errors in the client, and just warn the user
# that they may be missing generated files.
print("Building the generated files so that tools can find them...")
subprocess.check_call([bazel, "build", "--keep_going"] + generated_file_labels)
# Also build some specific targets that depend on external packages so those are
# fetched and linked into the Bazel execution root. We try to use cheap files
# where possible, but in some cases need to create a virtual include directory.
subprocess.check_call(
[
bazel,
"build",
"--keep_going",
"@llvm-project//llvm:LICENSE.TXT",
"@com_google_absl//:LICENSE",
"@com_google_googletest//:LICENSE",
"@com_googlesource_code_re2//:LICENSE",
"@com_github_google_benchmark//:benchmark",
"@com_google_libprotobuf_mutator//:LICENSE",
"@com_google_protobuf//:any_proto",
]
)
# Directly build these labels so that indexing can find them. Allow this to
# fail in case there are build errors in the client, and just warn the user
# that they may be missing generated files.
subprocess.check_call(
[bazel, "build", "--keep_going"] + generated_file_labels
)
# Manually translate the label to a user friendly path into the Bazel output
# symlinks.
def _label_to_path(s: str) -> Path:
# Map external repositories to their part of the output tree.
s = re.sub(r"^@([^/]+)//", r"bazel-bin/external/\1/", s)
# Map this repository to the root of the output tree.
s = s if not s.startswith("//") else "bazel-bin/" + s[len("//") :]
# Replace the colon used to mark the package name with a slash.
s = s.replace(":", "/")
# Convert to a native path.
return Path(s)
def main() -> None:
scripts_utils.chdir_repo_root()
bazel = scripts_utils.locate_bazel()
_build_generated_files(bazel)
print("Generating compile_commands.json (may take a few minutes)...")
subprocess.run([bazel, "run", "@hedron_compile_commands//:refresh_all"])
generated_files = [_label_to_path(label) for label in generated_file_labels]
# Generate compile_commands.json with an entry for each C++ input.
entries = [
{
"directory": str(directory),
"file": str(f),
"arguments": arguments + [str(f)],
}
for f in carbon_files + llvm_files + generated_files
]
with open("compile_commands.json", "w") as json_file:
json.dump(entries, json_file, indent=2)
if __name__ == "__main__":
main()