carbon-language/carbon-lang
1
0
Fork 0
mirror of https://github.com/carbon-language/carbon-lang.git synced 2026-10-05 15:51:07 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Add a flag to build a single ASTContext shared across all compilations (#7567)

Browse Source 
Instead of building one Clang `ASTContext` per compilation, the
`--share-cpp-ast` flag causes us to build a single `ASTContext` and
share it across all contexts. One new abstraction is added: `CppDomain`
represents the Carbon-side view of a Clang AST that might be shared
across multiple `SemIR::File`s. This object owns the Clang instance and
the AST.

For now, we have no isolation between the C++ state exposed to different
Carbon compilations, and we have no multiplexing of generated LLVM IR
from C++ into different Carbon compilations, so the mode is not usable
yet. The plan is to keep it behind a flag until it's ready.

Assisted-by: Gemini via Antigravity
This commit is contained in:
Richard Smith
2026-07-28 00:19:40 +00:00
committed by GitHub
parent 4e77f9b6c8
commit 7d89ac98c7
25 changed files with 449 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files
toolchain/check/cpp/generate_ast.cpp
+188 -120
View File
@@ -137,7 +137,9 @@ class CarbonExternalASTSource : public SemIR::ReadOnlyASTSource {
explicit CarbonExternalASTSource(Context* context)
: ReadOnlyASTSource(context->sem_ir()), context_(context) {}
auto StartTranslationUnit(clang::ASTConsumer* consumer) -> void override;
// Builds the top-level C++ namespace `Carbon` and adds it to the translation
// unit.
auto BuildCarbonNamespace() -> void;
// Look up decls for `decl_name` inside `decl_context`, adding the decls to
// `decl_context`. Returns true if any decls were added.
@@ -178,10 +180,6 @@ class CarbonExternalASTSource : public SemIR::ReadOnlyASTSource {
}
private:
// Builds the top-level C++ namespace `Carbon` and adds it to the translation
// unit.
auto BuildCarbonNamespace() -> void;
// Map a Carbon entity to a Clang NamedDecl. Returns null if the entity cannot
// currently be represented in C++.
auto MapInstIdToClangDeclOrType(LookupResult lookup)
@@ -218,11 +216,6 @@ char CarbonExternalASTSource::id;
} // namespace
void CarbonExternalASTSource::StartTranslationUnit(
clang::ASTConsumer* /*Consumer*/) {
BuildCarbonNamespace();
}
auto CarbonExternalASTSource::MapInstIdToClangDeclOrType(LookupResult lookup)
-> std::variant<clang::NamedDecl*, clang::QualType> {
auto target_inst_id = lookup.scope_result.target_inst_id();
@@ -321,16 +314,24 @@ auto CarbonExternalASTSource::BuildCarbonNamespace() -> void {
auto& ast_context = context_->ast_context();
auto* identifier = &ast_context.Idents.get(carbon_namespace_name);
// Create the namespace and add it to the translation unit scope.
auto* decl_context = ast_context.getTranslationUnitDecl();
auto* carbon_cpp_namespace = clang::NamespaceDecl::Create(
ast_context, decl_context, /*Inline=*/false, clang::SourceLocation(),
clang::SourceLocation(), identifier, /*PrevDecl=*/nullptr,
/*Nested=*/false);
decl_context->addDecl(carbon_cpp_namespace);
// We provide custom lookup results within this namespace.
carbon_cpp_namespace->setHasExternalVisibleStorage();
// Check if it already exists.
clang::NamespaceDecl* carbon_cpp_namespace = nullptr;
auto lookup_result = decl_context->lookup(identifier);
if (!lookup_result.empty()) {
carbon_cpp_namespace = cast<clang::NamespaceDecl>(lookup_result.front());
} else {
// Create it if it doesn't exist.
carbon_cpp_namespace = clang::NamespaceDecl::Create(
ast_context, decl_context, /*Inline=*/false, clang::SourceLocation(),
clang::SourceLocation(), identifier, /*PrevDecl=*/nullptr,
/*Nested=*/false);
decl_context->addDecl(carbon_cpp_namespace);
// We provide custom lookup results within this namespace.
carbon_cpp_namespace->setHasExternalVisibleStorage();
}
// Register this file's package scope as corresponding to the `Carbon`
// namespace in C++.
@@ -610,34 +611,65 @@ static auto ParseTopLevelDecls(clang::Parser& parser,
}
}
// Injects the C++ code in `buffer` into the Clang preprocessor and parses it
// as top-level declarations. Returns true on success, false if entering the
// source file fails.
static auto InjectAndParse(Context& context,
std::unique_ptr<llvm::MemoryBuffer> buffer) -> bool {
auto* cpp_context = context.cpp_context();
CARBON_CHECK(cpp_context);
clang::Sema& sema = cpp_context->sema();
clang::Preprocessor& preprocessor = sema.getPreprocessor();
clang::Parser& parser = cpp_context->parser();
clang::FileID file_id =
preprocessor.getSourceManager().createFileID(std::move(buffer));
if (preprocessor.EnterSourceFile(file_id, nullptr, clang::SourceLocation())) {
return false;
}
if (parser.getCurToken().is(clang::tok::eof)) {
parser.ConsumeToken();
}
ParseTopLevelDecls(parser, sema.getASTConsumer());
return true;
}
namespace {
// An action and a set of registered Clang callbacks used to generate an AST
// from a set of Cpp imports.
class GenerateASTAction : public clang::ASTFrontendAction {
public:
explicit GenerateASTAction(
Context& context,
std::unique_ptr<CppDiagnosticListener> diagnostic_listener)
: context_(&context),
diagnostic_listener_(std::move(diagnostic_listener)) {}
explicit GenerateASTAction(llvm::StringRef filename,
llvm::LLVMContext* llvm_context)
: filename_(filename), llvm_context_(llvm_context) {}
auto code_generator() const -> clang::CodeGenerator* {
return code_generator_;
}
auto TakeParser() -> std::unique_ptr<clang::Parser> {
return std::move(parser_);
}
protected:
auto CreateASTConsumer(clang::CompilerInstance& clang_instance,
llvm::StringRef /*file*/)
-> std::unique_ptr<clang::ASTConsumer> override {
auto& cpp_file = *context_->sem_ir().cpp_file();
if (!cpp_file.llvm_context()) {
if (!llvm_context_) {
return std::make_unique<clang::ASTConsumer>();
}
auto code_generator =
std::unique_ptr<clang::CodeGenerator>(clang::CreateLLVMCodeGen(
cpp_file.diagnostics(), context_->sem_ir().filename(),
clang_instance.getDiagnostics(), filename_,
clang_instance.getVirtualFileSystemPtr(),
clang_instance.getHeaderSearchOpts(),
clang_instance.getPreprocessorOpts(),
clang_instance.getCodeGenOpts(), *cpp_file.llvm_context()));
cpp_file.SetCodeGenerator(code_generator.get());
clang_instance.getCodeGenOpts(), *llvm_context_));
code_generator_ = code_generator.get();
return code_generator;
}
@@ -657,18 +689,12 @@ class GenerateASTAction : public clang::ASTFrontendAction {
clang_instance.createSema(getTranslationUnitKind(),
/*CompletionConsumer=*/nullptr);
auto parser_ptr = std::make_unique<clang::Parser>(
clang_instance.getPreprocessor(), clang_instance.getSema(),
/*SkipFunctionBodies=*/false);
auto& parser = *parser_ptr;
parser_ = std::make_unique<clang::Parser>(clang_instance.getPreprocessor(),
clang_instance.getSema(),
/*SkipFunctionBodies=*/false);
clang_instance.getPreprocessor().EnterMainSourceFile();
parser.Initialize();
auto cpp_context =
std::make_unique<CppContext>(clang_instance, std::move(parser_ptr));
cpp_context->set_diagnostic_listener(std::move(diagnostic_listener_));
context_->set_cpp_context(std::move(cpp_context));
parser_->Initialize();
if (auto* source = clang_instance.getASTContext().getExternalSource()) {
source->StartTranslationUnit(&clang_instance.getASTConsumer());
@@ -676,40 +702,47 @@ class GenerateASTAction : public clang::ASTFrontendAction {
clang_instance.getSema().ActOnStartOfTranslationUnit();
ParseTopLevelDecls(parser, clang_instance.getASTConsumer());
ParseTopLevelDecls(*parser_, clang_instance.getASTConsumer());
}
private:
Context* context_;
std::unique_ptr<CppDiagnosticListener> diagnostic_listener_;
std::string filename_;
llvm::LLVMContext* llvm_context_;
clang::CodeGenerator* code_generator_ = nullptr;
std::unique_ptr<clang::Parser> parser_;
};
} // namespace
auto GenerateAst(Context& context,
llvm::ArrayRef<Parse::Tree::PackagingNames> imports,
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fs,
llvm::LLVMContext* llvm_context,
std::shared_ptr<clang::CompilerInvocation> base_invocation)
-> bool {
CARBON_CHECK(!context.cpp_context());
CARBON_CHECK(!context.sem_ir().cpp_file());
// Initializes the Clang state by building a new compiler invocation,
// creating a diagnostics engine, and parsing a dummy main file containing a
// semicolon. Returns the initialized state, or null on failure.
auto InitializeCppDomain(
Diagnostics::Consumer& consumer, llvm::StringRef filename,
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fs,
llvm::LLVMContext* llvm_context,
std::shared_ptr<clang::CompilerInvocation> base_invocation)
-> std::shared_ptr<CppDomain> {
std::shared_ptr<clang::CompilerInstance> clang_instance;
llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diags;
// Build a new invocation.
auto invocation =
std::make_shared<ShallowCopyCompilerInvocation>(*base_invocation);
// Ask Clang to not leak memory.
invocation->getFrontendOpts().DisableFree = false;
auto diagnostic_consumer =
MakeDiagnosticConsumer(context.emitter().consumer(), invocation);
auto* diagnostic_consumer_ptr = diagnostic_consumer.get();
// Build a diagnostics engine.
llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diags(
clang::CompilerInstance::createDiagnostics(
*fs, invocation->getDiagnosticOpts(), diagnostic_consumer.release(),
/*ShouldOwnClient=*/true));
diags = clang::CompilerInstance::createDiagnostics(
*fs, invocation->getDiagnosticOpts(),
MakeDiagnosticConsumer(consumer, invocation).release(),
/*ShouldOwnClient=*/true);
// Ensure any diagnostics emitted in this function are flushed before we
// return.
auto on_exit =
llvm::scope_exit([&]() { FlushDiagnosticConsumer(*diags->getClient()); });
// Extract the input from the frontend invocation and make sure it makes
// sense.
@@ -719,47 +752,31 @@ auto GenerateAst(Context& context,
inputs[0].getKind().getFormat() == clang::InputKind::Source);
llvm::StringRef file_name = inputs[0].getFile();
// Remap the imports file name to the corresponding `#include`s.
// TODO: Modify the frontend options to specify this memory buffer as input
// instead of remapping the file.
std::string includes = GenerateCppIncludesHeaderCode(context, imports);
auto includes_buffer =
llvm::MemoryBuffer::getMemBufferCopy(includes, file_name);
// Remap the input file to a dummy buffer containing a semicolon to start
// with an empty AST. Clang requires at least one token in the main file
// to avoid assertion failures if it later encounters module declarations.
// TODO: See if we can fix this by injecting code into the main file rather
// than entering nested buffers.
auto empty_buffer = llvm::MemoryBuffer::getMemBuffer(";");
invocation->getPreprocessorOpts().addRemappedFile(file_name,
includes_buffer.release());
empty_buffer.release());
auto clang_instance_ptr =
std::make_unique<clang::CompilerInstance>(invocation);
auto& clang_instance = *clang_instance_ptr;
context.sem_ir().set_cpp_file(std::make_unique<SemIR::CppFile>(
std::move(clang_instance_ptr), llvm_context));
clang_instance = std::make_shared<clang::CompilerInstance>(invocation);
// Register an annotation scope to flush any Clang diagnostics when we return.
// This ensures C++ diagnostics get flushed before `diags` is destroyed, and
// that diagnostics created here don't interleave with later Carbon
// diagnostics.
Diagnostics::AnnotationScope annotate_diagnostics(&context.emitter(),
[](auto& /*builder*/) {});
clang_instance.setDiagnostics(diags);
clang_instance.setVirtualFileSystem(fs);
clang_instance.createFileManager();
clang_instance.createSourceManager();
if (!clang_instance.createTarget()) {
return false;
clang_instance->setDiagnostics(diags);
clang_instance->setVirtualFileSystem(fs);
clang_instance->createFileManager();
clang_instance->createSourceManager();
if (!clang_instance->createTarget()) {
return nullptr;
}
GenerateASTAction action(context, MakeContextDiagnosticListener(
*diagnostic_consumer_ptr, context));
if (!action.BeginSourceFile(clang_instance, inputs[0])) {
return false;
GenerateASTAction action(filename, llvm_context);
if (!action.BeginSourceFile(*clang_instance, inputs[0])) {
return nullptr;
}
// The AST context is now available, so the mangle context (used to compute
// stable identities for imported C++ types) can be created.
context.sem_ir().cpp_file()->CreateMangleContext();
auto& ast = clang_instance.getASTContext();
auto& ast = clang_instance->getASTContext();
// Always build a multiplex source, even if there's only one child
// source. During lowering, the `CarbonExternalASTSource` can no longer be
@@ -777,18 +794,73 @@ auto GenerateAst(Context& context,
ast.getExternalSource())) {
multiplex_source->AddSource(existing_source);
}
multiplex_source->AddSource(
llvm::makeIntrusiveRefCnt<CarbonExternalASTSource>(&context));
ast.setExternalSource(std::move(multiplex_source_ref_cnt_ptr));
if (llvm::Error error = action.Execute()) {
// `Execute` currently never fails, but its contract allows it to.
context.TODO(SemIR::LocId::None, "failed to execute clang action: " +
llvm::toString(std::move(error)));
return false;
CARBON_FATAL("Failed to execute clang action: {0}",
llvm::toString(std::move(error)));
}
return true;
auto parser = action.TakeParser();
CARBON_CHECK(parser);
return std::make_shared<CppDomain>(
CppDomain{.clang_instance = std::move(clang_instance),
.parser = std::move(parser),
.code_generator = action.code_generator(),
.llvm_context = llvm_context});
}
auto GenerateAst(Context& context,
llvm::ArrayRef<Parse::Tree::PackagingNames> imports,
CppDomain& domain) -> bool {
CARBON_CHECK(!context.cpp_context());
CARBON_CHECK(!context.sem_ir().cpp_file());
// Register an annotation scope to flush any Clang diagnostics when we
// return. This ensures C++ diagnostics get flushed before `diags` is
// destroyed, and that diagnostics created here don't interleave with later
// Carbon diagnostics.
Diagnostics::AnnotationScope annotate_diagnostics(&context.emitter(),
[](auto& /*builder*/) {});
auto clang_instance = domain.clang_instance;
auto parser = domain.parser;
// Set up CppFile for the current SemIR::File.
auto cpp_file =
std::make_unique<SemIR::CppFile>(clang_instance, domain.llvm_context);
if (domain.code_generator) {
cpp_file->SetCodeGenerator(domain.code_generator);
}
context.sem_ir().set_cpp_file(std::move(cpp_file));
// Set up CppContext for the current Context.
context.set_cpp_context(std::make_unique<CppContext>(
*clang_instance, parser,
MakeContextDiagnosticListener(
*clang_instance->getDiagnostics().getClient(), context)));
// The AST context is now available, so the mangle context (used to compute
// stable identities for imported C++ types) can be created.
context.sem_ir().cpp_file()->CreateMangleContext();
// Add an external source referring to this context.
auto* multiplex_source = cast<clang::MultiplexExternalSemaSource>(
context.ast_context().getExternalSource());
auto ast_source =
llvm::makeIntrusiveRefCnt<CarbonExternalASTSource>(&context);
multiplex_source->AddSource(ast_source);
// Map the package scope to the Carbon namespace.
ast_source->BuildCarbonNamespace();
// Inject the imports-as-#includes buffer.
std::string includes = GenerateCppIncludesHeaderCode(context, imports);
auto buffer =
llvm::MemoryBuffer::getMemBufferCopy(includes, "<shared cpp imports>");
return InjectAndParse(context, std::move(buffer));
}
auto InjectAstFromInlineCode(Context& context, SemIR::LocId loc_id,
@@ -796,10 +868,6 @@ auto InjectAstFromInlineCode(Context& context, SemIR::LocId loc_id,
auto* cpp_context = context.cpp_context();
CARBON_CHECK(cpp_context);
clang::Sema& sema = cpp_context->sema();
clang::Preprocessor& preprocessor = sema.getPreprocessor();
clang::Parser& parser = cpp_context->parser();
RawStringOstream code_stream;
AppendInlineCode(context, code_stream,
context.parse_tree().node_token(loc_id.node_id()),
@@ -807,22 +875,9 @@ auto InjectAstFromInlineCode(Context& context, SemIR::LocId loc_id,
auto buffer = llvm::MemoryBuffer::getMemBufferCopy(code_stream.TakeStr(),
"<inline c++>");
clang::FileID file_id =
preprocessor.getSourceManager().createFileID(std::move(buffer));
if (preprocessor.EnterSourceFile(file_id, nullptr, clang::SourceLocation())) {
// Clang will have generated a suitable error. There's nothing more to do
// here.
return;
}
// The parser will typically have an EOF as its cached current token; consume
// that so we can reach the newly-injected tokens.
if (parser.getCurToken().is(clang::tok::eof)) {
parser.ConsumeToken();
}
ParseTopLevelDecls(parser, sema.getASTConsumer());
// Clang will have generated a suitable error if this fails. There's nothing
// more to do here.
InjectAndParse(context, std::move(buffer));
}
auto FinishAst(Context& context) -> void {
@@ -830,7 +885,13 @@ auto FinishAst(Context& context) -> void {
return;
}
context.cpp_context()->sema().ActOnEndOfTranslationUnit();
// Finalize the per-Context AST fragment. The final ActOnEndOfTranslationUnit
// call for the CppDomain is performed in FinalizeCppDomain once all files
// sharing the domain have been checked.
context.cpp_context()->sema().ActOnEndOfTranslationUnitFragment(
clang::TUFragmentKind::Normal);
FlushDiagnosticConsumer(
*context.cpp_context()->sema().getDiagnostics().getClient());
context.emitter().Flush();
// Remove the `CarbonExternalASTSource` installed in `GenerateAst` and
@@ -841,8 +902,7 @@ auto FinishAst(Context& context) -> void {
auto* multiplex_source = cast<clang::MultiplexExternalSemaSource>(
context.ast_context().getExternalSource());
multiplex_source->EraseIf([](const auto& src) {
// `CarbonExternalASTSource` inherits from `ReadOnlyASTSource`.
return llvm::isa<SemIR::ReadOnlyASTSource>(src.get());
return llvm::isa<CarbonExternalASTSource>(src.get());
});
multiplex_source->AddSource(
llvm::makeIntrusiveRefCnt<SemIR::ReadOnlyASTSource>(context.sem_ir()));
@@ -851,4 +911,12 @@ auto FinishAst(Context& context) -> void {
context.set_cpp_context(nullptr);
}
auto FinalizeCppDomain(CppDomain& domain) -> void {
if (domain.clang_instance) {
domain.clang_instance->getSema().ActOnEndOfTranslationUnit();
FlushDiagnosticConsumer(
*domain.clang_instance->getDiagnostics().getClient());
}
}
} // namespace Carbon::Check