From 6114df59eeac3904ab5f13534280a0f63520931d Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Wed, 10 Dec 2025 10:55:22 -0800 Subject: [PATCH] Factor out a `Check::CppContext` holding C++-specific check state (#6482) * Move `Sema` access from `CppFile` into `CppContext`. * Move the mangle context from `SemIR::File` into `CppContext`. * Move source location mapping state from `Context` into `CppContext`. Also factor out the `GenerateAst` function that builds the `CppContext` and `CppFile` into its own file. --- toolchain/check/BUILD | 4 + toolchain/check/check_unit.cpp | 4 +- toolchain/check/context.h | 25 +- toolchain/check/cpp/context.cpp | 22 ++ toolchain/check/cpp/context.h | 51 ++++ toolchain/check/cpp/generate_ast.cpp | 385 +++++++++++++++++++++++++++ toolchain/check/cpp/generate_ast.h | 26 ++ toolchain/check/cpp/import.cpp | 360 +------------------------ toolchain/check/cpp/location.cpp | 2 +- toolchain/check/cpp/thunk.cpp | 2 +- toolchain/sem_ir/cpp_file.h | 3 +- toolchain/sem_ir/file.cpp | 2 - toolchain/sem_ir/file.h | 7 - 13 files changed, 512 insertions(+), 381 deletions(-) create mode 100644 toolchain/check/cpp/context.cpp create mode 100644 toolchain/check/cpp/context.h create mode 100644 toolchain/check/cpp/generate_ast.cpp create mode 100644 toolchain/check/cpp/generate_ast.h diff --git a/toolchain/check/BUILD b/toolchain/check/BUILD index 8c96157a17c2..5919dbcc1b7f 100644 --- a/toolchain/check/BUILD +++ b/toolchain/check/BUILD @@ -23,7 +23,10 @@ cc_library( "convert.cpp", "cpp/access.cpp", "cpp/call.cpp", + "cpp/context.cpp", "cpp/custom_type_mapping.cpp", + "cpp/generate_ast.cpp", + "cpp/generate_ast.h", "cpp/impl_lookup.cpp", "cpp/import.cpp", "cpp/location.cpp", @@ -78,6 +81,7 @@ cc_library( "convert.h", "cpp/access.h", "cpp/call.h", + "cpp/context.h", "cpp/custom_type_mapping.h", "cpp/impl_lookup.h", "cpp/import.h", diff --git a/toolchain/check/check_unit.cpp b/toolchain/check/check_unit.cpp index 97520369c773..439df3d58de2 100644 --- a/toolchain/check/check_unit.cpp +++ b/toolchain/check/check_unit.cpp @@ -580,10 +580,10 @@ auto CheckUnit::FinishRun() -> void { CheckPoisonedConcreteImplLookupQueries(); CheckImpls(); - if (auto* cpp_file = context_.sem_ir().cpp_file()) { + if (auto* cpp_context = context_.cpp_context()) { // Ask Clang to perform any cleanups required, including instantiating used // templates. - cpp_file->sema().ActOnEndOfTranslationUnit(); + cpp_context->sema().ActOnEndOfTranslationUnit(); context_.emitter().Flush(); } diff --git a/toolchain/check/context.h b/toolchain/check/context.h index fbefa66bf20c..13f551e77870 100644 --- a/toolchain/check/context.h +++ b/toolchain/check/context.h @@ -12,6 +12,7 @@ #include "llvm/ADT/SmallVector.h" #include "toolchain/base/canonical_value_store.h" #include "toolchain/base/value_store.h" +#include "toolchain/check/cpp/context.h" #include "toolchain/check/decl_introducer_state.h" #include "toolchain/check/decl_name_stack.h" #include "toolchain/check/deferred_definition_worklist.h" @@ -84,6 +85,14 @@ class Context { auto sem_ir() -> SemIR::File& { return *sem_ir_; } auto sem_ir() const -> const SemIR::File& { return *sem_ir_; } + auto cpp_context() -> CppContext* { return cpp_context_.get(); } + + // TODO: Remove this and pass the C++ context to the constructor. + auto set_cpp_context(std::unique_ptr cpp_context) { + CARBON_CHECK(cpp_context, "C++ context set more than once"); + cpp_context_ = std::move(cpp_context); + } + // Convenience functions for major phase data. auto parse_tree() const -> const Parse::Tree& { return sem_ir_->parse_tree(); @@ -158,11 +167,6 @@ class Context { return import_ir_constant_values_; } - auto cpp_carbon_file_locations() - -> llvm::SmallVector& { - return cpp_carbon_file_locations_; - } - auto definitions_required_by_decl() -> llvm::SmallVector& { return definitions_required_by_decl_; } @@ -330,9 +334,9 @@ class Context { return sem_ir().import_ir_insts(); } auto ast_context() -> clang::ASTContext& { - return sem_ir().cpp_file()->ast_context(); + return cpp_context()->ast_context(); } - auto clang_sema() -> clang::Sema& { return sem_ir().cpp_file()->sema(); } + auto clang_sema() -> clang::Sema& { return cpp_context()->sema(); } auto clang_decls() -> SemIR::ClangDeclStore& { return sem_ir().clang_decls(); } @@ -374,6 +378,9 @@ class Context { // The total number of files. int total_ir_count_; + // The C++ checking context. + std::unique_ptr cpp_context_; + // Whether to print verbose output. llvm::raw_ostream* vlog_stream_; @@ -440,10 +447,6 @@ class Context { // Inline 0 elements because it's expected to require heap allocation. llvm::SmallVector import_ir_constant_values_; - // Per-Carbon-file start locations for corresponding Clang source buffers. - // Owned and managed by code in cpp/location.cpp. - llvm::SmallVector cpp_carbon_file_locations_; - // Declaration instructions of entities that should have definitions by the // end of the current source file. llvm::SmallVector definitions_required_by_decl_; diff --git a/toolchain/check/cpp/context.cpp b/toolchain/check/cpp/context.cpp new file mode 100644 index 000000000000..5a651f6911b4 --- /dev/null +++ b/toolchain/check/cpp/context.cpp @@ -0,0 +1,22 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "toolchain/check/cpp/context.h" + +#include "clang/AST/Mangle.h" + +namespace Carbon::Check { + +CppContext::CppContext(clang::ASTUnit* ast_unit) : ast_unit_(ast_unit) {} + +CppContext::~CppContext() = default; + +auto CppContext::clang_mangle_context() -> clang::MangleContext& { + if (!clang_mangle_context_) { + clang_mangle_context_.reset(ast_context().createMangleContext()); + } + return *clang_mangle_context_; +} + +} // namespace Carbon::Check diff --git a/toolchain/check/cpp/context.h b/toolchain/check/cpp/context.h new file mode 100644 index 000000000000..0162ebeadd57 --- /dev/null +++ b/toolchain/check/cpp/context.h @@ -0,0 +1,51 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef CARBON_TOOLCHAIN_CHECK_CPP_CONTEXT_H_ +#define CARBON_TOOLCHAIN_CHECK_CPP_CONTEXT_H_ + +#include + +#include "clang/Basic/SourceLocation.h" +#include "clang/Frontend/ASTUnit.h" +#include "llvm/ADT/SmallVector.h" + +namespace Carbon::Check { + +// Context for C++ code during check. +// +// This stores state for a Clang AST and Sema, as well as any additional +// information needed to perform mapping between Carbon and C++ types, +// declarations, and similar values. +class CppContext { + public: + explicit CppContext(clang::ASTUnit* ast_unit); + ~CppContext(); + + auto ast_context() -> clang::ASTContext& { + return ast_unit_->getASTContext(); + } + auto sema() -> clang::Sema& { return ast_unit_->getSema(); } + + auto clang_mangle_context() -> clang::MangleContext&; + + auto carbon_file_locations() -> llvm::SmallVector& { + return carbon_file_locations_; + } + + private: + // The ASTUnit is owned by the `CppFile`. + clang::ASTUnit* ast_unit_; + + // Per-Carbon-file start locations for corresponding Clang source buffers. + // Owned and managed by code in location.cpp. + llvm::SmallVector carbon_file_locations_; + + // The Clang mangle context for the target in the ASTContext. + std::unique_ptr clang_mangle_context_; +}; + +} // namespace Carbon::Check + +#endif // CARBON_TOOLCHAIN_CHECK_CPP_CONTEXT_H_ diff --git a/toolchain/check/cpp/generate_ast.cpp b/toolchain/check/cpp/generate_ast.cpp new file mode 100644 index 000000000000..64be332ef683 --- /dev/null +++ b/toolchain/check/cpp/generate_ast.cpp @@ -0,0 +1,385 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "toolchain/check/cpp/generate_ast.h" + +#include +#include + +#include "clang/AST/ASTContext.h" +#include "clang/Basic/FileManager.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/CompilerInvocation.h" +#include "clang/Frontend/TextDiagnostic.h" +#include "clang/Lex/PreprocessorOptions.h" +#include "common/check.h" +#include "common/raw_string_ostream.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/raw_ostream.h" +#include "toolchain/check/context.h" +#include "toolchain/diagnostics/diagnostic.h" +#include "toolchain/diagnostics/diagnostic_emitter.h" +#include "toolchain/diagnostics/format_providers.h" +#include "toolchain/parse/node_ids.h" + +namespace Carbon::Check { + +// Add a line marker directive pointing at the location of the `import Cpp` +// declaration in the Carbon source file. This will cause Clang's diagnostics +// machinery to track and report the location in Carbon code where the import +// was written. +static auto GenerateLineMarker(Context& context, llvm::raw_ostream& out, + int line) { + out << "# " << line << " \"" + << FormatEscaped(context.tokens().source().filename()) << "\"\n"; +} + +// Generates C++ file contents to #include all requested imports. +static auto GenerateCppIncludesHeaderCode( + Context& context, llvm::ArrayRef imports) + -> std::string { + std::string code; + llvm::raw_string_ostream code_stream(code); + for (const Parse::Tree::PackagingNames& import : imports) { + if (import.inline_body_id.has_value()) { + // Expand `import Cpp inline "code";` directly into the specified code. + auto code_token = context.parse_tree().node_token(import.inline_body_id); + + // Compute the line number on which the C++ code starts. Usually the code + // is specified as a block string literal and starts on the line after the + // start of the string token. + // TODO: Determine if this is a block string literal without calling + // `GetTokenText`, which re-lexes the string. + int line = context.tokens().GetLineNumber(code_token); + if (context.tokens().GetTokenText(code_token).contains('\n')) { + ++line; + } + + GenerateLineMarker(context, code_stream, line); + code_stream << context.string_literal_values().Get( + context.tokens().GetStringLiteralValue(code_token)) + << "\n"; + // TODO: Inject a clang pragma here to produce an error if there are + // unclosed scopes at the end of this inline C++ fragment. + } else if (import.library_id.has_value()) { + // Translate `import Cpp library "foo.h";` into `#include "foo.h"`. + GenerateLineMarker(context, code_stream, + context.tokens().GetLineNumber( + context.parse_tree().node_token(import.node_id))); + auto name = context.string_literal_values().Get(import.library_id); + if (name.starts_with('<') && name.ends_with('>')) { + code_stream << "#include <" + << FormatEscaped(name.drop_front().drop_back()) << ">\n"; + } else { + code_stream << "#include \"" << FormatEscaped(name) << "\"\n"; + } + } + } + + // Inject a declaration of placement operator new, because the code we + // generate in thunks depends on it for placement new expressions. Clang has + // special-case logic for lowering a new-expression using this, so a + // definition is not required. + // TODO: This is a hack. We should be able to directly generate Clang AST to + // construct objects in-place without this. + // TODO: Once we can rely on libc++ being available, consider including + // `<__new/placement_new_delete.h>` instead. + code_stream << R"(# 1 "" +#undef constexpr +#if __cplusplus > 202302L +constexpr +#endif +#undef void +#undef operator +#undef new +void* operator new(__SIZE_TYPE__, void*) +#if __cplusplus < 201103L +#undef throw +throw() +#else +#undef noexcept +noexcept +#endif +; +)"; + + return code; +} + +// Adds the given source location and an `ImportIRInst` referring to it in +// `ImportIRId::Cpp`. +static auto AddImportIRInst(SemIR::File& file, + clang::SourceLocation clang_source_loc) + -> SemIR::ImportIRInstId { + SemIR::ClangSourceLocId clang_source_loc_id = + file.clang_source_locs().Add(clang_source_loc); + return file.import_ir_insts().Add(SemIR::ImportIRInst(clang_source_loc_id)); +} + +namespace { + +// Used to convert Clang diagnostics to Carbon diagnostics. +// +// Handling of Clang notes is a little subtle: as far as Clang is concerned, +// notes are separate diagnostics, not connected to the error or warning that +// precedes them. But in Carbon's diagnostics system, notes are part of the +// enclosing diagnostic. To handle this, we buffer Clang diagnostics until we +// reach a point where we know we're not in the middle of a diagnostic, and then +// emit a diagnostic along with all of its notes. This is triggered when adding +// or removing a Carbon context note, which could otherwise get attached to the +// wrong C++ diagnostics, and at the end of the Carbon program. +class CarbonClangDiagnosticConsumer : public clang::DiagnosticConsumer { + public: + // Creates an instance with the location that triggers calling Clang. The + // `context` is not stored here, and the diagnostics consumer is expected to + // outlive it. + explicit CarbonClangDiagnosticConsumer( + Context& context, std::shared_ptr invocation) + : sem_ir_(&context.sem_ir()), + emitter_(&context.emitter()), + invocation_(std::move(invocation)) { + emitter_->AddFlushFn([this] { EmitDiagnostics(); }); + } + + ~CarbonClangDiagnosticConsumer() override { + // Do not inspect `emitter_` here; it's typically destroyed before the + // consumer is. + // TODO: If Clang produces diagnostics after check finishes, they'll get + // added to the list of pending diagnostics and never emitted. + CARBON_CHECK(diagnostic_infos_.empty(), + "Missing flush before destroying diagnostic consumer"); + } + + // Generates a Carbon warning for each Clang warning and a Carbon error for + // each Clang error or fatal. + auto HandleDiagnostic(clang::DiagnosticsEngine::Level diag_level, + const clang::Diagnostic& info) -> void override { + DiagnosticConsumer::HandleDiagnostic(diag_level, info); + + SemIR::ImportIRInstId clang_import_ir_inst_id = + AddImportIRInst(*sem_ir_, info.getLocation()); + + llvm::SmallString<256> message; + info.FormatDiagnostic(message); + + // Render a code snippet including any highlighted ranges and fixit hints. + // TODO: Also include the #include stack and macro expansion stack in the + // diagnostic output in some way. + RawStringOstream snippet_stream; + if (!info.hasSourceManager()) { + // If we don't have a source manager, this is an error from early in the + // frontend. Don't produce a snippet. + CARBON_CHECK(info.getLocation().isInvalid()); + } else { + CodeContextRenderer(snippet_stream, invocation_->getLangOpts(), + invocation_->getDiagnosticOpts()) + .emitDiagnostic( + clang::FullSourceLoc(info.getLocation(), info.getSourceManager()), + diag_level, message, info.getRanges(), info.getFixItHints()); + } + + diagnostic_infos_.push_back({.level = diag_level, + .import_ir_inst_id = clang_import_ir_inst_id, + .message = message.str().str(), + .snippet = snippet_stream.TakeStr()}); + } + + // Returns the diagnostic to use for a given Clang diagnostic level. + static auto GetDiagnostic(clang::DiagnosticsEngine::Level level) + -> const Diagnostics::DiagnosticBase& { + switch (level) { + case clang::DiagnosticsEngine::Ignored: { + CARBON_FATAL("Emitting an ignored diagnostic"); + break; + } + case clang::DiagnosticsEngine::Note: { + CARBON_DIAGNOSTIC(CppInteropParseNote, Note, "{0}", std::string); + return CppInteropParseNote; + } + case clang::DiagnosticsEngine::Remark: + case clang::DiagnosticsEngine::Warning: { + // TODO: Add a distinct Remark level to Carbon diagnostics, and stop + // mapping remarks to warnings. + CARBON_DIAGNOSTIC(CppInteropParseWarning, Warning, "{0}", std::string); + return CppInteropParseWarning; + } + case clang::DiagnosticsEngine::Error: + case clang::DiagnosticsEngine::Fatal: { + CARBON_DIAGNOSTIC(CppInteropParseError, Error, "{0}", std::string); + return CppInteropParseError; + } + } + } + + // Outputs Carbon diagnostics based on the collected Clang diagnostics. Must + // be called after the AST is set in the context. + auto EmitDiagnostics() -> void { + CARBON_CHECK( + sem_ir_->cpp_file(), + "Attempted to emit C++ diagnostics before the C++ file is set"); + + for (size_t i = 0; i != diagnostic_infos_.size(); ++i) { + const ClangDiagnosticInfo& info = diagnostic_infos_[i]; + auto builder = emitter_->Build(SemIR::LocId(info.import_ir_inst_id), + GetDiagnostic(info.level), info.message); + builder.OverrideSnippet(info.snippet); + for (; i + 1 < diagnostic_infos_.size() && + diagnostic_infos_[i + 1].level == clang::DiagnosticsEngine::Note; + ++i) { + const ClangDiagnosticInfo& note_info = diagnostic_infos_[i + 1]; + builder + .Note(SemIR::LocId(note_info.import_ir_inst_id), + GetDiagnostic(note_info.level), note_info.message) + .OverrideSnippet(note_info.snippet); + } + // TODO: This will apply all current Carbon annotation functions. We + // should instead track how Clang's context notes and Carbon's annotation + // functions are interleaved, and interleave the notes in the same order. + builder.Emit(); + } + diagnostic_infos_.clear(); + } + + private: + // A diagnostics renderer based on clang's TextDiagnostic that captures just + // the code context (the snippet). + class CodeContextRenderer : public clang::TextDiagnostic { + protected: + using TextDiagnostic::TextDiagnostic; + + void emitDiagnosticMessage( + clang::FullSourceLoc /*loc*/, clang::PresumedLoc /*ploc*/, + clang::DiagnosticsEngine::Level /*level*/, llvm::StringRef /*message*/, + llvm::ArrayRef /*ranges*/, + clang::DiagOrStoredDiag /*info*/) override {} + void emitDiagnosticLoc( + clang::FullSourceLoc /*loc*/, clang::PresumedLoc /*ploc*/, + clang::DiagnosticsEngine::Level /*level*/, + llvm::ArrayRef /*ranges*/) override {} + + // emitCodeContext is inherited from clang::TextDiagnostic. + + void emitIncludeLocation(clang::FullSourceLoc /*loc*/, + clang::PresumedLoc /*ploc*/) override {} + void emitImportLocation(clang::FullSourceLoc /*loc*/, + clang::PresumedLoc /*ploc*/, + llvm::StringRef /*module_name*/) override {} + void emitBuildingModuleLocation(clang::FullSourceLoc /*loc*/, + clang::PresumedLoc /*ploc*/, + llvm::StringRef /*module_name*/) override {} + + // beginDiagnostic and endDiagnostic are inherited from + // clang::TextDiagnostic in case it wants to do any setup / teardown work. + }; + + // Information on a Clang diagnostic that can be converted to a Carbon + // diagnostic. + struct ClangDiagnosticInfo { + // The Clang diagnostic level. + clang::DiagnosticsEngine::Level level; + + // The ID of the ImportIR instruction referring to the Clang source + // location. + SemIR::ImportIRInstId import_ir_inst_id; + + // The Clang diagnostic textual message. + std::string message; + + // The code snippet produced by clang. + std::string snippet; + }; + + // The Carbon file that this C++ compilation is attached to. + SemIR::File* sem_ir_; + + // The diagnostic emitter that we're emitting diagnostics into. + DiagnosticEmitterBase* emitter_; + + // The compiler invocation that is producing the diagnostics. + std::shared_ptr invocation_; + + // Collects the information for all Clang diagnostics to be converted to + // Carbon diagnostics after the context has been initialized with the Clang + // AST. + llvm::SmallVector diagnostic_infos_; +}; + +// A wrapper around a clang::CompilerInvocation that allows us to make a shallow +// copy of most of the invocation and only make a deep copy of the parts that we +// want to change. +// +// clang::CowCompilerInvocation almost allows this, but doesn't derive from +// CompilerInvocation or support shallow copies from a CompilerInvocation, so is +// not useful to us as we can't build an ASTUnit from it. +class ShallowCopyCompilerInvocation : public clang::CompilerInvocation { + public: + explicit ShallowCopyCompilerInvocation( + const clang::CompilerInvocation& invocation) { + shallow_copy_assign(invocation); + + // The preprocessor options are modified to hold a replacement includes + // buffer, so make our own version of those options. + PPOpts = std::make_shared(*PPOpts); + } +}; + +} // namespace + +auto GenerateAst(Context& context, + llvm::ArrayRef imports, + llvm::IntrusiveRefCntPtr fs, + std::shared_ptr base_invocation) + -> bool { + CARBON_CHECK(!context.cpp_context()); + CARBON_CHECK(!context.sem_ir().cpp_file()); + + auto invocation = + std::make_shared(*base_invocation); + + // Build a diagnostics engine. + llvm::IntrusiveRefCntPtr diags( + clang::CompilerInstance::createDiagnostics( + *fs, invocation->getDiagnosticOpts(), + new CarbonClangDiagnosticConsumer(context, invocation), + /*ShouldOwnClient=*/true)); + + // Extract the input from the frontend invocation and make sure it makes + // sense. + const auto& inputs = invocation->getFrontendOpts().Inputs; + CARBON_CHECK(inputs.size() == 1 && + inputs[0].getKind().getLanguage() == clang::Language::CXX && + inputs[0].getKind().getFormat() == clang::InputKind::Source); + llvm::StringRef file_name = inputs[0].getFile(); + + // Remap the imports file name to the corresponding `#include`s. + // TODO: Modify the frontend options to specify this memory buffer as input + // instead of remapping the file. + std::string includes = GenerateCppIncludesHeaderCode(context, imports); + auto includes_buffer = + llvm::MemoryBuffer::getMemBufferCopy(includes, file_name); + invocation->getPreprocessorOpts().addRemappedFile(file_name, + includes_buffer.release()); + + clang::DiagnosticErrorTrap trap(*diags); + + // Create the AST unit. + auto ast = clang::ASTUnit::LoadFromCompilerInvocation( + invocation, std::make_shared(), nullptr, + diags, new clang::FileManager(invocation->getFileSystemOpts(), fs)); + + // Attach the AST to SemIR. This needs to be done before we can emit any + // diagnostics, so their locations can be properly interpreted by our + // diagnostics machinery. + context.set_cpp_context(std::make_unique(ast.get())); + context.sem_ir().set_cpp_file( + std::make_unique(std::move(ast))); + + // Emit any diagnostics we queued up while building the AST. + context.emitter().Flush(); + + return !trap.hasErrorOccurred(); +} + +} // namespace Carbon::Check diff --git a/toolchain/check/cpp/generate_ast.h b/toolchain/check/cpp/generate_ast.h new file mode 100644 index 000000000000..33e173eb9eca --- /dev/null +++ b/toolchain/check/cpp/generate_ast.h @@ -0,0 +1,26 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef CARBON_TOOLCHAIN_CHECK_CPP_GENERATE_AST_H_ +#define CARBON_TOOLCHAIN_CHECK_CPP_GENERATE_AST_H_ + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/Support/VirtualFileSystem.h" +#include "toolchain/check/context.h" + +namespace Carbon::Check { + +// Generates a Clang AST for the given C++ imports and sets it as the context's +// `cpp_context` and the SemIR's `cpp_file`. Returns a bool that represents +// whether compilation was successful. +auto GenerateAst(Context& context, + llvm::ArrayRef imports, + llvm::IntrusiveRefCntPtr fs, + std::shared_ptr base_invocation) + -> bool; + +} // namespace Carbon::Check + +#endif // CARBON_TOOLCHAIN_CHECK_CPP_GENERATE_AST_H_ diff --git a/toolchain/check/cpp/import.cpp b/toolchain/check/cpp/import.cpp index 7eb9613ae1b8..9b62232877c8 100644 --- a/toolchain/check/cpp/import.cpp +++ b/toolchain/check/cpp/import.cpp @@ -15,18 +15,13 @@ #include "clang/AST/RecordLayout.h" #include "clang/AST/UnresolvedSet.h" #include "clang/AST/VTableBuilder.h" -#include "clang/Basic/FileManager.h" -#include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/CompilerInvocation.h" -#include "clang/Frontend/TextDiagnostic.h" -#include "clang/Lex/PreprocessorOptions.h" #include "clang/Sema/Lookup.h" #include "clang/Sema/Overload.h" #include "common/check.h" #include "common/ostream.h" #include "common/raw_string_ostream.h" #include "llvm/ADT/IntrusiveRefCntPtr.h" -#include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/raw_ostream.h" #include "toolchain/base/int.h" @@ -39,6 +34,7 @@ #include "toolchain/check/convert.h" #include "toolchain/check/cpp/access.h" #include "toolchain/check/cpp/custom_type_mapping.h" +#include "toolchain/check/cpp/generate_ast.h" #include "toolchain/check/cpp/location.h" #include "toolchain/check/cpp/macros.h" #include "toolchain/check/cpp/thunk.h" @@ -55,9 +51,6 @@ #include "toolchain/check/pattern_match.h" #include "toolchain/check/type.h" #include "toolchain/check/type_completion.h" -#include "toolchain/diagnostics/diagnostic.h" -#include "toolchain/diagnostics/diagnostic_emitter.h" -#include "toolchain/diagnostics/format_providers.h" #include "toolchain/parse/node_ids.h" #include "toolchain/sem_ir/clang_decl.h" #include "toolchain/sem_ir/class.h" @@ -71,88 +64,6 @@ namespace Carbon::Check { -// Add a line marker directive pointing at the location of the `import Cpp` -// declaration in the Carbon source file. This will cause Clang's diagnostics -// machinery to track and report the location in Carbon code where the import -// was written. -static auto GenerateLineMarker(Context& context, llvm::raw_ostream& out, - int line) { - out << "# " << line << " \"" - << FormatEscaped(context.tokens().source().filename()) << "\"\n"; -} - -// Generates C++ file contents to #include all requested imports. -static auto GenerateCppIncludesHeaderCode( - Context& context, llvm::ArrayRef imports) - -> std::string { - std::string code; - llvm::raw_string_ostream code_stream(code); - for (const Parse::Tree::PackagingNames& import : imports) { - if (import.inline_body_id.has_value()) { - // Expand `import Cpp inline "code";` directly into the specified code. - auto code_token = context.parse_tree().node_token(import.inline_body_id); - - // Compute the line number on which the C++ code starts. Usually the code - // is specified as a block string literal and starts on the line after the - // start of the string token. - // TODO: Determine if this is a block string literal without calling - // `GetTokenText`, which re-lexes the string. - int line = context.tokens().GetLineNumber(code_token); - if (context.tokens().GetTokenText(code_token).contains('\n')) { - ++line; - } - - GenerateLineMarker(context, code_stream, line); - code_stream << context.string_literal_values().Get( - context.tokens().GetStringLiteralValue(code_token)) - << "\n"; - // TODO: Inject a clang pragma here to produce an error if there are - // unclosed scopes at the end of this inline C++ fragment. - } else if (import.library_id.has_value()) { - // Translate `import Cpp library "foo.h";` into `#include "foo.h"`. - GenerateLineMarker(context, code_stream, - context.tokens().GetLineNumber( - context.parse_tree().node_token(import.node_id))); - auto name = context.string_literal_values().Get(import.library_id); - if (name.starts_with('<') && name.ends_with('>')) { - code_stream << "#include <" - << FormatEscaped(name.drop_front().drop_back()) << ">\n"; - } else { - code_stream << "#include \"" << FormatEscaped(name) << "\"\n"; - } - } - } - - // Inject a declaration of placement operator new, because the code we - // generate in thunks depends on it for placement new expressions. Clang has - // special-case logic for lowering a new-expression using this, so a - // definition is not required. - // TODO: This is a hack. We should be able to directly generate Clang AST to - // construct objects in-place without this. - // TODO: Once we can rely on libc++ being available, consider including - // `<__new/placement_new_delete.h>` instead. - code_stream << R"(# 1 "" -#undef constexpr -#if __cplusplus > 202302L -constexpr -#endif -#undef void -#undef operator -#undef new -void* operator new(__SIZE_TYPE__, void*) -#if __cplusplus < 201103L -#undef throw -throw() -#else -#undef noexcept -noexcept -#endif -; -)"; - - return code; -} - // Adds the name to the scope with the given `access_kind` and `inst_id`. // `inst_id` must have a value. static auto AddNameToScope(Context& context, SemIR::NameScopeId scope_id, @@ -180,269 +91,6 @@ static auto AddImportIRInst(SemIR::File& file, return file.import_ir_insts().Add(SemIR::ImportIRInst(clang_source_loc_id)); } -namespace { - -// Used to convert Clang diagnostics to Carbon diagnostics. -// -// Handling of Clang notes is a little subtle: as far as Clang is concerned, -// notes are separate diagnostics, not connected to the error or warning that -// precedes them. But in Carbon's diagnostics system, notes are part of the -// enclosing diagnostic. To handle this, we buffer Clang diagnostics until we -// reach a point where we know we're not in the middle of a diagnostic, and then -// emit a diagnostic along with all of its notes. This is triggered when adding -// or removing a Carbon context note, which could otherwise get attached to the -// wrong C++ diagnostics, and at the end of the Carbon program. -class CarbonClangDiagnosticConsumer : public clang::DiagnosticConsumer { - public: - // Creates an instance with the location that triggers calling Clang. The - // `context` is not stored here, and the diagnostics consumer is expected to - // outlive it. - explicit CarbonClangDiagnosticConsumer( - Context& context, std::shared_ptr invocation) - : sem_ir_(&context.sem_ir()), - emitter_(&context.emitter()), - invocation_(std::move(invocation)) { - emitter_->AddFlushFn([this] { EmitDiagnostics(); }); - } - - ~CarbonClangDiagnosticConsumer() override { - // Do not inspect `emitter_` here; it's typically destroyed before the - // consumer is. - // TODO: If Clang produces diagnostics after check finishes, they'll get - // added to the list of pending diagnostics and never emitted. - CARBON_CHECK(diagnostic_infos_.empty(), - "Missing flush before destroying diagnostic consumer"); - } - - // Generates a Carbon warning for each Clang warning and a Carbon error for - // each Clang error or fatal. - auto HandleDiagnostic(clang::DiagnosticsEngine::Level diag_level, - const clang::Diagnostic& info) -> void override { - DiagnosticConsumer::HandleDiagnostic(diag_level, info); - - SemIR::ImportIRInstId clang_import_ir_inst_id = - AddImportIRInst(*sem_ir_, info.getLocation()); - - llvm::SmallString<256> message; - info.FormatDiagnostic(message); - - // Render a code snippet including any highlighted ranges and fixit hints. - // TODO: Also include the #include stack and macro expansion stack in the - // diagnostic output in some way. - RawStringOstream snippet_stream; - if (!info.hasSourceManager()) { - // If we don't have a source manager, this is an error from early in the - // frontend. Don't produce a snippet. - CARBON_CHECK(info.getLocation().isInvalid()); - } else { - CodeContextRenderer(snippet_stream, invocation_->getLangOpts(), - invocation_->getDiagnosticOpts()) - .emitDiagnostic( - clang::FullSourceLoc(info.getLocation(), info.getSourceManager()), - diag_level, message, info.getRanges(), info.getFixItHints()); - } - - diagnostic_infos_.push_back({.level = diag_level, - .import_ir_inst_id = clang_import_ir_inst_id, - .message = message.str().str(), - .snippet = snippet_stream.TakeStr()}); - } - - // Returns the diagnostic to use for a given Clang diagnostic level. - static auto GetDiagnostic(clang::DiagnosticsEngine::Level level) - -> const Diagnostics::DiagnosticBase& { - switch (level) { - case clang::DiagnosticsEngine::Ignored: { - CARBON_FATAL("Emitting an ignored diagnostic"); - break; - } - case clang::DiagnosticsEngine::Note: { - CARBON_DIAGNOSTIC(CppInteropParseNote, Note, "{0}", std::string); - return CppInteropParseNote; - } - case clang::DiagnosticsEngine::Remark: - case clang::DiagnosticsEngine::Warning: { - // TODO: Add a distinct Remark level to Carbon diagnostics, and stop - // mapping remarks to warnings. - CARBON_DIAGNOSTIC(CppInteropParseWarning, Warning, "{0}", std::string); - return CppInteropParseWarning; - } - case clang::DiagnosticsEngine::Error: - case clang::DiagnosticsEngine::Fatal: { - CARBON_DIAGNOSTIC(CppInteropParseError, Error, "{0}", std::string); - return CppInteropParseError; - } - } - } - - // Outputs Carbon diagnostics based on the collected Clang diagnostics. Must - // be called after the AST is set in the context. - auto EmitDiagnostics() -> void { - CARBON_CHECK( - sem_ir_->cpp_file(), - "Attempted to emit C++ diagnostics before the C++ file is set"); - - for (size_t i = 0; i != diagnostic_infos_.size(); ++i) { - const ClangDiagnosticInfo& info = diagnostic_infos_[i]; - auto builder = emitter_->Build(SemIR::LocId(info.import_ir_inst_id), - GetDiagnostic(info.level), info.message); - builder.OverrideSnippet(info.snippet); - for (; i + 1 < diagnostic_infos_.size() && - diagnostic_infos_[i + 1].level == clang::DiagnosticsEngine::Note; - ++i) { - const ClangDiagnosticInfo& note_info = diagnostic_infos_[i + 1]; - builder - .Note(SemIR::LocId(note_info.import_ir_inst_id), - GetDiagnostic(note_info.level), note_info.message) - .OverrideSnippet(note_info.snippet); - } - // TODO: This will apply all current Carbon annotation functions. We - // should instead track how Clang's context notes and Carbon's annotation - // functions are interleaved, and interleave the notes in the same order. - builder.Emit(); - } - diagnostic_infos_.clear(); - } - - private: - // A diagnostics renderer based on clang's TextDiagnostic that captures just - // the code context (the snippet). - class CodeContextRenderer : public clang::TextDiagnostic { - protected: - using TextDiagnostic::TextDiagnostic; - - void emitDiagnosticMessage( - clang::FullSourceLoc /*loc*/, clang::PresumedLoc /*ploc*/, - clang::DiagnosticsEngine::Level /*level*/, llvm::StringRef /*message*/, - llvm::ArrayRef /*ranges*/, - clang::DiagOrStoredDiag /*info*/) override {} - void emitDiagnosticLoc( - clang::FullSourceLoc /*loc*/, clang::PresumedLoc /*ploc*/, - clang::DiagnosticsEngine::Level /*level*/, - llvm::ArrayRef /*ranges*/) override {} - - // emitCodeContext is inherited from clang::TextDiagnostic. - - void emitIncludeLocation(clang::FullSourceLoc /*loc*/, - clang::PresumedLoc /*ploc*/) override {} - void emitImportLocation(clang::FullSourceLoc /*loc*/, - clang::PresumedLoc /*ploc*/, - llvm::StringRef /*module_name*/) override {} - void emitBuildingModuleLocation(clang::FullSourceLoc /*loc*/, - clang::PresumedLoc /*ploc*/, - llvm::StringRef /*module_name*/) override {} - - // beginDiagnostic and endDiagnostic are inherited from - // clang::TextDiagnostic in case it wants to do any setup / teardown work. - }; - - // Information on a Clang diagnostic that can be converted to a Carbon - // diagnostic. - struct ClangDiagnosticInfo { - // The Clang diagnostic level. - clang::DiagnosticsEngine::Level level; - - // The ID of the ImportIR instruction referring to the Clang source - // location. - SemIR::ImportIRInstId import_ir_inst_id; - - // The Clang diagnostic textual message. - std::string message; - - // The code snippet produced by clang. - std::string snippet; - }; - - // The Carbon file that this C++ compilation is attached to. - SemIR::File* sem_ir_; - - // The diagnostic emitter that we're emitting diagnostics into. - DiagnosticEmitterBase* emitter_; - - // The compiler invocation that is producing the diagnostics. - std::shared_ptr invocation_; - - // Collects the information for all Clang diagnostics to be converted to - // Carbon diagnostics after the context has been initialized with the Clang - // AST. - llvm::SmallVector diagnostic_infos_; -}; - -// A wrapper around a clang::CompilerInvocation that allows us to make a shallow -// copy of most of the invocation and only make a deep copy of the parts that we -// want to change. -// -// clang::CowCompilerInvocation almost allows this, but doesn't derive from -// CompilerInvocation or support shallow copies from a CompilerInvocation, so is -// not useful to us as we can't build an ASTUnit from it. -class ShallowCopyCompilerInvocation : public clang::CompilerInvocation { - public: - explicit ShallowCopyCompilerInvocation( - const clang::CompilerInvocation& invocation) { - shallow_copy_assign(invocation); - - // The preprocessor options are modified to hold a replacement includes - // buffer, so make our own version of those options. - PPOpts = std::make_shared(*PPOpts); - } -}; - -} // namespace - -// Generates a Clang AST for the C++ imports and sets it in the context's -// `sem_ir`. Returns a bool that represents whether compilation was successful. -// TODO: Consider to always have a (non-null) AST even if there are no Cpp -// imports. -static auto GenerateAst( - Context& context, llvm::ArrayRef imports, - llvm::IntrusiveRefCntPtr fs, - std::shared_ptr base_invocation) -> bool { - auto invocation = - std::make_shared(*base_invocation); - - // Build a diagnostics engine. - llvm::IntrusiveRefCntPtr diags( - clang::CompilerInstance::createDiagnostics( - *fs, invocation->getDiagnosticOpts(), - new CarbonClangDiagnosticConsumer(context, invocation), - /*ShouldOwnClient=*/true)); - - // Extract the input from the frontend invocation and make sure it makes - // sense. - const auto& inputs = invocation->getFrontendOpts().Inputs; - CARBON_CHECK(inputs.size() == 1 && - inputs[0].getKind().getLanguage() == clang::Language::CXX && - inputs[0].getKind().getFormat() == clang::InputKind::Source); - llvm::StringRef file_name = inputs[0].getFile(); - - // Remap the imports file name to the corresponding `#include`s. - // TODO: Modify the frontend options to specify this memory buffer as input - // instead of remapping the file. - std::string includes = GenerateCppIncludesHeaderCode(context, imports); - auto includes_buffer = - llvm::MemoryBuffer::getMemBufferCopy(includes, file_name); - invocation->getPreprocessorOpts().addRemappedFile(file_name, - includes_buffer.release()); - - clang::DiagnosticErrorTrap trap(*diags); - - // Create the AST unit. - auto ast = clang::ASTUnit::LoadFromCompilerInvocation( - invocation, std::make_shared(), nullptr, - diags, new clang::FileManager(invocation->getFileSystemOpts(), fs)); - - // Attach the AST to SemIR. This needs to be done before we can emit any - // diagnostics, so their locations can be properly interpreted by our - // diagnostics machinery. - context.sem_ir().set_cpp_file( - std::make_unique(std::move(ast))); - - // Emit any diagnostics we queued up while building the AST. - context.emitter().Flush(); - - return !trap.hasErrorOccurred(); -} - // Adds a namespace for the `Cpp` import and returns its `NameScopeId`. static auto AddNamespace(Context& context, PackageNameId cpp_package_id, llvm::ArrayRef imports) @@ -468,11 +116,11 @@ auto ImportCpp(Context& context, llvm::IntrusiveRefCntPtr fs, std::shared_ptr invocation) -> void { if (imports.empty()) { + // TODO: Consider always having a (non-null) AST even if there are no Cpp + // imports. return; } - CARBON_CHECK(!context.sem_ir().cpp_file()); - PackageNameId package_id = imports.front().package_id; CARBON_CHECK( llvm::all_of(imports, [&](const Parse::Tree::PackagingNames& import) { @@ -2508,7 +2156,7 @@ auto ImportClassDefinitionForClangDecl(Context& context, SemIR::LocId loc_id, // Ask Clang whether the type is complete. This triggers template // instantiation if necessary. clang::DiagnosticErrorTrap trap(cpp_file->diagnostics()); - if (!cpp_file->sema().isCompleteType( + if (!context.cpp_context()->sema().isCompleteType( loc, context.ast_context().getCanonicalTagType(clang_decl))) { // Type is incomplete. Nothing more to do, but tell the caller if we // produced an error. diff --git a/toolchain/check/cpp/location.cpp b/toolchain/check/cpp/location.cpp index 299976756039..0e76fbe93d09 100644 --- a/toolchain/check/cpp/location.cpp +++ b/toolchain/check/cpp/location.cpp @@ -33,7 +33,7 @@ static auto GetFileInfo(Context& context, SemIR::CheckIRId ir_id) -> FileInfo { } // If we've seen this file before, reuse the same FileID. - auto& file_start_locs = context.cpp_carbon_file_locations(); + auto& file_start_locs = context.cpp_context()->carbon_file_locations(); if (static_cast(file_start_locs.size()) <= file_index) { // Never valid; prepare a slot for the caching below. file_start_locs.resize(file_index + 1); diff --git a/toolchain/check/cpp/thunk.cpp b/toolchain/check/cpp/thunk.cpp index a7a294eecebe..486149b70ff5 100644 --- a/toolchain/check/cpp/thunk.cpp +++ b/toolchain/check/cpp/thunk.cpp @@ -349,7 +349,7 @@ static auto CreateThunkFunctionDecl( thunk_function_decl->addAttr(clang::AsmLabelAttr::CreateImplicit( ast_context, GenerateThunkMangledName( - *context.sem_ir().clang_mangle_context(), *callee_info.decl, + context.cpp_context()->clang_mangle_context(), *callee_info.decl, callee_info.num_params - callee_info.has_explicit_object_parameter()), clang_loc)); diff --git a/toolchain/sem_ir/cpp_file.h b/toolchain/sem_ir/cpp_file.h index 00c23c699640..71f23ab4de67 100644 --- a/toolchain/sem_ir/cpp_file.h +++ b/toolchain/sem_ir/cpp_file.h @@ -35,6 +35,8 @@ class CppFile { auto source_manager() const -> const clang::SourceManager& { return ast_unit_->getSourceManager(); } + // TODO: This doesn't really belong here, but is currently used by lowering + // because Clang's code generation may produce diagnostics. auto diagnostics() const -> clang::DiagnosticsEngine& { return ast_unit_->getDiagnostics(); } @@ -46,7 +48,6 @@ class CppFile { auto ast_context() const -> const clang::ASTContext& { return ast_unit_->getASTContext(); } - auto sema() -> clang::Sema& { return ast_unit_->getSema(); } // Visit all top-level declarations in the file. auto VisitLocalTopLevelDecls( diff --git a/toolchain/sem_ir/file.cpp b/toolchain/sem_ir/file.cpp index a06dd6fe2423..2cf14925bd49 100644 --- a/toolchain/sem_ir/file.cpp +++ b/toolchain/sem_ir/file.cpp @@ -8,7 +8,6 @@ #include #include -#include "clang/AST/Mangle.h" #include "common/check.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" @@ -224,7 +223,6 @@ auto File::CollectMemUsage(MemUsage& mem_usage, llvm::StringRef label) const auto File::set_cpp_file(std::unique_ptr cpp_file) -> void { cpp_file_ = std::move(cpp_file); - clang_mangle_context_.reset(cpp_file_->ast_context().createMangleContext()); } } // namespace Carbon::SemIR diff --git a/toolchain/sem_ir/file.h b/toolchain/sem_ir/file.h index 98988f845c60..997103f2185b 100644 --- a/toolchain/sem_ir/file.h +++ b/toolchain/sem_ir/file.h @@ -234,9 +234,6 @@ class File : public Printable { // `File` and initialize the pointer in the constructor instead of using a // setter. auto set_cpp_file(std::unique_ptr cpp_file) -> void; - auto clang_mangle_context() -> clang::MangleContext* { - return clang_mangle_context_.get(); - } auto clang_decls() -> ClangDeclStore& { return clang_decls_; } auto clang_decls() const -> const ClangDeclStore& { return clang_decls_; } auto names() const -> NameStoreWrapper { @@ -383,10 +380,6 @@ class File : public Printable { // imports. std::unique_ptr cpp_file_; - // The Clang mangle context for the target in the ASTContext. Initialized - // together with `cpp_file_`. - std::unique_ptr clang_mangle_context_; - // Clang AST declarations pointing to the AST and their mapped Carbon // instructions. When calling `Lookup()`, `inst_id` is ignored. `Add()` will // not add multiple entries with the same `decl` and different `inst_id`.