Perform Clang IRGen during check (#6569)

Background:
https://docs.google.com/document/d/1wi85FRiWh4X9A-gCYMVGKR40-q5fM6-3JaSpePk-XCY/edit?usp=sharing
And specifically this work is essentially an alternative to #5543

Clang's code generation is implemented through an ASTListener
(clang::CodeGenerator) that is attached throughout Clang's
parsing/sema/code
generation phases and acts on Clang AST incrementally throughout that
process.

Prior to this patch, Carbon has only created the CodeGenerator during
Carbon's
`lower` phase, missing out on key callbacks that would be made by Clang
during
`check`. Some of these issues were addressed by #6237 and #6483 - but
there were
still remaining cases where the delayed processing lead to missing
functionality.

With #6483 much of the Clang code that made multithreaded complexity of
#5543 is
no longer present, and we have access to the point of ASTListener
registration
so we can register the CodeGenerator there and consume its resulting
llvm::Module during lower.

Examples of some of the bugs this addresses are seen in the linked doc,
and
checked in as tests in this change in
`clang_code_generator_callbacks.carbon`

An indicental bug that's also fixed, and caused all the other test case
churn,
is that the `CodeGenerator` created during `lower` wasn't getting passed
the
Clang `CodeGenOpts` and was creating its own default - so, most notably,
optimization flags were not respected. This meant that the LLVM IR from
Clang
was always -O0 style IR (optnone, no inlinehint, no TBAA, etc). With
this
change, now the Clang IRGen gets the real `CodeGenOpts` and respects
optimization/other flags specified there.

This is only meant to be a rough proof of concept - I'm totally open to
reworking this in any way (even quite substantially) if folks have ideas
about
how this should be implemented most generally/elegantly/etc.

toolchain/check/cpp/generate_ast.cpp

@@ -9,6 +9,7 @@
 
 #include "clang/AST/ASTContext.h"
 #include "clang/Basic/FileManager.h"
+#include "clang/CodeGen/ModuleBuilder.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/CompilerInvocation.h"
 #include "clang/Frontend/FrontendAction.h"
@@ -330,21 +331,6 @@ class ShallowCopyCompilerInvocation : public clang::CompilerInvocation {
   }
 };
 
-// An AST consumer that tracks top-level declarations so they can be handed off
-// to code generation later.
-class BufferingConsumer : public clang::ASTConsumer {
- public:
-  explicit BufferingConsumer(SemIR::CppFile& file) : file_(&file) {}
-
-  auto HandleTopLevelDecl(clang::DeclGroupRef decl_group) -> bool override {
-    file_->decl_groups().push_back(decl_group);
-    return true;
-  }
-
- private:
-  SemIR::CppFile* file_;
-};
-
 // An action and a set of registered Clang callbacks used to generate an AST
 // from a set of Cpp imports.
 class GenerateASTAction : public clang::ASTFrontendAction {
@@ -352,10 +338,22 @@ class GenerateASTAction : public clang::ASTFrontendAction {
   explicit GenerateASTAction(Context& context) : context_(&context) {}
 
  protected:
-  auto CreateASTConsumer(clang::CompilerInstance& /*clang_instance*/,
-                         llvm::StringRef /*file*/)
+  auto CreateASTConsumer(clang::CompilerInstance& clang_instance,
+                         llvm::StringRef file)
       -> std::unique_ptr<clang::ASTConsumer> override {
-    return std::make_unique<BufferingConsumer>(*context_->sem_ir().cpp_file());
+    auto& cpp_file = *context_->sem_ir().cpp_file();
+    if (!cpp_file.llvm_context()) {
+      return std::make_unique<clang::ASTConsumer>();
+    }
+    auto code_generator =
+        std::unique_ptr<clang::CodeGenerator>(clang::CreateLLVMCodeGen(
+            cpp_file.diagnostics(), file,
+            clang_instance.getVirtualFileSystemPtr(),
+            clang_instance.getHeaderSearchOpts(),
+            clang_instance.getPreprocessorOpts(),
+            clang_instance.getCodeGenOpts(), *cpp_file.llvm_context()));
+    cpp_file.SetCodeGenerator(code_generator.get());
+    return code_generator;
   }
 
   auto BeginSourceFileAction(clang::CompilerInstance& /*clang_instance*/)
@@ -414,6 +412,7 @@ class GenerateASTAction : public clang::ASTFrontendAction {
 auto GenerateAst(Context& context,
                  llvm::ArrayRef<Parse::Tree::PackagingNames> imports,
                  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fs,
+                 llvm::LLVMContext* llvm_context,
                  std::shared_ptr<clang::CompilerInvocation> base_invocation)
     -> bool {
   CARBON_CHECK(!context.cpp_context());
@@ -454,8 +453,8 @@ auto GenerateAst(Context& context,
   auto clang_instance_ptr =
       std::make_unique<clang::CompilerInstance>(invocation);
   auto& clang_instance = *clang_instance_ptr;
-  context.sem_ir().set_cpp_file(
-      std::make_unique<SemIR::CppFile>(std::move(clang_instance_ptr)));
+  context.sem_ir().set_cpp_file(std::make_unique<SemIR::CppFile>(
+      std::move(clang_instance_ptr), llvm_context));
 
   clang_instance.setDiagnostics(diags);
   clang_instance.setVirtualFileSystem(fs);