Delay finishing the C++ translation unit until we reach the real EOF. (#6489)

Instead of parsing a complete C++ translation unit and then interacting
with the translation unit further after the fact, delay finishing the
translation unit until we finish the Carbon check phase. This fixes some
issues where we would produce duplicated or incorrect diagnostics at the
end of the C++ translation unit, particularly for unused declarations.
Now we're in control of how we parse the translation unit, also disable
parsing of C++20 modules if the syntax appears within `import Cpp
inline` code.

Keep the same clang parser alive throughout check, and use it instead of
building a new one when parsing macros. This resolves issues where the
translation unit scope was destroyed too early, resulting in unqualified
lookup within macros being unable to find global scope entities.

---------

Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

toolchain/check/cpp/generate_ast.cpp

@@ -14,7 +14,9 @@
 #include "clang/Frontend/FrontendAction.h"
 #include "clang/Frontend/TextDiagnostic.h"
 #include "clang/Lex/PreprocessorOptions.h"
+#include "clang/Parse/Parser.h"
 #include "clang/Sema/ExternalSemaSource.h"
+#include "clang/Sema/Sema.h"
 #include "common/check.h"
 #include "common/raw_string_ostream.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
@@ -365,6 +367,44 @@ class GenerateASTAction : public clang::ASTFrontendAction {
     return true;
   }
 
+  // Parse the imports and inline C++ fragments. This is notionally very similar
+  // to `clang::ParseAST`, which `ASTFrontendAction::ExecuteAction` calls, but
+  // this version doesn't parse C++20 modules and stops just before reaching the
+  // end of the translation unit.
+  auto ExecuteAction() -> void override {
+    clang::CompilerInstance& clang_instance = getCompilerInstance();
+    clang_instance.createSema(getTranslationUnitKind(),
+                              /*CompletionConsumer=*/nullptr);
+
+    context_->cpp_context()->set_parser(std::make_unique<clang::Parser>(
+        clang_instance.getPreprocessor(), clang_instance.getSema(),
+        /*SkipFunctionBodies=*/false));
+    auto& parser = context_->cpp_context()->parser();
+
+    clang_instance.getPreprocessor().EnterMainSourceFile();
+    if (auto* source = clang_instance.getASTContext().getExternalSource()) {
+      source->StartTranslationUnit(&clang_instance.getASTConsumer());
+    }
+
+    parser.Initialize();
+    clang_instance.getSema().ActOnStartOfTranslationUnit();
+
+    // Don't allow C++20 module declarations in inline Cpp code fragments.
+    auto module_import_state = clang::Sema::ModuleImportState::NotACXX20Module;
+
+    // Parse top-level declarations until we see EOF. Do not parse EOF, as that
+    // will cause the parser to end the translation unit prematurely.
+    while (parser.getCurToken().isNot(clang::tok::eof)) {
+      clang::Parser::DeclGroupPtrTy decl_group;
+      bool eof = parser.ParseTopLevelDecl(decl_group, module_import_state);
+      CARBON_CHECK(!eof);
+      if (decl_group && !clang_instance.getASTConsumer().HandleTopLevelDecl(
+                            decl_group.get())) {
+        break;
+      }
+    }
+  }
+
  private:
   Context* context_;
 };
@@ -447,4 +487,17 @@ auto GenerateAst(Context& context,
   return !trap.hasErrorOccurred();
 }
 
+auto FinishAst(Context& context) -> void {
+  if (!context.cpp_context()) {
+    return;
+  }
+
+  context.cpp_context()->sema().ActOnEndOfTranslationUnit();
+
+  // We don't call FrontendAction::EndSourceFile, because that destroys the AST.
+  context.set_cpp_context(nullptr);
+
+  context.emitter().Flush();
+}
+
 }  // namespace Carbon::Check