Switch custom error stream output to diagnostic (#4846)

This switches most error printing to use diagnostics instead of direct
stream writes, even when not a specific file diagnostic. I'm allowing
empty filenames for this use-case.

This allows a little more specific testing to validate coverage of
output using the diagnostic coverage test. I'm adding a few tests to
cover things that weren't previously tested.

Separately, this also forces a little more standardization in format...
considering how changes like #4568 show effort being spent to _mirror_
diagnostic style, my thought is now to just use diagnostic code where
possible.

Note this also allows incrementally better testing of the language
server; I'm changing the crash fix from #4847 in favor of diagnostic
testing.

---------

Co-authored-by: Chandler Carruth <chandlerc@gmail.com>

toolchain/driver/clang_subcommand.cpp

@@ -51,8 +51,10 @@ auto ClangSubcommand::Run(DriverEnv& driver_env) -> DriverResult {
   // Don't run Clang when fuzzing, it is known to not be reliable under fuzzing
   // due to many unfixed issues.
   if (driver_env.fuzzing) {
-    *driver_env.error_stream
-        << "error: cannot run `clang` subcommand productively when fuzzing\n";
+    CARBON_DIAGNOSTIC(
+        ClangFuzzingDisallowed, Error,
+        "preventing fuzzing of `clang` subcommand due to library crashes");
+    driver_env.emitter.Emit(ClangFuzzingDisallowed);
     return {.success = false};
   }
 

toolchain/driver/compile_subcommand.cpp

@@ -11,6 +11,7 @@
 #include "toolchain/base/timings.h"
 #include "toolchain/check/check.h"
 #include "toolchain/codegen/codegen.h"
+#include "toolchain/diagnostics/diagnostic_emitter.h"
 #include "toolchain/diagnostics/sorting_diagnostic_consumer.h"
 #include "toolchain/lex/lex.h"
 #include "toolchain/lower/lower.h"
@@ -22,28 +23,6 @@
 
 namespace Carbon {
 
-auto operator<<(llvm::raw_ostream& out, CompileOptions::Phase phase)
-    -> llvm::raw_ostream& {
-  switch (phase) {
-    case CompileOptions::Phase::Lex:
-      out << "lex";
-      break;
-    case CompileOptions::Phase::Parse:
-      out << "parse";
-      break;
-    case CompileOptions::Phase::Check:
-      out << "check";
-      break;
-    case CompileOptions::Phase::Lower:
-      out << "lower";
-      break;
-    case CompileOptions::Phase::CodeGen:
-      out << "codegen";
-      break;
-  }
-  return out;
-}
-
 auto CompileOptions::Build(CommandLine::CommandBuilder& b) -> void {
   b.AddStringPositionalArg(
       {
@@ -128,15 +107,6 @@ of a binary object file instead. Ignored for other `--output` values.
       },
       [&](auto& arg_b) { arg_b.Set(&force_obj_output); });
 
-  b.AddFlag(
-      {
-          .name = "include-diagnostic-kind",
-          .help = R"""(
-When printing diagnostics, include the diagnostic kind as part of output. This
-applies to each message that forms a diagnostic, not just the primary message.
-)""",
-      },
-      [&](auto& arg_b) { arg_b.Set(&include_diagnostic_kind); });
   b.AddFlag(
       {
           .name = "stream-errors",
@@ -296,30 +266,49 @@ can be written to standard output as these phases progress.
 
 CompileSubcommand::CompileSubcommand() : DriverSubcommand(SubcommandInfo) {}
 
-// Returns an error for trying to dump a non-executed phase's output.
-static auto DumpPhaseError(llvm::StringLiteral requested_dump,
-                           CompileOptions::Phase phase) -> Error {
-  return Error(llvm::formatv(
-      "requested dumping {0} but compile phase is limited to `{1}`",
-      requested_dump, phase));
+// Returns a string for printing the phase in a diagnostic.
+static auto PhaseToString(CompileOptions::Phase phase) -> std::string {
+  switch (phase) {
+    case CompileOptions::Phase::Lex:
+      return "lex";
+    case CompileOptions::Phase::Parse:
+      return "parse";
+    case CompileOptions::Phase::Check:
+      return "check";
+    case CompileOptions::Phase::Lower:
+      return "lower";
+    case CompileOptions::Phase::CodeGen:
+      return "codegen";
+  }
 }
 
-auto CompileSubcommand::ValidateOptions() const -> ErrorOr<Success> {
+auto CompileSubcommand::ValidateOptions(NoLocDiagnosticEmitter& emitter) const
+    -> bool {
+  CARBON_DIAGNOSTIC(
+      CompilePhaseFlagConflict, Error,
+      "requested dumping {0} but compile phase is limited to `{1}`",
+      std::string, std::string);
   using Phase = CompileOptions::Phase;
   switch (options_.phase) {
     case Phase::Lex:
       if (options_.dump_parse_tree) {
-        return DumpPhaseError("parse tree", options_.phase);
+        emitter.Emit(CompilePhaseFlagConflict, "parse tree",
+                     PhaseToString(options_.phase));
+        return false;
       }
       [[fallthrough]];
     case Phase::Parse:
       if (options_.dump_sem_ir) {
-        return DumpPhaseError("SemIR", options_.phase);
+        emitter.Emit(CompilePhaseFlagConflict, "SemIR",
+                     PhaseToString(options_.phase));
+        return false;
       }
       [[fallthrough]];
     case Phase::Check:
       if (options_.dump_llvm_ir) {
-        return DumpPhaseError("LLVM IR", options_.phase);
+        emitter.Emit(CompilePhaseFlagConflict, "LLVM IR",
+                     PhaseToString(options_.phase));
+        return false;
       }
       [[fallthrough]];
     case Phase::Lower:
@@ -327,7 +316,7 @@ auto CompileSubcommand::ValidateOptions() const -> ErrorOr<Success> {
       // Everything can be dumped in these phases.
       break;
   }
-  return Success();
+  return true;
 }
 
 namespace {
@@ -673,9 +662,13 @@ auto CompilationUnit::RunCodeGenHelper() -> bool {
     if (output_filename.empty()) {
       if (!source_->is_regular_file()) {
         // Don't invent file names like `-.o` or `/dev/stdin.o`.
-        *driver_env_->error_stream
-            << "error: output file name must be specified for input `"
-            << input_filename_ << "` that is not a regular file\n";
+        // TODO: Consider rephrasing the diagnostic to use the file as the
+        // `Emit` location.
+        CARBON_DIAGNOSTIC(CompileInputNotRegularFile, Error,
+                          "output file name must be specified for input `{0}` "
+                          "that is not a regular file",
+                          std::string);
+        driver_env_->emitter.Emit(CompileInputNotRegularFile, input_filename_);
         return false;
       }
       output_filename = input_filename_;
@@ -694,9 +687,13 @@ auto CompilationUnit::RunCodeGenHelper() -> bool {
     llvm::raw_fd_ostream output_file(output_filename, ec,
                                      llvm::sys::fs::OF_None);
     if (ec) {
-      *driver_env_->error_stream << "error: could not open output file '"
-                                 << output_filename << "': " << ec.message()
-                                 << "\n";
+      // TODO: Consider rephrasing the diagnostic to use the file as the `Emit`
+      // location.
+      CARBON_DIAGNOSTIC(CompileOutputFileOpenError, Error,
+                        "could not open output file `{0}`: {1}", std::string,
+                        std::string);
+      driver_env_->emitter.Emit(CompileOutputFileOpenError,
+                                output_filename.str().str(), ec.message());
       return false;
     }
     if (options_.asm_output) {
@@ -745,8 +742,7 @@ auto CompilationUnit::IncludeInDumps(llvm::StringRef filename) const -> bool {
 }  // namespace
 
 auto CompileSubcommand::Run(DriverEnv& driver_env) -> DriverResult {
-  if (auto validate = ValidateOptions(); !validate.ok()) {
-    *driver_env.error_stream << "error: " << validate.error() << "\n";
+  if (!ValidateOptions(driver_env.emitter)) {
     return {.success = false};
   }
 
@@ -759,27 +755,28 @@ auto CompileSubcommand::Run(DriverEnv& driver_env) -> DriverResult {
     if (auto find = driver_env.installation->ReadPreludeManifest(); find.ok()) {
       prelude = std::move(*find);
     } else {
-      *driver_env.error_stream << "error: " << find.error() << "\n";
+      // TODO: Change ReadPreludeManifest to produce diagnostics.
+      CARBON_DIAGNOSTIC(CompilePreludeManifestError, Error, "{0}", std::string);
+      driver_env.emitter.Emit(CompilePreludeManifestError,
+                              PrintToString(find.error()));
       return {.success = false};
     }
   }
 
   // Prepare CompilationUnits before building scope exit handlers.
-  StreamDiagnosticConsumer stream_consumer(*driver_env.error_stream,
-                                           options_.include_diagnostic_kind);
   llvm::SmallVector<std::unique_ptr<CompilationUnit>> units;
   units.reserve(prelude.size() + options_.input_filenames.size());
 
   // Add the prelude files.
   for (const auto& input_filename : prelude) {
     units.push_back(std::make_unique<CompilationUnit>(
-        driver_env, options_, &stream_consumer, input_filename));
+        driver_env, options_, &driver_env.consumer, input_filename));
   }
 
   // Add the input source files.
   for (const auto& input_filename : options_.input_filenames) {
     units.push_back(std::make_unique<CompilationUnit>(
-        driver_env, options_, &stream_consumer, input_filename));
+        driver_env, options_, &driver_env.consumer, input_filename));
   }
 
   auto on_exit = llvm::make_scope_exit([&]() {
@@ -789,7 +786,7 @@ auto CompileSubcommand::Run(DriverEnv& driver_env) -> DriverResult {
       unit->PostCompile();
     }
 
-    stream_consumer.Flush();
+    driver_env.consumer.Flush();
   });
 
   PrettyStackTraceFunction flush_on_crash([&](llvm::raw_ostream& out) {
@@ -801,14 +798,14 @@ auto CompileSubcommand::Run(DriverEnv& driver_env) -> DriverResult {
       out << "Flushing diagnostics\n";
     } else {
       out << "Pending diagnostics:\n";
-      stream_consumer.set_stream(&out);
+      driver_env.consumer.set_stream(&out);
     }
 
     for (auto& unit : units) {
       unit->FlushForStackTrace();
     }
-    stream_consumer.Flush();
-    stream_consumer.set_stream(driver_env.error_stream);
+    driver_env.consumer.Flush();
+    driver_env.consumer.set_stream(driver_env.error_stream);
   });
 
   // Returns a DriverResult object. Called whenever Compile returns.

toolchain/driver/compile_subcommand.h

@@ -52,7 +52,6 @@ struct CompileOptions {
   bool dump_asm = false;
   bool dump_mem_usage = false;
   bool dump_timings = false;
-  bool include_diagnostic_kind = false;
   bool stream_errors = false;
   bool preorder_parse_tree = false;
   bool builtin_sem_ir = false;
@@ -75,8 +74,9 @@ class CompileSubcommand : public DriverSubcommand {
 
  private:
   // Does custom validation of the compile-subcommand options structure beyond
-  // what the command line parsing library supports.
-  auto ValidateOptions() const -> ErrorOr<Success>;
+  // what the command line parsing library supports. Diagnoses and returns false
+  // on failure.
+  auto ValidateOptions(NoLocDiagnosticEmitter& emitter) const -> bool;
 
   CompileOptions options_;
 };

toolchain/driver/driver.cpp

@@ -24,8 +24,9 @@ struct Options {
 
   auto Build(CommandLine::CommandBuilder& b) -> void;
 
-  bool verbose;
-  bool fuzzing;
+  bool verbose = false;
+  bool fuzzing = false;
+  bool include_diagnostic_kind = false;
 
   ClangSubcommand clang;
   CompileSubcommand compile;
@@ -73,6 +74,16 @@ auto Options::Build(CommandLine::CommandBuilder& b) -> void {
       },
       [&](CommandLine::FlagBuilder& arg_b) { arg_b.Set(&fuzzing); });
 
+  b.AddFlag(
+      {
+          .name = "include-diagnostic-kind",
+          .help = R"""(
+When printing diagnostics, include the diagnostic kind as part of output. This
+applies to each message that forms a diagnostic, not just the primary message.
+)""",
+      },
+      [&](auto& arg_b) { arg_b.Set(&include_diagnostic_kind); });
+
   clang.AddTo(b, &selected_subcommand);
   compile.AddTo(b, &selected_subcommand);
   format.AddTo(b, &selected_subcommand);
@@ -84,8 +95,9 @@ auto Options::Build(CommandLine::CommandBuilder& b) -> void {
 
 auto Driver::RunCommand(llvm::ArrayRef<llvm::StringRef> args) -> DriverResult {
   if (driver_env_.installation->error()) {
-    *driver_env_.error_stream << "error: " << *driver_env_.installation->error()
-                              << "\n";
+    CARBON_DIAGNOSTIC(DriverInstallInvalid, Error, "{0}", std::string);
+    driver_env_.emitter.Emit(DriverInstallInvalid,
+                             driver_env_.installation->error()->str());
     return {.success = false};
   }
 
@@ -95,8 +107,15 @@ auto Driver::RunCommand(llvm::ArrayRef<llvm::StringRef> args) -> DriverResult {
       args, *driver_env_.output_stream, Options::Info,
       [&](CommandLine::CommandBuilder& b) { options.Build(b); });
 
+  // Regardless of whether the parse succeeded, try to use the diagnostic kind
+  // flag.
+  driver_env_.consumer.set_include_diagnostic_kind(
+      options.include_diagnostic_kind);
+
   if (!result.ok()) {
-    *driver_env_.error_stream << "error: " << result.error() << "\n";
+    CARBON_DIAGNOSTIC(DriverCommandLineParseFailed, Error, "{0}", std::string);
+    driver_env_.emitter.Emit(DriverCommandLineParseFailed,
+                             PrintToString(result.error()));
     return {.success = false};
   } else if (*result == CommandLine::ParseResult::MetaSuccess) {
     return {.success = true};

toolchain/driver/driver_env.h

@@ -6,9 +6,11 @@
 #define CARBON_TOOLCHAIN_DRIVER_DRIVER_ENV_H_
 
 #include <cstdio>
+#include <utility>
 
 #include "common/ostream.h"
 #include "llvm/Support/VirtualFileSystem.h"
+#include "toolchain/diagnostics/diagnostic_emitter.h"
 #include "toolchain/install/install_paths.h"
 
 namespace Carbon {
@@ -24,7 +26,9 @@ struct DriverEnv {
         input_stream(input_stream),
         output_stream(output_stream),
         error_stream(error_stream),
-        fuzzing(fuzzing) {}
+        fuzzing(fuzzing),
+        consumer(error_stream),
+        emitter(&consumer) {}
 
   // The filesystem for source code.
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fs;
@@ -39,13 +43,19 @@ struct DriverEnv {
   // Error output; stderr.
   llvm::raw_pwrite_stream* error_stream;
 
-  // For CARBON_VLOG.
-  llvm::raw_pwrite_stream* vlog_stream = nullptr;
-
   // Tracks when the driver is being fuzzed. This allows specific commands to
   // error rather than perform operations that aren't well behaved during
   // fuzzing.
   bool fuzzing;
+
+  // A diagnostic consumer, to be able to connect output.
+  StreamDiagnosticConsumer consumer;
+
+  // A diagnostic emitter that has no locations.
+  NoLocDiagnosticEmitter emitter;
+
+  // For CARBON_VLOG.
+  llvm::raw_pwrite_stream* vlog_stream = nullptr;
 };
 
 }  // namespace Carbon

toolchain/driver/driver_test.cpp

@@ -225,10 +225,5 @@ TEST_F(DriverTest, FileOutput) {
   EXPECT_THAT(ReadFile("test.s"), ContainsRegex("Main:"));
 }
 
-TEST_F(DriverTest, LanguageServerNoStdin) {
-  EXPECT_FALSE(driver_.RunCommand({"language-server"}).success);
-  EXPECT_THAT(test_error_stream_.TakeStr(), HasSubstr("requires input_stream"));
-}
-
 }  // namespace
 }  // namespace Carbon

toolchain/driver/format_subcommand.cpp

@@ -56,9 +56,10 @@ auto FormatSubcommand::Run(DriverEnv& driver_env) -> DriverResult {
   DriverResult result = {.success = true};
   if (options_.input_filenames.size() > 1 &&
       !options_.output_filename.empty()) {
-    *driver_env.error_stream
-        << "error: cannot format multiple input files when "
-           "--output is set\n";
+    CARBON_DIAGNOSTIC(FormatMultipleFilesToOneOutput, Error,
+                      "multiple input files are being provided; --output only "
+                      "works with one input");
+    driver_env.emitter.Emit(FormatMultipleFilesToOneOutput);
     result.success = false;
     return result;
   }
@@ -68,22 +69,20 @@ auto FormatSubcommand::Run(DriverEnv& driver_env) -> DriverResult {
     result.per_file_success.back().second = false;
   };
 
-  StreamDiagnosticConsumer consumer(*driver_env.error_stream,
-                                    /*include_diagnostic_kind=*/false);
   for (auto& f : options_.input_filenames) {
     // Push a result, which we'll update on failure.
     result.per_file_success.push_back({f.str(), true});
 
     // TODO: Consider refactoring this for sharing with compile.
     // TODO: Decide what to do with `-` when there are multiple arguments.
-    auto source =
-        SourceBuffer::MakeFromFileOrStdin(*driver_env.fs, f, consumer);
+    auto source = SourceBuffer::MakeFromFileOrStdin(*driver_env.fs, f,
+                                                    driver_env.consumer);
     if (!source) {
       mark_per_file_error();
       continue;
     }
     SharedValueStores value_stores;
-    auto tokens = Lex::Lex(value_stores, *source, consumer);
+    auto tokens = Lex::Lex(value_stores, *source, driver_env.consumer);
 
     RawStringOstream buffer;
     if (Format::Format(tokens, buffer)) {

toolchain/driver/language_server_subcommand.cpp

@@ -4,6 +4,7 @@
 
 #include "toolchain/driver/language_server_subcommand.h"
 
+#include "toolchain/diagnostics/diagnostic_consumer.h"
 #include "toolchain/language_server/language_server.h"
 
 namespace Carbon {
@@ -20,18 +21,16 @@ LanguageServerSubcommand::LanguageServerSubcommand()
 
 auto LanguageServerSubcommand::Run(DriverEnv& driver_env) -> DriverResult {
   if (!driver_env.input_stream) {
-    *driver_env.error_stream
-        << "error: language-server requires input_stream\n";
+    CARBON_DIAGNOSTIC(LanguageServerMissingInputStream, Error,
+                      "language-server requires input_stream");
+    driver_env.emitter.Emit(LanguageServerMissingInputStream);
     return {.success = false};
   }
 
-  auto err =
+  bool success =
       LanguageServer::Run(driver_env.input_stream, *driver_env.output_stream,
-                          *driver_env.error_stream);
-  if (!err.ok()) {
-    *driver_env.error_stream << "error: " << err.error() << "\n";
-  }
-  return {.success = err.ok()};
+                          *driver_env.error_stream, driver_env.consumer);
+  return {.success = success};
 }
 
 }  // namespace Carbon

toolchain/driver/testdata/compile/multifile_raw_and_textual_ir.carbon

@@ -2,7 +2,7 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: compile --no-prelude-import --phase=check --dump-sem-ir --dump-raw-sem-ir %s
+// ARGS: --include-diagnostic-kind compile --no-prelude-import --phase=check --dump-sem-ir --dump-raw-sem-ir %s
 //
 // Check that we can combine textual IR and raw IR dumping in one compile.
 //

toolchain/driver/testdata/compile/multifile_raw_ir.carbon

@@ -2,7 +2,7 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: compile --no-prelude-import  --phase=check --dump-raw-sem-ir %s
+// ARGS: --include-diagnostic-kind compile --no-prelude-import  --phase=check --dump-raw-sem-ir %s
 //
 // Check that raw IR dumping works as expected.
 //

toolchain/driver/testdata/compile/raw_and_textual_ir.carbon

@@ -2,7 +2,7 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: compile --no-prelude-import --phase=check --dump-sem-ir --dump-raw-sem-ir %s
+// ARGS: --include-diagnostic-kind compile --no-prelude-import --phase=check --dump-sem-ir --dump-raw-sem-ir %s
 //
 // Check that we can combine textual IR and raw IR dumping in one compile.
 //

toolchain/driver/testdata/compile/raw_ir.carbon

@@ -2,7 +2,7 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: compile --no-prelude-import --phase=check --dump-raw-sem-ir %s
+// ARGS: --include-diagnostic-kind compile --no-prelude-import --phase=check --dump-raw-sem-ir %s
 //
 // Check that raw IR dumping works as expected.
 //

toolchain/driver/testdata/compile/textual_ir.carbon

@@ -2,7 +2,7 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: compile --no-prelude-import --phase=check --dump-sem-ir %s
+// ARGS: --include-diagnostic-kind compile --no-prelude-import --phase=check --dump-sem-ir %s
 //
 // Check that the command-line flag to dump textual IR works.
 //

toolchain/driver/testdata/dump_mem_usage.carbon

@@ -2,7 +2,7 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: compile --phase=check --dump-mem-usage %s
+// ARGS: --include-diagnostic-kind compile --phase=check --dump-mem-usage %s
 //
 // To test this file alone, run:
 //   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/driver/testdata/dump_mem_usage.carbon

toolchain/driver/testdata/dump_shared_values.carbon

@@ -2,7 +2,7 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: compile --phase=lex --dump-shared-values %s
+// ARGS: --include-diagnostic-kind compile --phase=lex --dump-shared-values %s
 //
 // AUTOUPDATE
 // TIP: To test this file alone, run:

toolchain/driver/testdata/dump_timings.carbon

@@ -2,7 +2,7 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: compile --dump-timings --phase=check %s
+// ARGS: --include-diagnostic-kind compile --dump-timings --phase=check %s
 //
 // SET-CHECK-SUBSET
 //

toolchain/driver/testdata/fail_clang_fuzzing.cpp

@@ -0,0 +1,15 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// ARGS: --include-diagnostic-kind --fuzzing clang foo.cpp
+//
+// SET-CAPTURE-CONSOLE-OUTPUT
+// clang-format off
+// AUTOUPDATE
+// TIP: To test this file alone, run:
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/driver/testdata/fail_clang_fuzzing.cpp
+// TIP: To dump output, run:
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/driver/testdata/fail_clang_fuzzing.cpp
+// CHECK:STDERR: error: preventing fuzzing of `clang` subcommand due to library crashes [ClangFuzzingDisallowed]
+// CHECK:STDERR:

toolchain/driver/testdata/fail_clang_no_args.cpp

@@ -2,7 +2,7 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: clang --
+// ARGS: --include-diagnostic-kind clang --
 //
 // SET-CAPTURE-CONSOLE-OUTPUT
 // clang-format off

toolchain/driver/testdata/fail_dump_phase_conflict.carbon

@@ -0,0 +1,13 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// ARGS: --include-diagnostic-kind compile --phase=parse --dump-sem-ir %s
+//
+// AUTOUPDATE
+// TIP: To test this file alone, run:
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/driver/testdata/fail_dump_phase_conflict.carbon
+// TIP: To dump output, run:
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/driver/testdata/fail_dump_phase_conflict.carbon
+// CHECK:STDERR: error: requested dumping SemIR but compile phase is limited to `parse` [CompilePhaseFlagConflict]
+// CHECK:STDERR:

toolchain/driver/testdata/fail_flag.carbon

@@ -2,12 +2,12 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: compile --non-existent-flag
+// ARGS: --include-diagnostic-kind compile --non-existent-flag
 //
 // AUTOUPDATE
 // TIP: To test this file alone, run:
 // TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/driver/testdata/fail_flag.carbon
 // TIP: To dump output, run:
 // TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/driver/testdata/fail_flag.carbon
-// CHECK:STDERR: error: unknown option `--non-existent-flag`
+// CHECK:STDERR: error: unknown option `--non-existent-flag` [DriverCommandLineParseFailed]
 // CHECK:STDERR:

toolchain/driver/testdata/fail_flush_errors.carbon

@@ -2,7 +2,7 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: compile --no-prelude-import %s
+// ARGS: --include-diagnostic-kind compile --no-prelude-import %s
 //
 // AUTOUPDATE
 // TIP: To test this file alone, run:
@@ -13,7 +13,7 @@
 fn F() {
   // Create diagnostics containing string references, and trigger reallocation
   // of the string table.
-  // CHECK:STDERR: fail_flush_errors.carbon:[[@LINE+4]]:3: error: name `undeclared1` not found
+  // CHECK:STDERR: fail_flush_errors.carbon:[[@LINE+4]]:3: error: name `undeclared1` not found [NameNotFound]
   // CHECK:STDERR:   undeclared1;
   // CHECK:STDERR:   ^~~~~~~~~~~
   // CHECK:STDERR:
@@ -23,7 +23,7 @@ fn F() {
   // literal storage. Use a hex escape to ensure that the tokenized buffer
   // allocates separate storage for the result.
   "undec\x6Cared2";
-  // CHECK:STDERR: fail_flush_errors.carbon:[[@LINE+4]]:3: error: name `undeclared2` not found
+  // CHECK:STDERR: fail_flush_errors.carbon:[[@LINE+4]]:3: error: name `undeclared2` not found [NameNotFound]
   // CHECK:STDERR:   undeclared2;
   // CHECK:STDERR:   ^~~~~~~~~~~
   // CHECK:STDERR:
@@ -31,7 +31,7 @@ fn F() {
 
   // Add the name into the string table via a declaration rather than an expression.
   if (true) { var undeclared3: () = (); }
-  // CHECK:STDERR: fail_flush_errors.carbon:[[@LINE+4]]:3: error: name `undeclared3` not found
+  // CHECK:STDERR: fail_flush_errors.carbon:[[@LINE+4]]:3: error: name `undeclared3` not found [NameNotFound]
   // CHECK:STDERR:   undeclared3;
   // CHECK:STDERR:   ^~~~~~~~~~~
   // CHECK:STDERR:

toolchain/driver/testdata/fail_input_is_directory.carbon

@@ -2,12 +2,12 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: compile .
+// ARGS: --include-diagnostic-kind compile .
 //
 // AUTOUPDATE
 // TIP: To test this file alone, run:
 // TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/driver/testdata/fail_input_is_directory.carbon
 // TIP: To dump output, run:
 // TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/driver/testdata/fail_input_is_directory.carbon
-// CHECK:STDERR: .: error: error opening file for read: Invalid argument
+// CHECK:STDERR: .: error: error opening file for read: Invalid argument [ErrorOpeningFile]
 // CHECK:STDERR:

toolchain/driver/testdata/fail_missing_file.carbon

@@ -2,7 +2,7 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: compile --include-diagnostic-kind not_file.carbon
+// ARGS: --include-diagnostic-kind compile not_file.carbon
 //
 // AUTOUPDATE
 // TIP: To test this file alone, run:

toolchain/driver/testdata/fail_missing_stdin_output.carbon

@@ -2,12 +2,12 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: compile --no-prelude-import -
+// ARGS: --include-diagnostic-kind compile --no-prelude-import -
 //
 // AUTOUPDATE
 // TIP: To test this file alone, run:
 // TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/driver/testdata/fail_missing_stdin_output.carbon
 // TIP: To dump output, run:
 // TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/driver/testdata/fail_missing_stdin_output.carbon
-// CHECK:STDERR: error: output file name must be specified for input `-` that is not a regular file
+// CHECK:STDERR: error: output file name must be specified for input `-` that is not a regular file [CompileInputNotRegularFile]
 // CHECK:STDERR:

toolchain/driver/testdata/fail_output_is_directory.carbon

@@ -0,0 +1,15 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// ARGS: --include-diagnostic-kind compile --no-prelude-import --output=/ %s
+//
+// NOAUTOUPDATE
+// TIP: To test this file alone, run:
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/driver/testdata/fail_output_is_directory.carbon
+// TIP: To dump output, run:
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/driver/testdata/fail_output_is_directory.carbon
+
+// Don't depend on a specific filesystem error.
+// CHECK:STDERR: error: could not open output file `/`: {{.*}} [CompileOutputFileOpenError]
+// CHECK:STDERR:

toolchain/driver/testdata/stdin.carbon

@@ -2,7 +2,7 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: compile - --no-prelude-import --phase=check --dump-sem-ir
+// ARGS: --include-diagnostic-kind compile - --no-prelude-import --phase=check --dump-sem-ir
 //
 // AUTOUPDATE
 // TIP: To test this file alone, run: