Switch custom error stream output to diagnostic (#4846)

This switches most error printing to use diagnostics instead of direct
stream writes, even when not a specific file diagnostic. I'm allowing
empty filenames for this use-case.

This allows a little more specific testing to validate coverage of
output using the diagnostic coverage test. I'm adding a few tests to
cover things that weren't previously tested.

Separately, this also forces a little more standardization in format...
considering how changes like #4568 show effort being spent to _mirror_
diagnostic style, my thought is now to just use diagnostic code where
possible.

Note this also allows incrementally better testing of the language
server; I'm changing the crash fix from #4847 in favor of diagnostic
testing.

---------

Co-authored-by: Chandler Carruth <chandlerc@gmail.com>

toolchain/driver/testdata/compile/multifile_raw_and_textual_ir.carbon

@@ -2,7 +2,7 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: compile --no-prelude-import --phase=check --dump-sem-ir --dump-raw-sem-ir %s
+// ARGS: --include-diagnostic-kind compile --no-prelude-import --phase=check --dump-sem-ir --dump-raw-sem-ir %s
 //
 // Check that we can combine textual IR and raw IR dumping in one compile.
 //

toolchain/driver/testdata/compile/multifile_raw_ir.carbon

@@ -2,7 +2,7 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: compile --no-prelude-import  --phase=check --dump-raw-sem-ir %s
+// ARGS: --include-diagnostic-kind compile --no-prelude-import  --phase=check --dump-raw-sem-ir %s
 //
 // Check that raw IR dumping works as expected.
 //

toolchain/driver/testdata/compile/raw_and_textual_ir.carbon

@@ -2,7 +2,7 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: compile --no-prelude-import --phase=check --dump-sem-ir --dump-raw-sem-ir %s
+// ARGS: --include-diagnostic-kind compile --no-prelude-import --phase=check --dump-sem-ir --dump-raw-sem-ir %s
 //
 // Check that we can combine textual IR and raw IR dumping in one compile.
 //

toolchain/driver/testdata/compile/raw_ir.carbon

@@ -2,7 +2,7 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: compile --no-prelude-import --phase=check --dump-raw-sem-ir %s
+// ARGS: --include-diagnostic-kind compile --no-prelude-import --phase=check --dump-raw-sem-ir %s
 //
 // Check that raw IR dumping works as expected.
 //

toolchain/driver/testdata/compile/textual_ir.carbon

@@ -2,7 +2,7 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: compile --no-prelude-import --phase=check --dump-sem-ir %s
+// ARGS: --include-diagnostic-kind compile --no-prelude-import --phase=check --dump-sem-ir %s
 //
 // Check that the command-line flag to dump textual IR works.
 //

toolchain/driver/testdata/dump_mem_usage.carbon

@@ -2,7 +2,7 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: compile --phase=check --dump-mem-usage %s
+// ARGS: --include-diagnostic-kind compile --phase=check --dump-mem-usage %s
 //
 // To test this file alone, run:
 //   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/driver/testdata/dump_mem_usage.carbon

toolchain/driver/testdata/dump_shared_values.carbon

@@ -2,7 +2,7 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: compile --phase=lex --dump-shared-values %s
+// ARGS: --include-diagnostic-kind compile --phase=lex --dump-shared-values %s
 //
 // AUTOUPDATE
 // TIP: To test this file alone, run:

toolchain/driver/testdata/dump_timings.carbon

@@ -2,7 +2,7 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: compile --dump-timings --phase=check %s
+// ARGS: --include-diagnostic-kind compile --dump-timings --phase=check %s
 //
 // SET-CHECK-SUBSET
 //

toolchain/driver/testdata/fail_clang_fuzzing.cpp

@@ -0,0 +1,15 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// ARGS: --include-diagnostic-kind --fuzzing clang foo.cpp
+//
+// SET-CAPTURE-CONSOLE-OUTPUT
+// clang-format off
+// AUTOUPDATE
+// TIP: To test this file alone, run:
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/driver/testdata/fail_clang_fuzzing.cpp
+// TIP: To dump output, run:
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/driver/testdata/fail_clang_fuzzing.cpp
+// CHECK:STDERR: error: preventing fuzzing of `clang` subcommand due to library crashes [ClangFuzzingDisallowed]
+// CHECK:STDERR:

toolchain/driver/testdata/fail_clang_no_args.cpp

@@ -2,7 +2,7 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: clang --
+// ARGS: --include-diagnostic-kind clang --
 //
 // SET-CAPTURE-CONSOLE-OUTPUT
 // clang-format off

toolchain/driver/testdata/fail_dump_phase_conflict.carbon

@@ -0,0 +1,13 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// ARGS: --include-diagnostic-kind compile --phase=parse --dump-sem-ir %s
+//
+// AUTOUPDATE
+// TIP: To test this file alone, run:
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/driver/testdata/fail_dump_phase_conflict.carbon
+// TIP: To dump output, run:
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/driver/testdata/fail_dump_phase_conflict.carbon
+// CHECK:STDERR: error: requested dumping SemIR but compile phase is limited to `parse` [CompilePhaseFlagConflict]
+// CHECK:STDERR:

toolchain/driver/testdata/fail_flag.carbon

@@ -2,12 +2,12 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: compile --non-existent-flag
+// ARGS: --include-diagnostic-kind compile --non-existent-flag
 //
 // AUTOUPDATE
 // TIP: To test this file alone, run:
 // TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/driver/testdata/fail_flag.carbon
 // TIP: To dump output, run:
 // TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/driver/testdata/fail_flag.carbon
-// CHECK:STDERR: error: unknown option `--non-existent-flag`
+// CHECK:STDERR: error: unknown option `--non-existent-flag` [DriverCommandLineParseFailed]
 // CHECK:STDERR:

toolchain/driver/testdata/fail_flush_errors.carbon

@@ -2,7 +2,7 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: compile --no-prelude-import %s
+// ARGS: --include-diagnostic-kind compile --no-prelude-import %s
 //
 // AUTOUPDATE
 // TIP: To test this file alone, run:
@@ -13,7 +13,7 @@
 fn F() {
   // Create diagnostics containing string references, and trigger reallocation
   // of the string table.
-  // CHECK:STDERR: fail_flush_errors.carbon:[[@LINE+4]]:3: error: name `undeclared1` not found
+  // CHECK:STDERR: fail_flush_errors.carbon:[[@LINE+4]]:3: error: name `undeclared1` not found [NameNotFound]
   // CHECK:STDERR:   undeclared1;
   // CHECK:STDERR:   ^~~~~~~~~~~
   // CHECK:STDERR:
@@ -23,7 +23,7 @@ fn F() {
   // literal storage. Use a hex escape to ensure that the tokenized buffer
   // allocates separate storage for the result.
   "undec\x6Cared2";
-  // CHECK:STDERR: fail_flush_errors.carbon:[[@LINE+4]]:3: error: name `undeclared2` not found
+  // CHECK:STDERR: fail_flush_errors.carbon:[[@LINE+4]]:3: error: name `undeclared2` not found [NameNotFound]
   // CHECK:STDERR:   undeclared2;
   // CHECK:STDERR:   ^~~~~~~~~~~
   // CHECK:STDERR:
@@ -31,7 +31,7 @@ fn F() {
 
   // Add the name into the string table via a declaration rather than an expression.
   if (true) { var undeclared3: () = (); }
-  // CHECK:STDERR: fail_flush_errors.carbon:[[@LINE+4]]:3: error: name `undeclared3` not found
+  // CHECK:STDERR: fail_flush_errors.carbon:[[@LINE+4]]:3: error: name `undeclared3` not found [NameNotFound]
   // CHECK:STDERR:   undeclared3;
   // CHECK:STDERR:   ^~~~~~~~~~~
   // CHECK:STDERR:

toolchain/driver/testdata/fail_input_is_directory.carbon

@@ -2,12 +2,12 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: compile .
+// ARGS: --include-diagnostic-kind compile .
 //
 // AUTOUPDATE
 // TIP: To test this file alone, run:
 // TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/driver/testdata/fail_input_is_directory.carbon
 // TIP: To dump output, run:
 // TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/driver/testdata/fail_input_is_directory.carbon
-// CHECK:STDERR: .: error: error opening file for read: Invalid argument
+// CHECK:STDERR: .: error: error opening file for read: Invalid argument [ErrorOpeningFile]
 // CHECK:STDERR:

toolchain/driver/testdata/fail_missing_file.carbon

@@ -2,7 +2,7 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: compile --include-diagnostic-kind not_file.carbon
+// ARGS: --include-diagnostic-kind compile not_file.carbon
 //
 // AUTOUPDATE
 // TIP: To test this file alone, run:

toolchain/driver/testdata/fail_missing_stdin_output.carbon

@@ -2,12 +2,12 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: compile --no-prelude-import -
+// ARGS: --include-diagnostic-kind compile --no-prelude-import -
 //
 // AUTOUPDATE
 // TIP: To test this file alone, run:
 // TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/driver/testdata/fail_missing_stdin_output.carbon
 // TIP: To dump output, run:
 // TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/driver/testdata/fail_missing_stdin_output.carbon
-// CHECK:STDERR: error: output file name must be specified for input `-` that is not a regular file
+// CHECK:STDERR: error: output file name must be specified for input `-` that is not a regular file [CompileInputNotRegularFile]
 // CHECK:STDERR:

toolchain/driver/testdata/fail_output_is_directory.carbon

@@ -0,0 +1,15 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// ARGS: --include-diagnostic-kind compile --no-prelude-import --output=/ %s
+//
+// NOAUTOUPDATE
+// TIP: To test this file alone, run:
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/driver/testdata/fail_output_is_directory.carbon
+// TIP: To dump output, run:
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/driver/testdata/fail_output_is_directory.carbon
+
+// Don't depend on a specific filesystem error.
+// CHECK:STDERR: error: could not open output file `/`: {{.*}} [CompileOutputFileOpenError]
+// CHECK:STDERR:

toolchain/driver/testdata/stdin.carbon

@@ -2,7 +2,7 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: compile - --no-prelude-import --phase=check --dump-sem-ir
+// ARGS: --include-diagnostic-kind compile - --no-prelude-import --phase=check --dump-sem-ir
 //
 // AUTOUPDATE
 // TIP: To test this file alone, run: