diff --git a/toolchain/driver/driver.cpp b/toolchain/driver/driver.cpp
index 8636605b7d73..7eb32e6c5663 100644
--- a/toolchain/driver/driver.cpp
+++ b/toolchain/driver/driver.cpp
@@ -162,24 +162,28 @@ auto Driver::RunDumpSubcommand(DiagnosticConsumer& consumer,
     return false;
   }
 
+  bool has_errors = false;
+
   CARBON_VLOG() << "*** TokenizedBuffer::Lex ***\n";
   auto tokenized_source = TokenizedBuffer::Lex(*source, consumer);
+  has_errors |= tokenized_source.has_errors();
   CARBON_VLOG() << "*** TokenizedBuffer::Lex done ***\n";
   if (dump_mode == DumpMode::TokenizedBuffer) {
     CARBON_VLOG() << "Finishing output.";
     consumer.Flush();
     tokenized_source.Print(output_stream_);
-    return !tokenized_source.has_errors();
+    return !has_errors;
   }
   CARBON_VLOG() << "tokenized_buffer: " << tokenized_source;
 
   CARBON_VLOG() << "*** ParseTree::Parse ***\n";
   auto parse_tree = ParseTree::Parse(tokenized_source, consumer, vlog_stream_);
+  has_errors |= parse_tree.has_errors();
   CARBON_VLOG() << "*** ParseTree::Parse done ***\n";
   if (dump_mode == DumpMode::ParseTree) {
     consumer.Flush();
     parse_tree.Print(output_stream_, parse_tree_preorder);
-    return !tokenized_source.has_errors() && !parse_tree.has_errors();
+    return !has_errors;
   }
   CARBON_VLOG() << "parse_tree: " << parse_tree;
 
@@ -187,12 +191,12 @@ auto Driver::RunDumpSubcommand(DiagnosticConsumer& consumer,
   CARBON_VLOG() << "*** SemanticsIR::MakeFromParseTree ***\n";
   const SemanticsIR semantics_ir = SemanticsIR::MakeFromParseTree(
       builtin_ir, tokenized_source, parse_tree, consumer, vlog_stream_);
+  has_errors |= semantics_ir.has_errors();
   CARBON_VLOG() << "*** SemanticsIR::MakeFromParseTree done ***\n";
   if (dump_mode == DumpMode::SemanticsIR) {
     consumer.Flush();
     semantics_ir.Print(output_stream_);
-    // TODO: Return false when SemanticsIR has errors (not supported right now).
-    return !tokenized_source.has_errors() && !parse_tree.has_errors();
+    return !has_errors;
   }
   CARBON_VLOG() << "semantics_ir: " << semantics_ir;
 
diff --git a/toolchain/semantics/semantics_ir.cpp b/toolchain/semantics/semantics_ir.cpp
index bfb5a64d1210..6d00f7f96f3c 100644
--- a/toolchain/semantics/semantics_ir.cpp
+++ b/toolchain/semantics/semantics_ir.cpp
@@ -66,9 +66,11 @@ auto SemanticsIR::MakeFromParseTree(const SemanticsIR& builtin_ir,
 
   TokenizedBuffer::TokenLocationTranslator translator(
       &tokens, /*last_line_lexed_to_column=*/nullptr);
-  TokenDiagnosticEmitter emitter(translator, consumer);
+  ErrorTrackingDiagnosticConsumer err_tracker(consumer);
+  TokenDiagnosticEmitter emitter(translator, err_tracker);
   SemanticsParseTreeHandler(tokens, emitter, parse_tree, semantics, vlog_stream)
       .Build();
+  semantics.has_errors_ = err_tracker.seen_error();
   return semantics;
 }
 
diff --git a/toolchain/semantics/semantics_ir.h b/toolchain/semantics/semantics_ir.h
index 5418bc1ab2d4..6195ac7438ea 100644
--- a/toolchain/semantics/semantics_ir.h
+++ b/toolchain/semantics/semantics_ir.h
@@ -56,6 +56,9 @@ class SemanticsIR {
   // Prints the full IR.
   auto Print(llvm::raw_ostream& out) const -> void;
 
+  // Returns true if there were errors creating the semantics IR.
+  auto has_errors() const -> bool { return has_errors_; }
+
  private:
   friend class SemanticsParseTreeHandler;
 
@@ -128,6 +131,8 @@ class SemanticsIR {
     return node_id;
   }
 
+  bool has_errors_ = false;
+
   // Related IRs. There will always be at least 2 entries, the builtin IR (used
   // for references of builtins) followed by the current IR (used for references
   // crossing node blocks).
diff --git a/toolchain/semantics/testdata/operators/type_mismatch.carbon b/toolchain/semantics/testdata/operators/fail_type_mismatch.carbon
similarity index 93%
rename from toolchain/semantics/testdata/operators/type_mismatch.carbon
rename to toolchain/semantics/testdata/operators/fail_type_mismatch.carbon
index 95290b1cbd28..b9af16e205d7 100644
--- a/toolchain/semantics/testdata/operators/type_mismatch.carbon
+++ b/toolchain/semantics/testdata/operators/fail_type_mismatch.carbon
@@ -3,7 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 // AUTOUPDATE
-// RUN: %{carbon-run-semantics}
+// RUN: %{not} %{carbon-run-semantics}
 // CHECK:STDOUT: cross_reference_irs.size == 2,
 // CHECK:STDOUT: cross_references = {
 // CHECK:STDOUT:   node_xref0 = xref(ir0, block0, node0);
@@ -32,6 +32,6 @@
 // CHECK:STDOUT: }
 
 fn Main() {
-  // CHECK:STDERR: {{.*}}/toolchain/semantics/testdata/operators/type_mismatch.carbon:[[@LINE+1]]:13: Type mismatch
+  // CHECK:STDERR: {{.*}}/toolchain/semantics/testdata/operators/fail_type_mismatch.carbon:[[@LINE+1]]:13: Type mismatch
   return 12 + 3.4;
 }
diff --git a/toolchain/semantics/testdata/operators/type_mismatch_once.carbon b/toolchain/semantics/testdata/operators/fail_type_mismatch_once.carbon
similarity index 94%
rename from toolchain/semantics/testdata/operators/type_mismatch_once.carbon
rename to toolchain/semantics/testdata/operators/fail_type_mismatch_once.carbon
index e5b33f93b6b3..98d8375ab59e 100644
--- a/toolchain/semantics/testdata/operators/type_mismatch_once.carbon
+++ b/toolchain/semantics/testdata/operators/fail_type_mismatch_once.carbon
@@ -3,7 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 // AUTOUPDATE
-// RUN: %{carbon-run-semantics}
+// RUN: %{not} %{carbon-run-semantics}
 // CHECK:STDOUT: cross_reference_irs.size == 2,
 // CHECK:STDOUT: cross_references = {
 // CHECK:STDOUT:   node_xref0 = xref(ir0, block0, node0);
@@ -37,6 +37,6 @@
 fn Main() {
   // The following line has two mismatches, but after the first, it shouldn't
   // keep erroring.
-  // CHECK:STDERR: {{.*}}/toolchain/semantics/testdata/operators/type_mismatch_once.carbon:[[@LINE+1]]:13: Type mismatch
+  // CHECK:STDERR: {{.*}}/toolchain/semantics/testdata/operators/fail_type_mismatch_once.carbon:[[@LINE+1]]:13: Type mismatch
   return 12 + 3.4 + 12;
 }
diff --git a/toolchain/semantics/testdata/var/init_type_mismatch.carbon b/toolchain/semantics/testdata/var/fail_init_type_mismatch.carbon
similarity index 89%
rename from toolchain/semantics/testdata/var/init_type_mismatch.carbon
rename to toolchain/semantics/testdata/var/fail_init_type_mismatch.carbon
index 73d8c8fb2a86..a1865da743a7 100644
--- a/toolchain/semantics/testdata/var/init_type_mismatch.carbon
+++ b/toolchain/semantics/testdata/var/fail_init_type_mismatch.carbon
@@ -3,7 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 // AUTOUPDATE
-// RUN: %{carbon-run-semantics}
+// RUN: %{not} %{carbon-run-semantics}
 // CHECK:STDOUT: cross_reference_irs.size == 2,
 // CHECK:STDOUT: cross_references = {
 // CHECK:STDOUT:   node_xref0 = xref(ir0, block0, node0);
@@ -32,6 +32,6 @@
 // CHECK:STDOUT: }
 
 fn Main() {
-  // CHECK:STDERR: {{.*}}/toolchain/semantics/testdata/var/init_type_mismatch.carbon:[[@LINE+1]]:19: Type mismatch
+  // CHECK:STDERR: {{.*}}/toolchain/semantics/testdata/var/fail_init_type_mismatch.carbon:[[@LINE+1]]:19: Type mismatch
   var x: i32 = 1.0;
 }