diff --git a/testing/file_test/file_test_base.cpp b/testing/file_test/file_test_base.cpp
index 7234ee125508..40a94fb6a062 100644
--- a/testing/file_test/file_test_base.cpp
+++ b/testing/file_test/file_test_base.cpp
@@ -73,11 +73,12 @@ static auto CompareFailPrefix(llvm::StringRef filename, bool success) -> void {
   if (success) {
     EXPECT_FALSE(filename.starts_with("fail_"))
         << "`" << filename
-        << "` succeeded; if this is correct, add a `fail_` prefix.";
+        << "` succeeded; if success is expected, remove the `fail_` "
+           "prefix.";
   } else {
     EXPECT_TRUE(filename.starts_with("fail_"))
         << "`" << filename
-        << "` failed; if this is correct, remove the `fail_` prefix.";
+        << "` failed; if failure is expected, add the `fail_` prefix.";
   }
 }
 
diff --git a/toolchain/driver/testdata/fail_missing_file.carbon b/toolchain/driver/testdata/fail_missing_file.carbon
index 21770fbfc30c..ada4d3149a56 100644
--- a/toolchain/driver/testdata/fail_missing_file.carbon
+++ b/toolchain/driver/testdata/fail_missing_file.carbon
@@ -2,7 +2,7 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// ARGS: compile nonexistent.carbon
+// ARGS: compile not_file.carbon
 //
 // AUTOUPDATE
-// CHECK:STDERR: nonexistent.carbon: ERROR: Error opening file for read: No such file or directory
+// CHECK:STDERR: not_file.carbon: ERROR: Error opening file for read: No such file or directory
diff --git a/toolchain/testing/file_test.cpp b/toolchain/testing/file_test.cpp
index 7a4cebdb9a93..84e15ecb4537 100644
--- a/toolchain/testing/file_test.cpp
+++ b/toolchain/testing/file_test.cpp
@@ -7,6 +7,7 @@
 
 #include <string>
 
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/FormatVariadic.h"
@@ -29,16 +30,18 @@ class ToolchainFileTest : public FileTestBase {
            llvm::raw_pwrite_stream& stderr) -> ErrorOr<RunResult> override {
     Driver driver(fs, stdout, stderr);
     auto driver_result = driver.RunCommand(test_args);
-    if (std::find(test_args.begin(), test_args.end(), "%s") ==
-        test_args.end()) {
-      // Files weren't forwarded as an argument, so don't use per_file_success.
-      // This primarily occurs in driver tests with invalid filename arguments,
-      // which we wouldn't want to try validating.
-      return {{.success = driver_result.success}};
-    } else {
-      return {{.success = driver_result.success,
-               .per_file_success = std::move(driver_result.per_file_success)}};
-    }
+
+    RunResult result{
+        .success = driver_result.success,
+        .per_file_success = std::move(driver_result.per_file_success)};
+    // Drop entries that don't look like a file. Note this can empty out the
+    // list.
+    llvm::erase_if(result.per_file_success,
+                   [](std::pair<llvm::StringRef, bool> entry) {
+                     return entry.first == "." || entry.first == "-" ||
+                            entry.first.starts_with("not_file");
+                   });
+    return result;
   }
 
   auto GetDefaultArgs() -> llvm::SmallVector<std::string> override {