diff --git a/bazel/testing/lit_test.bzl b/bazel/testing/lit_test.bzl
index 011eaae6640c..0cc14e7c5e6d 100644
--- a/bazel/testing/lit_test.bzl
+++ b/bazel/testing/lit_test.bzl
@@ -31,6 +31,6 @@ def lit_test(name, test_dir, data = None, **kwargs):
         srcs = ["//bazel/testing:lit_test.py"],
         main = "//bazel/testing:lit_test.py",
         data = data + native.glob([test_dir + "/**"]),
-        args = [test_dir, "--"],
+        args = ["--package_name=%s" % native.package_name(), "--test_dir=%s" % test_dir, "--"],
         **kwargs
     )
diff --git a/bazel/testing/lit_test.py b/bazel/testing/lit_test.py
index d22c835b88e2..22564200c42f 100644
--- a/bazel/testing/lit_test.py
+++ b/bazel/testing/lit_test.py
@@ -8,16 +8,18 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 import argparse
 import os
+from pathlib import Path
 import subprocess
 
-_PASSTHROUGH_FLAGS = ["filter", "filter-out"]
-
 
 def _parse_args():
     """Parses command line arguments, returning the result."""
     arg_parser = argparse.ArgumentParser(description=__doc__)
     arg_parser.add_argument(
-        "test_dir", help="The directory containing tests to run."
+        "--package_name", help="The directory containing tests to run."
+    )
+    arg_parser.add_argument(
+        "--test_dir", help="The directory containing tests to run."
     )
     arg_parser.add_argument(
         "lit_args", nargs="*", help="Arguments to pass through to lit."
@@ -25,54 +27,27 @@ def _parse_args():
     return arg_parser.parse_args()
 
 
-def _normalize(relative_base, target):
-    """Given a target, normalizes it to a relative path."""
-    assert target
-    if target.startswith(":"):
-        # Local target; :foo -> my/dir/foo
-        return os.path.join(relative_base, target[1:])
-    elif target[0].isalpha():
-        # Local target; foo -> my/dir/foo
-        return os.path.join(relative_base, target)
-
-    if ":" in target:
-        # Specified target; //foo:bar -> //foo/bar
-        target = target.replace(":", "/")
-    else:
-        # Default target; //foo -> //foo/foo
-        target = os.path.join(target, os.path.basename(target))
-
-    if target.startswith("@"):
-        return os.path.join("external/", target[1:])
-    elif target.startswith("//"):
-        return target[2:]
-    else:
-        raise ValueError("Unhandled target path: %s" % target)
-
-
 def main():
     parsed_args = _parse_args()
 
-    bin_dir = os.getcwd()
-
-    # Figure out the actual path for the test_dir.
-    relative_base = os.path.dirname(_normalize("", os.environ["TEST_TARGET"]))
-    test_dir = os.path.join(
-        bin_dir, _normalize(relative_base, parsed_args.test_dir)
-    )
-
     args = [
-        os.path.join(os.environ["TEST_SRCDIR"], "llvm-project/llvm/lit"),
-        "--path=%s" % bin_dir,
-        test_dir,
+        str(Path(os.environ["TEST_SRCDIR"]).joinpath("llvm-project/llvm/lit")),
+        str(
+            Path.cwd().joinpath(parsed_args.package_name, parsed_args.test_dir)
+        ),
         "-sv",
     ]
 
+    # Force tests to be explicit about command paths.
+    env = os.environ.copy()
+    del env["PATH"]
+
     # Run lit.
-    p = subprocess.run(args=args + parsed_args.lit_args)
-    # Do this instead of check_call to hide stack traces.
-    if p.returncode != 0:
-        exit("lit failed, exit code %d" % p.returncode)
+    try:
+        subprocess.check_call(args=args + parsed_args.lit_args, env=env)
+    except subprocess.CalledProcessError as e:
+        # Print without the stack trace.
+        print(e)
 
 
 if __name__ == "__main__":
diff --git a/executable_semantics/testdata/struct/fail_equality_type.carbon b/executable_semantics/testdata/struct/fail_equality_type.carbon
index b77e9ffc8333..6bca2a1b4d0e 100644
--- a/executable_semantics/testdata/struct/fail_equality_type.carbon
+++ b/executable_semantics/testdata/struct/fail_equality_type.carbon
@@ -2,7 +2,7 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// RUN: %{not} %{executable_semantics} %s 2>&1 2>&1 | FileCheck %s
+// RUN: %{not} %{executable_semantics} %s 2>&1 2>&1 | %{FileCheck} %s
 // AUTOUPDATE: %{executable_semantics} %s
 // CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/struct/fail_equality_type.carbon:16: type error in ==
 // CHECK: expected: {.x: i32, .y: i32}
diff --git a/executable_semantics/testdata/tuple/fail_equality_type.carbon b/executable_semantics/testdata/tuple/fail_equality_type.carbon
index 543055705bc1..23fe57a43187 100644
--- a/executable_semantics/testdata/tuple/fail_equality_type.carbon
+++ b/executable_semantics/testdata/tuple/fail_equality_type.carbon
@@ -2,7 +2,7 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// RUN: %{not} %{executable_semantics} %s 2>&1 2>&1 | FileCheck %s
+// RUN: %{not} %{executable_semantics} %s 2>&1 2>&1 | %{FileCheck} %s
 // AUTOUPDATE: %{executable_semantics} %s
 // CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/tuple/fail_equality_type.carbon:16: type error in ==
 // CHECK: expected: (i32, i32)