diff --git a/executable_semantics/testdata/assignment_copy1.carbon b/executable_semantics/testdata/assignment_copy/destruct_original.carbon
similarity index 100%
rename from executable_semantics/testdata/assignment_copy1.carbon
rename to executable_semantics/testdata/assignment_copy/destruct_original.carbon
diff --git a/executable_semantics/testdata/assignment_copy2.carbon b/executable_semantics/testdata/assignment_copy/reassign_original.carbon
similarity index 100%
rename from executable_semantics/testdata/assignment_copy2.carbon
rename to executable_semantics/testdata/assignment_copy/reassign_original.carbon
diff --git a/executable_semantics/testdata/choice1.carbon b/executable_semantics/testdata/basic_syntax/choice.carbon
similarity index 100%
rename from executable_semantics/testdata/choice1.carbon
rename to executable_semantics/testdata/basic_syntax/choice.carbon
diff --git a/executable_semantics/testdata/invalid_char.carbon b/executable_semantics/testdata/basic_syntax/fail_invalid_char.carbon
similarity index 76%
rename from executable_semantics/testdata/invalid_char.carbon
rename to executable_semantics/testdata/basic_syntax/fail_invalid_char.carbon
index a9d9f773de7b..b9ddd8da473c 100644
--- a/executable_semantics/testdata/invalid_char.carbon
+++ b/executable_semantics/testdata/basic_syntax/fail_invalid_char.carbon
@@ -7,7 +7,6 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/invalid_char.carbon:13: invalid character '\xEF' in source file.
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/basic_syntax/fail_invalid_char.carbon:12: invalid character '\xEF' in source file.
 
 �
diff --git a/executable_semantics/testdata/pattern_variable_fail.carbon b/executable_semantics/testdata/basic_syntax/fail_missing_var.carbon
similarity index 80%
rename from executable_semantics/testdata/pattern_variable_fail.carbon
rename to executable_semantics/testdata/basic_syntax/fail_missing_var.carbon
index 8bf5c0b538bf..164bb93b8943 100644
--- a/executable_semantics/testdata/pattern_variable_fail.carbon
+++ b/executable_semantics/testdata/basic_syntax/fail_missing_var.carbon
@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/pattern_variable_fail.carbon:17: syntax error, unexpected COLON
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/basic_syntax/fail_missing_var.carbon:16: syntax error, unexpected COLON
 
 package ExecutableSemanticsTest api;
 
diff --git a/executable_semantics/testdata/next.carbon b/executable_semantics/testdata/basic_syntax/next.carbon
similarity index 100%
rename from executable_semantics/testdata/next.carbon
rename to executable_semantics/testdata/basic_syntax/next.carbon
diff --git a/executable_semantics/testdata/placeholder_variable.carbon b/executable_semantics/testdata/basic_syntax/placeholder_variable.carbon
similarity index 100%
rename from executable_semantics/testdata/placeholder_variable.carbon
rename to executable_semantics/testdata/basic_syntax/placeholder_variable.carbon
diff --git a/executable_semantics/testdata/print1.carbon b/executable_semantics/testdata/basic_syntax/print.carbon
similarity index 100%
rename from executable_semantics/testdata/print1.carbon
rename to executable_semantics/testdata/basic_syntax/print.carbon
diff --git a/executable_semantics/testdata/record1.carbon b/executable_semantics/testdata/basic_syntax/record.carbon
similarity index 100%
rename from executable_semantics/testdata/record1.carbon
rename to executable_semantics/testdata/basic_syntax/record.carbon
diff --git a/executable_semantics/testdata/star.carbon b/executable_semantics/testdata/basic_syntax/star.carbon
similarity index 100%
rename from executable_semantics/testdata/star.carbon
rename to executable_semantics/testdata/basic_syntax/star.carbon
diff --git a/executable_semantics/testdata/pattern_init.carbon b/executable_semantics/testdata/basic_syntax/var_tuple.carbon
similarity index 100%
rename from executable_semantics/testdata/pattern_init.carbon
rename to executable_semantics/testdata/basic_syntax/var_tuple.carbon
diff --git a/executable_semantics/testdata/zero.carbon b/executable_semantics/testdata/basic_syntax/zero.carbon
similarity index 100%
rename from executable_semantics/testdata/zero.carbon
rename to executable_semantics/testdata/basic_syntax/zero.carbon
diff --git a/executable_semantics/testdata/block2.carbon b/executable_semantics/testdata/block/empty.carbon
similarity index 100%
rename from executable_semantics/testdata/block2.carbon
rename to executable_semantics/testdata/block/empty.carbon
diff --git a/executable_semantics/testdata/block1.carbon b/executable_semantics/testdata/block/shadowing.carbon
similarity index 100%
rename from executable_semantics/testdata/block1.carbon
rename to executable_semantics/testdata/block/shadowing.carbon
diff --git a/executable_semantics/testdata/class2.carbon b/executable_semantics/testdata/class/assign_member.carbon
similarity index 100%
rename from executable_semantics/testdata/class2.carbon
rename to executable_semantics/testdata/class/assign_member.carbon
diff --git a/executable_semantics/testdata/class_field_access_mismatch.carbon b/executable_semantics/testdata/class/fail_field_access_mismatch.carbon
similarity index 80%
rename from executable_semantics/testdata/class_field_access_mismatch.carbon
rename to executable_semantics/testdata/class/fail_field_access_mismatch.carbon
index 7c266a31b0aa..28eb051ad680 100644
--- a/executable_semantics/testdata/class_field_access_mismatch.carbon
+++ b/executable_semantics/testdata/class/fail_field_access_mismatch.carbon
@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/class_field_access_mismatch.carbon:21: class Point does not have a field named z
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/class/fail_field_access_mismatch.carbon:20: class Point does not have a field named z
 
 package ExecutableSemanticsTest api;
 
diff --git a/executable_semantics/testdata/class_field_mismatch.carbon b/executable_semantics/testdata/class/fail_field_mismatch.carbon
similarity index 84%
rename from executable_semantics/testdata/class_field_mismatch.carbon
rename to executable_semantics/testdata/class/fail_field_mismatch.carbon
index 629d050b3e8c..2b1a6f0b2557 100644
--- a/executable_semantics/testdata/class_field_mismatch.carbon
+++ b/executable_semantics/testdata/class/fail_field_mismatch.carbon
@@ -7,11 +7,10 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/class_field_mismatch.carbon:23: type error in call
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/class/fail_field_mismatch.carbon:22: type error in call
 // CHECK: expected: (x = i32, y = i32)
 // CHECK: actual: (x = i32, z = i32)
 
-
 package ExecutableSemanticsTest api;
 
 class Point {
diff --git a/executable_semantics/testdata/class_field_missing.carbon b/executable_semantics/testdata/class/fail_field_missing.carbon
similarity index 84%
rename from executable_semantics/testdata/class_field_missing.carbon
rename to executable_semantics/testdata/class/fail_field_missing.carbon
index 25d3e3f4694d..78e7fbd89e79 100644
--- a/executable_semantics/testdata/class_field_missing.carbon
+++ b/executable_semantics/testdata/class/fail_field_missing.carbon
@@ -7,11 +7,10 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/class_field_missing.carbon:23: type error in call
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/class/fail_field_missing.carbon:22: type error in call
 // CHECK: expected: (x = i32, y = i32)
 // CHECK: actual: (x = i32)
 
-
 package ExecutableSemanticsTest api;
 
 class Point {
diff --git a/executable_semantics/testdata/class3.carbon b/executable_semantics/testdata/class/temp.carbon
similarity index 100%
rename from executable_semantics/testdata/class3.carbon
rename to executable_semantics/testdata/class/temp.carbon
diff --git a/executable_semantics/testdata/class1.carbon b/executable_semantics/testdata/class/var.carbon
similarity index 100%
rename from executable_semantics/testdata/class1.carbon
rename to executable_semantics/testdata/class/var.carbon
diff --git a/executable_semantics/testdata/experimental_continuation5.carbon b/executable_semantics/testdata/experimental_continuation/await_maintains_scope.carbon
similarity index 100%
rename from executable_semantics/testdata/experimental_continuation5.carbon
rename to executable_semantics/testdata/experimental_continuation/await_maintains_scope.carbon
diff --git a/executable_semantics/testdata/experimental_continuation7.carbon b/executable_semantics/testdata/experimental_continuation/copy_in_continuation.carbon
similarity index 100%
rename from executable_semantics/testdata/experimental_continuation7.carbon
rename to executable_semantics/testdata/experimental_continuation/copy_in_continuation.carbon
diff --git a/executable_semantics/testdata/experimental_continuation1.carbon b/executable_semantics/testdata/experimental_continuation/creation_is_noop.carbon
similarity index 100%
rename from executable_semantics/testdata/experimental_continuation1.carbon
rename to executable_semantics/testdata/experimental_continuation/creation_is_noop.carbon
diff --git a/executable_semantics/testdata/experimental_continuation9.carbon b/executable_semantics/testdata/experimental_continuation/fail_lifetime.carbon
similarity index 85%
rename from executable_semantics/testdata/experimental_continuation9.carbon
rename to executable_semantics/testdata/experimental_continuation/fail_lifetime.carbon
index 2585d872d7d0..c49440f9cc2a 100644
--- a/executable_semantics/testdata/experimental_continuation9.carbon
+++ b/executable_semantics/testdata/experimental_continuation/fail_lifetime.carbon
@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: RUNTIME ERROR: {{.*}}/experimental_continuation9.carbon:22: undefined behavior: access to dead value 1
-
+// CHECK: RUNTIME ERROR: {{.*}}/executable_semantics/testdata/experimental_continuation/fail_lifetime.carbon:21: undefined behavior: access to dead value 1
 
 package ExecutableSemanticsTest api;
 
diff --git a/executable_semantics/testdata/experimental_continuation6.carbon b/executable_semantics/testdata/experimental_continuation/recursive.carbon
similarity index 100%
rename from executable_semantics/testdata/experimental_continuation6.carbon
rename to executable_semantics/testdata/experimental_continuation/recursive.carbon
diff --git a/executable_semantics/testdata/experimental_continuation2.carbon b/executable_semantics/testdata/experimental_continuation/run.carbon
similarity index 100%
rename from executable_semantics/testdata/experimental_continuation2.carbon
rename to executable_semantics/testdata/experimental_continuation/run.carbon
diff --git a/executable_semantics/testdata/experimental_continuation3.carbon b/executable_semantics/testdata/experimental_continuation/run_with_await.carbon
similarity index 100%
rename from executable_semantics/testdata/experimental_continuation3.carbon
rename to executable_semantics/testdata/experimental_continuation/run_with_await.carbon
diff --git a/executable_semantics/testdata/experimental_continuation4.carbon b/executable_semantics/testdata/experimental_continuation/shallow_copy.carbon
similarity index 100%
rename from executable_semantics/testdata/experimental_continuation4.carbon
rename to executable_semantics/testdata/experimental_continuation/shallow_copy.carbon
diff --git a/executable_semantics/testdata/fun_named_params2.carbon b/executable_semantics/testdata/fun_named_params2.carbon
deleted file mode 100644
index 888e6f0f4db5..000000000000
--- a/executable_semantics/testdata/fun_named_params2.carbon
+++ /dev/null
@@ -1,19 +0,0 @@
-// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
-// Exceptions. See /LICENSE for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-// RUN: not executable_semantics %s 2>&1 | \
-// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
-// RUN: not executable_semantics --trace %s 2>&1 | \
-// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
-// AUTOUPDATE: executable_semantics %s
-// CHECK: PROGRAM ERROR: {{.*}}/fun_named_params2.carbon:15: positional members must come before named members
-
-
-package ExecutableSemanticsTest api;
-
-fn f(x: i32, .d = y: i32, z: i32, .e = a: i32) => (x + y) - (z + a);
-
-fn main() -> i32 {
-  return 0;
-}
diff --git a/executable_semantics/testdata/fun5.carbon b/executable_semantics/testdata/function/auto_return_add.carbon
similarity index 100%
rename from executable_semantics/testdata/fun5.carbon
rename to executable_semantics/testdata/function/auto_return_add.carbon
diff --git a/executable_semantics/testdata/type_compute3.carbon b/executable_semantics/testdata/function/auto_return_modify_arg_type.carbon
similarity index 100%
rename from executable_semantics/testdata/type_compute3.carbon
rename to executable_semantics/testdata/function/auto_return_modify_arg_type.carbon
diff --git a/executable_semantics/testdata/type_compute2.carbon b/executable_semantics/testdata/function/auto_return_modify_return_type.carbon
similarity index 100%
rename from executable_semantics/testdata/type_compute2.carbon
rename to executable_semantics/testdata/function/auto_return_modify_return_type.carbon
diff --git a/executable_semantics/testdata/type_compute.carbon b/executable_semantics/testdata/function/auto_return_type.carbon
similarity index 100%
rename from executable_semantics/testdata/type_compute.carbon
rename to executable_semantics/testdata/function/auto_return_type.carbon
diff --git a/executable_semantics/testdata/fun4.carbon b/executable_semantics/testdata/function/empty_params.carbon
similarity index 100%
rename from executable_semantics/testdata/fun4.carbon
rename to executable_semantics/testdata/function/empty_params.carbon
diff --git a/executable_semantics/testdata/fun6_fail_type.carbon b/executable_semantics/testdata/function/fail_call_with_tuple.carbon
similarity index 85%
rename from executable_semantics/testdata/fun6_fail_type.carbon
rename to executable_semantics/testdata/function/fail_call_with_tuple.carbon
index c9657798ce04..ccc0fe516175 100644
--- a/executable_semantics/testdata/fun6_fail_type.carbon
+++ b/executable_semantics/testdata/function/fail_call_with_tuple.carbon
@@ -7,11 +7,10 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/fun6_fail_type.carbon:22: type error in call
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/function/fail_call_with_tuple.carbon:21: type error in call
 // CHECK: expected: (0 = i32, 1 = i32)
 // CHECK: actual: (0 = (0 = i32, 1 = i32))
 
-
 package ExecutableSemanticsTest api;
 
 fn f(x: i32, y: i32) -> i32 { return x + y; }
diff --git a/executable_semantics/testdata/function/fail_named_params_order.carbon b/executable_semantics/testdata/function/fail_named_params_order.carbon
new file mode 100644
index 000000000000..f49481341334
--- /dev/null
+++ b/executable_semantics/testdata/function/fail_named_params_order.carbon
@@ -0,0 +1,20 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: not executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: not executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: PROGRAM ERROR: {{.*}}/executable_semantics/testdata/function/fail_named_params_order.carbon:14: positional members must come before named members
+
+package ExecutableSemanticsTest api;
+
+fn f(x: i32, .d = y: i32, z: i32, .e = a: i32) -> i32 {
+  return (x + y) - (z + a);
+}
+
+fn main() -> i32 {
+  return 0;
+}
diff --git a/executable_semantics/testdata/funptr1.carbon b/executable_semantics/testdata/function/fnty.carbon
similarity index 100%
rename from executable_semantics/testdata/funptr1.carbon
rename to executable_semantics/testdata/function/fnty.carbon
diff --git a/executable_semantics/testdata/ignored_parameter.carbon b/executable_semantics/testdata/function/ignored_parameter.carbon
similarity index 100%
rename from executable_semantics/testdata/ignored_parameter.carbon
rename to executable_semantics/testdata/function/ignored_parameter.carbon
diff --git a/executable_semantics/testdata/fun3.carbon b/executable_semantics/testdata/function/multiple_args.carbon
similarity index 100%
rename from executable_semantics/testdata/fun3.carbon
rename to executable_semantics/testdata/function/multiple_args.carbon
diff --git a/executable_semantics/testdata/fun_named_params.carbon b/executable_semantics/testdata/function/named_params.carbon
similarity index 91%
rename from executable_semantics/testdata/fun_named_params.carbon
rename to executable_semantics/testdata/function/named_params.carbon
index 5fe6255a8810..574adc0c2719 100644
--- a/executable_semantics/testdata/fun_named_params.carbon
+++ b/executable_semantics/testdata/function/named_params.carbon
@@ -11,7 +11,9 @@
 
 package ExecutableSemanticsTest api;
 
-fn f(x: i32, .d = y: i32) => x + y;
+fn f(x: i32, .d = y: i32) -> i32 {
+  return x + y;
+}
 
 fn main() -> i32 {
   return f(1, .d = 2) - 3;
diff --git a/executable_semantics/testdata/fun2.carbon b/executable_semantics/testdata/function/param_lifetime.carbon
similarity index 100%
rename from executable_semantics/testdata/fun2.carbon
rename to executable_semantics/testdata/function/param_lifetime.carbon
diff --git a/executable_semantics/testdata/fun_recur.carbon b/executable_semantics/testdata/function/recursive.carbon
similarity index 100%
rename from executable_semantics/testdata/fun_recur.carbon
rename to executable_semantics/testdata/function/recursive.carbon
diff --git a/executable_semantics/testdata/fun1.carbon b/executable_semantics/testdata/function/return.carbon
similarity index 100%
rename from executable_semantics/testdata/fun1.carbon
rename to executable_semantics/testdata/function/return.carbon
diff --git a/executable_semantics/testdata/type_match.carbon b/executable_semantics/testdata/function/type_match.carbon
similarity index 100%
rename from executable_semantics/testdata/type_match.carbon
rename to executable_semantics/testdata/function/type_match.carbon
diff --git a/executable_semantics/testdata/generic_function_apply.carbon b/executable_semantics/testdata/generic_function/apply.carbon
similarity index 100%
rename from executable_semantics/testdata/generic_function_apply.carbon
rename to executable_semantics/testdata/generic_function/apply.carbon
diff --git a/executable_semantics/testdata/generic_function_fail3.carbon b/executable_semantics/testdata/generic_function/fail_not_addable.carbon
similarity index 82%
rename from executable_semantics/testdata/generic_function_fail3.carbon
rename to executable_semantics/testdata/generic_function/fail_not_addable.carbon
index d347bd2f195e..2abcd9b72017 100644
--- a/executable_semantics/testdata/generic_function_fail3.carbon
+++ b/executable_semantics/testdata/generic_function/fail_not_addable.carbon
@@ -7,11 +7,10 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/generic_function_fail3.carbon:18: type error in addition(1)
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/generic_function/fail_not_addable.carbon:17: type error in addition(1)
 // CHECK: expected: i32
 // CHECK: actual: T
 
-
 package ExecutableSemanticsTest api;
 
 fn id[T:! Type](x: T) -> T {
diff --git a/executable_semantics/testdata/generic_function_fail1.carbon b/executable_semantics/testdata/generic_function/fail_type_deduction_mismatch.carbon
similarity index 80%
rename from executable_semantics/testdata/generic_function_fail1.carbon
rename to executable_semantics/testdata/generic_function/fail_type_deduction_mismatch.carbon
index 8a62cf7d026b..bedec9a98d4d 100644
--- a/executable_semantics/testdata/generic_function_fail1.carbon
+++ b/executable_semantics/testdata/generic_function/fail_type_deduction_mismatch.carbon
@@ -7,11 +7,10 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/generic_function_fail1.carbon:22: type error in argument deduction
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/generic_function/fail_type_deduction_mismatch.carbon:21: type error in argument deduction
 // CHECK: expected: i32
 // CHECK: actual: Bool
 
-
 package ExecutableSemanticsTest api;
 
 fn fst[T:! Type](x: T, y: T) -> T {
diff --git a/executable_semantics/testdata/generic_function_fail2.carbon b/executable_semantics/testdata/generic_function/fail_type_deduction_unused.carbon
similarity index 77%
rename from executable_semantics/testdata/generic_function_fail2.carbon
rename to executable_semantics/testdata/generic_function/fail_type_deduction_unused.carbon
index 3321bb8a210a..0206f5a06219 100644
--- a/executable_semantics/testdata/generic_function_fail2.carbon
+++ b/executable_semantics/testdata/generic_function/fail_type_deduction_unused.carbon
@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/generic_function_fail2.carbon:20: could not deduce type argument for type parameter T
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/generic_function/fail_type_deduction_unused.carbon:19: could not deduce type argument for type parameter T
 
 package ExecutableSemanticsTest api;
 
diff --git a/executable_semantics/testdata/generic_function3.carbon b/executable_semantics/testdata/generic_function/non_generic_param.carbon
similarity index 100%
rename from executable_semantics/testdata/generic_function3.carbon
rename to executable_semantics/testdata/generic_function/non_generic_param.carbon
diff --git a/executable_semantics/testdata/generic_function1.carbon b/executable_semantics/testdata/generic_function/return_val.carbon
similarity index 100%
rename from executable_semantics/testdata/generic_function1.carbon
rename to executable_semantics/testdata/generic_function/return_val.carbon
diff --git a/executable_semantics/testdata/generic_function_swap.carbon b/executable_semantics/testdata/generic_function/swap.carbon
similarity index 100%
rename from executable_semantics/testdata/generic_function_swap.carbon
rename to executable_semantics/testdata/generic_function/swap.carbon
diff --git a/executable_semantics/testdata/generic_function_tuple_map.carbon b/executable_semantics/testdata/generic_function/tuple_map.carbon
similarity index 100%
rename from executable_semantics/testdata/generic_function_tuple_map.carbon
rename to executable_semantics/testdata/generic_function/tuple_map.carbon
diff --git a/executable_semantics/testdata/generic_function2.carbon b/executable_semantics/testdata/generic_function/type_matching.carbon
similarity index 100%
rename from executable_semantics/testdata/generic_function2.carbon
rename to executable_semantics/testdata/generic_function/type_matching.carbon
diff --git a/executable_semantics/testdata/global_variable8.carbon b/executable_semantics/testdata/global_variable/fail_init_order.carbon
similarity index 84%
rename from executable_semantics/testdata/global_variable8.carbon
rename to executable_semantics/testdata/global_variable/fail_init_order.carbon
index e8de448604c4..6c1a5deeeb9c 100644
--- a/executable_semantics/testdata/global_variable8.carbon
+++ b/executable_semantics/testdata/global_variable/fail_init_order.carbon
@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: RUNTIME ERROR: {{.*}}/global_variable8.carbon:18: could not find `y`
-
+// CHECK: RUNTIME ERROR: {{.*}}/executable_semantics/testdata/global_variable/fail_init_order.carbon:17: could not find `y`
 
 package ExecutableSemanticsTest api;
 
diff --git a/executable_semantics/testdata/global_variable3.carbon b/executable_semantics/testdata/global_variable/fail_init_type_mismatch.carbon
similarity index 81%
rename from executable_semantics/testdata/global_variable3.carbon
rename to executable_semantics/testdata/global_variable/fail_init_type_mismatch.carbon
index 0fb730d24cf3..1bc3dba302a2 100644
--- a/executable_semantics/testdata/global_variable3.carbon
+++ b/executable_semantics/testdata/global_variable/fail_init_type_mismatch.carbon
@@ -7,11 +7,10 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/global_variable3.carbon:19: type error in initializer of variable
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/global_variable/fail_init_type_mismatch.carbon:18: type error in initializer of variable
 // CHECK: expected: i32
 // CHECK: actual: Bool
 
-
 package ExecutableSemanticsTest api;
 
 // Test type checking of global variable. Error expected.
diff --git a/executable_semantics/testdata/global_variable1.carbon b/executable_semantics/testdata/global_variable/init_and_read.carbon
similarity index 100%
rename from executable_semantics/testdata/global_variable1.carbon
rename to executable_semantics/testdata/global_variable/init_and_read.carbon
diff --git a/executable_semantics/testdata/global_variable7.carbon b/executable_semantics/testdata/global_variable/init_from_function.carbon
similarity index 100%
rename from executable_semantics/testdata/global_variable7.carbon
rename to executable_semantics/testdata/global_variable/init_from_function.carbon
diff --git a/executable_semantics/testdata/global_variable6.carbon b/executable_semantics/testdata/global_variable/init_order.carbon
similarity index 100%
rename from executable_semantics/testdata/global_variable6.carbon
rename to executable_semantics/testdata/global_variable/init_order.carbon
diff --git a/executable_semantics/testdata/global_variable5.carbon b/executable_semantics/testdata/global_variable/shadowing.carbon
similarity index 84%
rename from executable_semantics/testdata/global_variable5.carbon
rename to executable_semantics/testdata/global_variable/shadowing.carbon
index 11f4321a5d83..761e949d77ef 100644
--- a/executable_semantics/testdata/global_variable5.carbon
+++ b/executable_semantics/testdata/global_variable/shadowing.carbon
@@ -7,11 +7,10 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/global_variable5.carbon:22: type error in return
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/global_variable/shadowing.carbon:21: type error in return
 // CHECK: expected: ()
 // CHECK: actual: i32
 
-
 package ExecutableSemanticsTest api;
 
 // Test overshadowing of global variable.
diff --git a/executable_semantics/testdata/global_variable4.carbon b/executable_semantics/testdata/global_variable/write.carbon
similarity index 100%
rename from executable_semantics/testdata/global_variable4.carbon
rename to executable_semantics/testdata/global_variable/write.carbon
diff --git a/executable_semantics/testdata/global_variable2.carbon b/executable_semantics/testdata/global_variable/write_from_function.carbon
similarity index 100%
rename from executable_semantics/testdata/global_variable2.carbon
rename to executable_semantics/testdata/global_variable/write_from_function.carbon
diff --git a/executable_semantics/testdata/if_else.carbon b/executable_semantics/testdata/if_else/if_else.carbon
similarity index 100%
rename from executable_semantics/testdata/if_else.carbon
rename to executable_semantics/testdata/if_else/if_else.carbon
diff --git a/executable_semantics/testdata/if_else_if.carbon b/executable_semantics/testdata/if_else/if_else_if.carbon
similarity index 100%
rename from executable_semantics/testdata/if_else_if.carbon
rename to executable_semantics/testdata/if_else/if_else_if.carbon
diff --git a/executable_semantics/testdata/if_else_if_else.carbon b/executable_semantics/testdata/if_else/if_else_if_else.carbon
similarity index 100%
rename from executable_semantics/testdata/if_else_if_else.carbon
rename to executable_semantics/testdata/if_else/if_else_if_else.carbon
diff --git a/executable_semantics/testdata/if_false.carbon b/executable_semantics/testdata/if_else/if_false.carbon
similarity index 100%
rename from executable_semantics/testdata/if_false.carbon
rename to executable_semantics/testdata/if_else/if_false.carbon
diff --git a/executable_semantics/testdata/if_nesting.carbon b/executable_semantics/testdata/if_else/if_nesting.carbon
similarity index 100%
rename from executable_semantics/testdata/if_nesting.carbon
rename to executable_semantics/testdata/if_else/if_nesting.carbon
diff --git a/executable_semantics/testdata/if_true.carbon b/executable_semantics/testdata/if_else/if_true.carbon
similarity index 100%
rename from executable_semantics/testdata/if_true.carbon
rename to executable_semantics/testdata/if_else/if_true.carbon
diff --git a/executable_semantics/testdata/import_nonexistent_library.carbon b/executable_semantics/testdata/import/fail_nonexistent_library.carbon
similarity index 91%
rename from executable_semantics/testdata/import_nonexistent_library.carbon
rename to executable_semantics/testdata/import/fail_nonexistent_library.carbon
index 44c6d2085320..eaa1576e0831 100644
--- a/executable_semantics/testdata/import_nonexistent_library.carbon
+++ b/executable_semantics/testdata/import/fail_nonexistent_library.carbon
@@ -2,6 +2,8 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
+// TODO: This SHOULD fail but doesn't presently.
+//
 // RUN: executable_semantics %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
 // RUN: executable_semantics --trace %s 2>&1 | \
diff --git a/executable_semantics/testdata/import_nonexistent_package.carbon b/executable_semantics/testdata/import/fail_nonexistent_package.carbon
similarity index 92%
rename from executable_semantics/testdata/import_nonexistent_package.carbon
rename to executable_semantics/testdata/import/fail_nonexistent_package.carbon
index eafa9359d2df..8d68d55ca0d0 100644
--- a/executable_semantics/testdata/import_nonexistent_package.carbon
+++ b/executable_semantics/testdata/import/fail_nonexistent_package.carbon
@@ -2,6 +2,8 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
+// TODO: This SHOULD fail but doesn't presently.
+//
 // RUN: executable_semantics %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
 // RUN: executable_semantics --trace %s 2>&1 | \
diff --git a/executable_semantics/testdata/import_late.carbon b/executable_semantics/testdata/import/fail_order.carbon
similarity index 79%
rename from executable_semantics/testdata/import_late.carbon
rename to executable_semantics/testdata/import/fail_order.carbon
index d4dd763f75cf..e7c182aa65f4 100644
--- a/executable_semantics/testdata/import_late.carbon
+++ b/executable_semantics/testdata/import/fail_order.carbon
@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/import_late.carbon:19: syntax error, unexpected IMPORT, expecting END_OF_FILE
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/import/fail_order.carbon:18: syntax error, unexpected IMPORT, expecting END_OF_FILE
 
 package ExecutableSemanticsTest api;
 
diff --git a/executable_semantics/testdata/match_any_int.carbon b/executable_semantics/testdata/match/any_int.carbon
similarity index 100%
rename from executable_semantics/testdata/match_any_int.carbon
rename to executable_semantics/testdata/match/any_int.carbon
diff --git a/executable_semantics/testdata/match_int.carbon b/executable_semantics/testdata/match/int.carbon
similarity index 100%
rename from executable_semantics/testdata/match_int.carbon
rename to executable_semantics/testdata/match/int.carbon
diff --git a/executable_semantics/testdata/match_int_default.carbon b/executable_semantics/testdata/match/int_default.carbon
similarity index 100%
rename from executable_semantics/testdata/match_int_default.carbon
rename to executable_semantics/testdata/match/int_default.carbon
diff --git a/executable_semantics/testdata/no_match.carbon b/executable_semantics/testdata/match/no_match.carbon
similarity index 100%
rename from executable_semantics/testdata/no_match.carbon
rename to executable_semantics/testdata/match/no_match.carbon
diff --git a/executable_semantics/testdata/match_placeholder.carbon b/executable_semantics/testdata/match/placeholder.carbon
similarity index 100%
rename from executable_semantics/testdata/match_placeholder.carbon
rename to executable_semantics/testdata/match/placeholder.carbon
diff --git a/executable_semantics/testdata/package_missing.carbon b/executable_semantics/testdata/package/fail_missing.carbon
similarity index 77%
rename from executable_semantics/testdata/package_missing.carbon
rename to executable_semantics/testdata/package/fail_missing.carbon
index b8e1e1af644f..85123adbc2f2 100644
--- a/executable_semantics/testdata/package_missing.carbon
+++ b/executable_semantics/testdata/package/fail_missing.carbon
@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/package_missing.carbon:13: syntax error, unexpected FN, expecting PACKAGE
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/package/fail_missing.carbon:12: syntax error, unexpected FN, expecting PACKAGE
 
 fn main() -> i32 {
   return 0;
diff --git a/executable_semantics/testdata/package_library.carbon b/executable_semantics/testdata/package/with_library.carbon
similarity index 100%
rename from executable_semantics/testdata/package_library.carbon
rename to executable_semantics/testdata/package/with_library.carbon
diff --git a/executable_semantics/testdata/return_empty_explicit.carbon b/executable_semantics/testdata/return/explicit_empty.carbon
similarity index 100%
rename from executable_semantics/testdata/return_empty_explicit.carbon
rename to executable_semantics/testdata/return/explicit_empty.carbon
diff --git a/executable_semantics/testdata/return_auto.carbon b/executable_semantics/testdata/return/fail_auto.carbon
similarity index 82%
rename from executable_semantics/testdata/return_auto.carbon
rename to executable_semantics/testdata/return/fail_auto.carbon
index 7ba06582a02b..53c378132199 100644
--- a/executable_semantics/testdata/return_auto.carbon
+++ b/executable_semantics/testdata/return/fail_auto.carbon
@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/return_auto.carbon:15: syntax error, unexpected AUTO
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/return/fail_auto.carbon:14: syntax error, unexpected AUTO
 
 package ExecutableSemanticsTest api;
 
diff --git a/executable_semantics/testdata/return_empty_implicit_fail1.carbon b/executable_semantics/testdata/return/fail_explicit_with_no_return.carbon
similarity index 72%
rename from executable_semantics/testdata/return_empty_implicit_fail1.carbon
rename to executable_semantics/testdata/return/fail_explicit_with_no_return.carbon
index 5dff75012eda..8514008f377a 100644
--- a/executable_semantics/testdata/return_empty_implicit_fail1.carbon
+++ b/executable_semantics/testdata/return/fail_explicit_with_no_return.carbon
@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/return_empty_implicit_fail1.carbon:16: control-flow reaches end of function that provides a `->` return type without reaching a return statement
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/return/fail_explicit_with_no_return.carbon:15: control-flow reaches end of function that provides a `->` return type without reaching a return statement
 
 package ExecutableSemanticsTest api;
 
diff --git a/executable_semantics/testdata/return_empty_implicit_fail2.carbon b/executable_semantics/testdata/return/fail_explicit_with_plain_return.carbon
similarity index 75%
rename from executable_semantics/testdata/return_empty_implicit_fail2.carbon
rename to executable_semantics/testdata/return/fail_explicit_with_plain_return.carbon
index a8f825e7cba7..2445f54a9f92 100644
--- a/executable_semantics/testdata/return_empty_implicit_fail2.carbon
+++ b/executable_semantics/testdata/return/fail_explicit_with_plain_return.carbon
@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/return_empty_implicit_fail2.carbon:16: return; should provide a return value, to match the function's signature.
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/return/fail_explicit_with_plain_return.carbon:15: return; should provide a return value, to match the function's signature.
 
 package ExecutableSemanticsTest api;
 
diff --git a/executable_semantics/testdata/return_empty_explicit_fail.carbon b/executable_semantics/testdata/return/fail_implicit_with_explicit_return.carbon
similarity index 74%
rename from executable_semantics/testdata/return_empty_explicit_fail.carbon
rename to executable_semantics/testdata/return/fail_implicit_with_explicit_return.carbon
index 24972f5ffb10..c20e6d34eed6 100644
--- a/executable_semantics/testdata/return_empty_explicit_fail.carbon
+++ b/executable_semantics/testdata/return/fail_implicit_with_explicit_return.carbon
@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/return_empty_explicit_fail.carbon:16: return (); should not provide a return value, to match the function's signature.
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/return/fail_implicit_with_explicit_return.carbon:15: return (); should not provide a return value, to match the function's signature.
 
 package ExecutableSemanticsTest api;
 
diff --git a/executable_semantics/testdata/return_empty_implicit1.carbon b/executable_semantics/testdata/return/implicit_with_no_return.carbon
similarity index 100%
rename from executable_semantics/testdata/return_empty_implicit1.carbon
rename to executable_semantics/testdata/return/implicit_with_no_return.carbon
diff --git a/executable_semantics/testdata/return_empty_implicit2.carbon b/executable_semantics/testdata/return/implicit_with_plain_return.carbon
similarity index 100%
rename from executable_semantics/testdata/return_empty_implicit2.carbon
rename to executable_semantics/testdata/return/implicit_with_plain_return.carbon
diff --git a/executable_semantics/testdata/string1.carbon b/executable_semantics/testdata/string/basic.carbon
similarity index 100%
rename from executable_semantics/testdata/string1.carbon
rename to executable_semantics/testdata/string/basic.carbon
diff --git a/executable_semantics/testdata/string_fail3.carbon b/executable_semantics/testdata/string/fail_hex_lower.carbon
similarity index 80%
rename from executable_semantics/testdata/string_fail3.carbon
rename to executable_semantics/testdata/string/fail_hex_lower.carbon
index b77ffba067bd..8456c75da6ef 100644
--- a/executable_semantics/testdata/string_fail3.carbon
+++ b/executable_semantics/testdata/string/fail_hex_lower.carbon
@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/string_fail3.carbon:16: Invalid escaping in string: "str\xaa"
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/string/fail_hex_lower.carbon:15: Invalid escaping in string: "str\xaa"
 
 package ExecutableSemanticsTest api;
 
diff --git a/executable_semantics/testdata/string_fail2.carbon b/executable_semantics/testdata/string/fail_hex_truncated.carbon
similarity index 80%
rename from executable_semantics/testdata/string_fail2.carbon
rename to executable_semantics/testdata/string/fail_hex_truncated.carbon
index c214552ae9bf..972b1437debf 100644
--- a/executable_semantics/testdata/string_fail2.carbon
+++ b/executable_semantics/testdata/string/fail_hex_truncated.carbon
@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/string_fail2.carbon:16: Invalid escaping in string: "str\x"
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/string/fail_hex_truncated.carbon:15: Invalid escaping in string: "str\x"
 
 package ExecutableSemanticsTest api;
 
diff --git a/executable_semantics/testdata/string_fail1.carbon b/executable_semantics/testdata/string/fail_invalid_escape.carbon
similarity index 80%
rename from executable_semantics/testdata/string_fail1.carbon
rename to executable_semantics/testdata/string/fail_invalid_escape.carbon
index 040ff097f577..ea97a7b189d9 100644
--- a/executable_semantics/testdata/string_fail1.carbon
+++ b/executable_semantics/testdata/string/fail_invalid_escape.carbon
@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/string_fail1.carbon:16: Invalid escaping in string: "str\e"
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/string/fail_invalid_escape.carbon:15: Invalid escaping in string: "str\e"
 
 package ExecutableSemanticsTest api;
 
diff --git a/executable_semantics/testdata/string_fail5.carbon b/executable_semantics/testdata/string/fail_newline.carbon
similarity index 80%
rename from executable_semantics/testdata/string_fail5.carbon
rename to executable_semantics/testdata/string/fail_newline.carbon
index fb092a9a249d..35f75ca2b58d 100644
--- a/executable_semantics/testdata/string_fail5.carbon
+++ b/executable_semantics/testdata/string/fail_newline.carbon
@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/string_fail5.carbon:16: invalid character '\x22' in source file.
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/string/fail_newline.carbon:15: invalid character '\x22' in source file.
 
 package ExecutableSemanticsTest api;
 
diff --git a/executable_semantics/testdata/string_fail4.carbon b/executable_semantics/testdata/string/fail_octal.carbon
similarity index 80%
rename from executable_semantics/testdata/string_fail4.carbon
rename to executable_semantics/testdata/string/fail_octal.carbon
index 163627f25c9c..c1930344c2d4 100644
--- a/executable_semantics/testdata/string_fail4.carbon
+++ b/executable_semantics/testdata/string/fail_octal.carbon
@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/string_fail4.carbon:16: Invalid escaping in string: "str\01"
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/string/fail_octal.carbon:15: Invalid escaping in string: "str\01"
 
 package ExecutableSemanticsTest api;
 
diff --git a/executable_semantics/testdata/string_fail6.carbon b/executable_semantics/testdata/string/fail_tab.carbon
similarity index 81%
rename from executable_semantics/testdata/string_fail6.carbon
rename to executable_semantics/testdata/string/fail_tab.carbon
index 152a0d92c7eb..511a27c5c6a8 100644
--- a/executable_semantics/testdata/string_fail6.carbon
+++ b/executable_semantics/testdata/string/fail_tab.carbon
@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/string_fail6.carbon:16: Invalid escaping in string: "new	line"
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/string/fail_tab.carbon:15: Invalid escaping in string: "new	line"
 
 package ExecutableSemanticsTest api;
 
diff --git a/executable_semantics/testdata/string3.carbon b/executable_semantics/testdata/string/hex.carbon
similarity index 100%
rename from executable_semantics/testdata/string3.carbon
rename to executable_semantics/testdata/string/hex.carbon
diff --git a/executable_semantics/testdata/string2.carbon b/executable_semantics/testdata/string/newline.carbon
similarity index 100%
rename from executable_semantics/testdata/string2.carbon
rename to executable_semantics/testdata/string/newline.carbon
diff --git a/executable_semantics/testdata/tuple_assign.carbon b/executable_semantics/testdata/tuple/assign.carbon
similarity index 100%
rename from executable_semantics/testdata/tuple_assign.carbon
rename to executable_semantics/testdata/tuple/assign.carbon
diff --git a/executable_semantics/testdata/tuple2.carbon b/executable_semantics/testdata/tuple/ending_comma.carbon
similarity index 100%
rename from executable_semantics/testdata/tuple2.carbon
rename to executable_semantics/testdata/tuple/ending_comma.carbon
diff --git a/executable_semantics/testdata/tuple_equality.carbon b/executable_semantics/testdata/tuple/equality.carbon
similarity index 100%
rename from executable_semantics/testdata/tuple_equality.carbon
rename to executable_semantics/testdata/tuple/equality.carbon
diff --git a/executable_semantics/testdata/tuple_equality2.carbon b/executable_semantics/testdata/tuple/equality_false.carbon
similarity index 100%
rename from executable_semantics/testdata/tuple_equality2.carbon
rename to executable_semantics/testdata/tuple/equality_false.carbon
diff --git a/executable_semantics/testdata/tuple_equality3.carbon b/executable_semantics/testdata/tuple/fail_equality_type.carbon
similarity index 81%
rename from executable_semantics/testdata/tuple_equality3.carbon
rename to executable_semantics/testdata/tuple/fail_equality_type.carbon
index c86edb49b810..2a812b5a68c0 100644
--- a/executable_semantics/testdata/tuple_equality3.carbon
+++ b/executable_semantics/testdata/tuple/fail_equality_type.carbon
@@ -3,6 +3,10 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 // RUN: not executable_semantics %s 2>&1 2>&1 | FileCheck %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/tuple/fail_equality_type.carbon:16: type error in ==
+// CHECK: expected: (0 = i32, 1 = i32)
+// CHECK: actual: (0 = i32)
 
 package ExecutableSemanticsTest api;
 
@@ -15,8 +19,3 @@ fn main() -> i32 {
     return 0;
   }
 }
-
-// AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/tuple_equality3.carbon:12: type error in ==
-// CHECK: expected: (0 = i32, 1 = i32)
-// CHECK: actual: (0 = i32)
diff --git a/executable_semantics/testdata/tuple5.carbon b/executable_semantics/testdata/tuple/fail_name_order.carbon
similarity index 80%
rename from executable_semantics/testdata/tuple5.carbon
rename to executable_semantics/testdata/tuple/fail_name_order.carbon
index 53eba00a67c1..8f65f52f819c 100644
--- a/executable_semantics/testdata/tuple5.carbon
+++ b/executable_semantics/testdata/tuple/fail_name_order.carbon
@@ -7,7 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: PROGRAM ERROR: {{.*}}/tuple5.carbon:17: Tuple field name 'y' does not match pattern field name 'x'
+// CHECK: PROGRAM ERROR: {{.*}}/executable_semantics/testdata/tuple/fail_name_order.carbon:17: Tuple field name 'y' does not match pattern field name 'x'
 
 package ExecutableSemanticsTest api;
 
diff --git a/executable_semantics/testdata/tuple4.carbon b/executable_semantics/testdata/tuple/fail_positional_order.carbon
similarity index 79%
rename from executable_semantics/testdata/tuple4.carbon
rename to executable_semantics/testdata/tuple/fail_positional_order.carbon
index 832b8628a5aa..7f87a04129b4 100644
--- a/executable_semantics/testdata/tuple4.carbon
+++ b/executable_semantics/testdata/tuple/fail_positional_order.carbon
@@ -7,7 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: PROGRAM ERROR: {{.*}}/tuple4.carbon:15: positional members must come before named members
+// CHECK: PROGRAM ERROR: {{.*}}/executable_semantics/testdata/tuple/fail_positional_order.carbon:15: positional members must come before named members
 
 package ExecutableSemanticsTest api;
 
diff --git a/executable_semantics/testdata/tuple_match.carbon b/executable_semantics/testdata/tuple/match.carbon
similarity index 100%
rename from executable_semantics/testdata/tuple_match.carbon
rename to executable_semantics/testdata/tuple/match.carbon
diff --git a/executable_semantics/testdata/tuple_match3.carbon b/executable_semantics/testdata/tuple/match_nested.carbon
similarity index 100%
rename from executable_semantics/testdata/tuple_match3.carbon
rename to executable_semantics/testdata/tuple/match_nested.carbon
diff --git a/executable_semantics/testdata/tuple_match2.carbon b/executable_semantics/testdata/tuple/match_with_named.carbon
similarity index 100%
rename from executable_semantics/testdata/tuple_match2.carbon
rename to executable_semantics/testdata/tuple/match_with_named.carbon
diff --git a/executable_semantics/testdata/tuple3.carbon b/executable_semantics/testdata/tuple/named.carbon
similarity index 100%
rename from executable_semantics/testdata/tuple3.carbon
rename to executable_semantics/testdata/tuple/named.carbon
diff --git a/executable_semantics/testdata/tuple1.carbon b/executable_semantics/testdata/tuple/no_ending_comma.carbon
similarity index 100%
rename from executable_semantics/testdata/tuple1.carbon
rename to executable_semantics/testdata/tuple/no_ending_comma.carbon
diff --git a/executable_semantics/testdata/while1.carbon b/executable_semantics/testdata/while/basic.carbon
similarity index 100%
rename from executable_semantics/testdata/while1.carbon
rename to executable_semantics/testdata/while/basic.carbon
diff --git a/executable_semantics/testdata/break1.carbon b/executable_semantics/testdata/while/break.carbon
similarity index 100%
rename from executable_semantics/testdata/break1.carbon
rename to executable_semantics/testdata/while/break.carbon
diff --git a/executable_semantics/testdata/continue1.carbon b/executable_semantics/testdata/while/continue.carbon
similarity index 100%
rename from executable_semantics/testdata/continue1.carbon
rename to executable_semantics/testdata/while/continue.carbon
diff --git a/executable_semantics/update_checks.py b/executable_semantics/update_checks.py
index e88a8705dc14..86ed7f4a81be 100755
--- a/executable_semantics/update_checks.py
+++ b/executable_semantics/update_checks.py
@@ -24,30 +24,28 @@ _AUTOUPDATE_MARKER = "// AUTOUPDATE: executable_semantics %s\n"
 def _get_tests():
     """Get the list of tests from the filesystem."""
     tests = set()
-    for path in list(os.listdir(_TESTDATA)):
-        f = os.path.basename(path)
-        if f == "lit.cfg":
-            # Ignore the lit config.
-            continue
-        basename, ext = os.path.splitext(f)
-        if ext == ".carbon":
-            tests.add(basename)
-        else:
-            sys.exit("Unrecognized file type in testdata: %s" % f)
+    for root, _, files in os.walk(_TESTDATA):
+        for f in files:
+            if f == "lit.cfg":
+                # Ignore the lit config.
+                continue
+            if os.path.splitext(f)[1] == ".carbon":
+                tests.add(os.path.join(root, f))
+            else:
+                sys.exit("Unrecognized file type in testdata: %s" % f)
     return tests
 
 
 def _update_check(test):
     """Updates the CHECK: lines for `test` by running executable_semantics."""
-    test_file = "%s/%s.carbon" % (_TESTDATA, test)
-    with open(test_file) as f:
+    with open(test) as f:
         orig_lines = f.readlines()
     if _AUTOUPDATE_MARKER not in orig_lines:
-        raise ValueError("No autoupdate marker in %s" % test_file)
+        raise ValueError("No autoupdate marker in %s" % test)
     # Run executable_semantics to general output.
     # (`bazel run` would serialize)
     p = subprocess.run(
-        ["%s/executable_semantics" % _BINDIR, test_file],
+        ["%s/executable_semantics" % _BINDIR, test],
         stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT,
     )
@@ -57,7 +55,7 @@ def _update_check(test):
     # when used.
     # TODO: Maybe revisit and see if lit can be convinced to give a
     # root-relative path.
-    out = out.replace(test_file, "{{.*}}/%s.carbon" % test)
+    out = out.replace(test, "{{.*}}/%s" % test)
 
     # Remove old OUT.
     lines_without_check = [
@@ -65,7 +63,7 @@ def _update_check(test):
     ]
     autoupdate_index = lines_without_check.index(_AUTOUPDATE_MARKER)
     assert autoupdate_index >= 0
-    with open(test_file, "w") as f:
+    with open(test, "w") as f:
         f.writelines(lines_without_check[: autoupdate_index + 1])
         f.writelines(["// CHECK: %s\n" % x for x in out.splitlines()])
         f.writelines(lines_without_check[autoupdate_index + 1 :])