Reorganize tests into dirs and rename numbered tests (#825)

There's a small update to update_checks.py to handle the recursive directories. Also, I'm only using one level of nesting in this PR but really no reason we can't do more. I'm just not sure what clustering is best right now.

As a pattern, I'm trying to name all failing tests `fail_*.carbon`.

executable_semantics/testdata/invalid_char.carbon → executable_semantics/testdata/basic_syntax/fail_invalid_char.carbon

@@ -7,7 +7,6 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/invalid_char.carbon:13: invalid character '\xEF' in source file.
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/basic_syntax/fail_invalid_char.carbon:12: invalid character '\xEF' in source file.
 
 �

executable_semantics/testdata/pattern_variable_fail.carbon → executable_semantics/testdata/basic_syntax/fail_missing_var.carbon

@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/pattern_variable_fail.carbon:17: syntax error, unexpected COLON
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/basic_syntax/fail_missing_var.carbon:16: syntax error, unexpected COLON
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/class_field_access_mismatch.carbon → executable_semantics/testdata/class/fail_field_access_mismatch.carbon

@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/class_field_access_mismatch.carbon:21: class Point does not have a field named z
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/class/fail_field_access_mismatch.carbon:20: class Point does not have a field named z
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/class_field_mismatch.carbon → executable_semantics/testdata/class/fail_field_mismatch.carbon

@@ -7,11 +7,10 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/class_field_mismatch.carbon:23: type error in call
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/class/fail_field_mismatch.carbon:22: type error in call
 // CHECK: expected: (x = i32, y = i32)
 // CHECK: actual: (x = i32, z = i32)
 
-
 package ExecutableSemanticsTest api;
 
 class Point {

executable_semantics/testdata/class_field_missing.carbon → executable_semantics/testdata/class/fail_field_missing.carbon

@@ -7,11 +7,10 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/class_field_missing.carbon:23: type error in call
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/class/fail_field_missing.carbon:22: type error in call
 // CHECK: expected: (x = i32, y = i32)
 // CHECK: actual: (x = i32)
 
-
 package ExecutableSemanticsTest api;
 
 class Point {

executable_semantics/testdata/experimental_continuation9.carbon → executable_semantics/testdata/experimental_continuation/fail_lifetime.carbon

@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: RUNTIME ERROR: {{.*}}/experimental_continuation9.carbon:22: undefined behavior: access to dead value 1
-
+// CHECK: RUNTIME ERROR: {{.*}}/executable_semantics/testdata/experimental_continuation/fail_lifetime.carbon:21: undefined behavior: access to dead value 1
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/fun_named_params2.carbon

@@ -1,19 +0,0 @@
-// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
-// Exceptions. See /LICENSE for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-// RUN: not executable_semantics %s 2>&1 | \
-// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
-// RUN: not executable_semantics --trace %s 2>&1 | \
-// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
-// AUTOUPDATE: executable_semantics %s
-// CHECK: PROGRAM ERROR: {{.*}}/fun_named_params2.carbon:15: positional members must come before named members
-
-
-package ExecutableSemanticsTest api;
-
-fn f(x: i32, .d = y: i32, z: i32, .e = a: i32) => (x + y) - (z + a);
-
-fn main() -> i32 {
-  return 0;
-}

executable_semantics/testdata/fun6_fail_type.carbon → executable_semantics/testdata/function/fail_call_with_tuple.carbon

@@ -7,11 +7,10 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/fun6_fail_type.carbon:22: type error in call
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/function/fail_call_with_tuple.carbon:21: type error in call
 // CHECK: expected: (0 = i32, 1 = i32)
 // CHECK: actual: (0 = (0 = i32, 1 = i32))
 
-
 package ExecutableSemanticsTest api;
 
 fn f(x: i32, y: i32) -> i32 { return x + y; }

executable_semantics/testdata/function/fail_named_params_order.carbon

@@ -0,0 +1,20 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: not executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: not executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: PROGRAM ERROR: {{.*}}/executable_semantics/testdata/function/fail_named_params_order.carbon:14: positional members must come before named members
+
+package ExecutableSemanticsTest api;
+
+fn f(x: i32, .d = y: i32, z: i32, .e = a: i32) -> i32 {
+  return (x + y) - (z + a);
+}
+
+fn main() -> i32 {
+  return 0;
+}

executable_semantics/testdata/fun_named_params.carbon → executable_semantics/testdata/function/named_params.carbon

@@ -11,7 +11,9 @@
 
 package ExecutableSemanticsTest api;
 
-fn f(x: i32, .d = y: i32) => x + y;
+fn f(x: i32, .d = y: i32) -> i32 {
+  return x + y;
+}
 
 fn main() -> i32 {
   return f(1, .d = 2) - 3;

executable_semantics/testdata/generic_function_fail3.carbon → executable_semantics/testdata/generic_function/fail_not_addable.carbon

@@ -7,11 +7,10 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/generic_function_fail3.carbon:18: type error in addition(1)
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/generic_function/fail_not_addable.carbon:17: type error in addition(1)
 // CHECK: expected: i32
 // CHECK: actual: T
 
-
 package ExecutableSemanticsTest api;
 
 fn id[T:! Type](x: T) -> T {

executable_semantics/testdata/generic_function_fail1.carbon → executable_semantics/testdata/generic_function/fail_type_deduction_mismatch.carbon

@@ -7,11 +7,10 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/generic_function_fail1.carbon:22: type error in argument deduction
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/generic_function/fail_type_deduction_mismatch.carbon:21: type error in argument deduction
 // CHECK: expected: i32
 // CHECK: actual: Bool
 
-
 package ExecutableSemanticsTest api;
 
 fn fst[T:! Type](x: T, y: T) -> T {

executable_semantics/testdata/generic_function_fail2.carbon → executable_semantics/testdata/generic_function/fail_type_deduction_unused.carbon

@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/generic_function_fail2.carbon:20: could not deduce type argument for type parameter T
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/generic_function/fail_type_deduction_unused.carbon:19: could not deduce type argument for type parameter T
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/global_variable8.carbon → executable_semantics/testdata/global_variable/fail_init_order.carbon

@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: RUNTIME ERROR: {{.*}}/global_variable8.carbon:18: could not find `y`
-
+// CHECK: RUNTIME ERROR: {{.*}}/executable_semantics/testdata/global_variable/fail_init_order.carbon:17: could not find `y`
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/global_variable3.carbon → executable_semantics/testdata/global_variable/fail_init_type_mismatch.carbon

@@ -7,11 +7,10 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/global_variable3.carbon:19: type error in initializer of variable
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/global_variable/fail_init_type_mismatch.carbon:18: type error in initializer of variable
 // CHECK: expected: i32
 // CHECK: actual: Bool
 
-
 package ExecutableSemanticsTest api;
 
 // Test type checking of global variable. Error expected.

executable_semantics/testdata/global_variable5.carbon → executable_semantics/testdata/global_variable/shadowing.carbon

@@ -7,11 +7,10 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/global_variable5.carbon:22: type error in return
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/global_variable/shadowing.carbon:21: type error in return
 // CHECK: expected: ()
 // CHECK: actual: i32
 
-
 package ExecutableSemanticsTest api;
 
 // Test overshadowing of global variable.

executable_semantics/testdata/import_nonexistent_library.carbon → executable_semantics/testdata/import/fail_nonexistent_library.carbon

@@ -2,6 +2,8 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
+// TODO: This SHOULD fail but doesn't presently.
+//
 // RUN: executable_semantics %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
 // RUN: executable_semantics --trace %s 2>&1 | \

executable_semantics/testdata/import_nonexistent_package.carbon → executable_semantics/testdata/import/fail_nonexistent_package.carbon

@@ -2,6 +2,8 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
+// TODO: This SHOULD fail but doesn't presently.
+//
 // RUN: executable_semantics %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
 // RUN: executable_semantics --trace %s 2>&1 | \

executable_semantics/testdata/import_late.carbon → executable_semantics/testdata/import/fail_order.carbon

@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/import_late.carbon:19: syntax error, unexpected IMPORT, expecting END_OF_FILE
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/import/fail_order.carbon:18: syntax error, unexpected IMPORT, expecting END_OF_FILE
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/package_missing.carbon → executable_semantics/testdata/package/fail_missing.carbon

@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/package_missing.carbon:13: syntax error, unexpected FN, expecting PACKAGE
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/package/fail_missing.carbon:12: syntax error, unexpected FN, expecting PACKAGE
 
 fn main() -> i32 {
   return 0;

executable_semantics/testdata/return_auto.carbon → executable_semantics/testdata/return/fail_auto.carbon

@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/return_auto.carbon:15: syntax error, unexpected AUTO
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/return/fail_auto.carbon:14: syntax error, unexpected AUTO
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/return_empty_implicit_fail1.carbon → executable_semantics/testdata/return/fail_explicit_with_no_return.carbon

@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/return_empty_implicit_fail1.carbon:16: control-flow reaches end of function that provides a `->` return type without reaching a return statement
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/return/fail_explicit_with_no_return.carbon:15: control-flow reaches end of function that provides a `->` return type without reaching a return statement
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/return_empty_implicit_fail2.carbon → executable_semantics/testdata/return/fail_explicit_with_plain_return.carbon

@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/return_empty_implicit_fail2.carbon:16: return; should provide a return value, to match the function's signature.
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/return/fail_explicit_with_plain_return.carbon:15: return; should provide a return value, to match the function's signature.
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/return_empty_explicit_fail.carbon → executable_semantics/testdata/return/fail_implicit_with_explicit_return.carbon

@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/return_empty_explicit_fail.carbon:16: return (); should not provide a return value, to match the function's signature.
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/return/fail_implicit_with_explicit_return.carbon:15: return (); should not provide a return value, to match the function's signature.
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/string_fail3.carbon → executable_semantics/testdata/string/fail_hex_lower.carbon

@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/string_fail3.carbon:16: Invalid escaping in string: "str\xaa"
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/string/fail_hex_lower.carbon:15: Invalid escaping in string: "str\xaa"
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/string_fail2.carbon → executable_semantics/testdata/string/fail_hex_truncated.carbon

@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/string_fail2.carbon:16: Invalid escaping in string: "str\x"
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/string/fail_hex_truncated.carbon:15: Invalid escaping in string: "str\x"
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/string_fail1.carbon → executable_semantics/testdata/string/fail_invalid_escape.carbon

@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/string_fail1.carbon:16: Invalid escaping in string: "str\e"
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/string/fail_invalid_escape.carbon:15: Invalid escaping in string: "str\e"
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/string_fail5.carbon → executable_semantics/testdata/string/fail_newline.carbon

@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/string_fail5.carbon:16: invalid character '\x22' in source file.
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/string/fail_newline.carbon:15: invalid character '\x22' in source file.
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/string_fail4.carbon → executable_semantics/testdata/string/fail_octal.carbon

@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/string_fail4.carbon:16: Invalid escaping in string: "str\01"
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/string/fail_octal.carbon:15: Invalid escaping in string: "str\01"
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/string_fail6.carbon → executable_semantics/testdata/string/fail_tab.carbon

@@ -7,8 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/string_fail6.carbon:16: Invalid escaping in string: "new	line"
-
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/string/fail_tab.carbon:15: Invalid escaping in string: "new	line"
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/tuple_equality3.carbon → executable_semantics/testdata/tuple/fail_equality_type.carbon

@@ -3,6 +3,10 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 // RUN: not executable_semantics %s 2>&1 2>&1 | FileCheck %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/tuple/fail_equality_type.carbon:16: type error in ==
+// CHECK: expected: (0 = i32, 1 = i32)
+// CHECK: actual: (0 = i32)
 
 package ExecutableSemanticsTest api;
 
@@ -15,8 +19,3 @@ fn main() -> i32 {
     return 0;
   }
 }
-
-// AUTOUPDATE: executable_semantics %s
-// CHECK: COMPILATION ERROR: {{.*}}/tuple_equality3.carbon:12: type error in ==
-// CHECK: expected: (0 = i32, 1 = i32)
-// CHECK: actual: (0 = i32)

executable_semantics/testdata/tuple5.carbon → executable_semantics/testdata/tuple/fail_name_order.carbon

@@ -7,7 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: PROGRAM ERROR: {{.*}}/tuple5.carbon:17: Tuple field name 'y' does not match pattern field name 'x'
+// CHECK: PROGRAM ERROR: {{.*}}/executable_semantics/testdata/tuple/fail_name_order.carbon:17: Tuple field name 'y' does not match pattern field name 'x'
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/tuple4.carbon → executable_semantics/testdata/tuple/fail_positional_order.carbon

@@ -7,7 +7,7 @@
 // RUN: not executable_semantics --trace %s 2>&1 | \
 // RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
 // AUTOUPDATE: executable_semantics %s
-// CHECK: PROGRAM ERROR: {{.*}}/tuple4.carbon:15: positional members must come before named members
+// CHECK: PROGRAM ERROR: {{.*}}/executable_semantics/testdata/tuple/fail_positional_order.carbon:15: positional members must come before named members
 
 package ExecutableSemanticsTest api;