Switch executable_semantics to lit/filecheck tests. (#819)

.pre-commit-config.yaml

@@ -86,14 +86,6 @@ repos:
         language: python
         files: ^executable_semantics/syntax/parser.ypp$
         pass_filenames: false
-      - id: executable-semantics-tests
-        name: Update list of executable_semantics tests
-        description: Updates executable_semantics/test_list.bzl
-        entry: executable_semantics/tests.py
-        language: python
-        files: ^executable_semantics/(test_list.bzl|testdata/.*\.carbon)$
-        pass_filenames: false
-        args: ['--update_list', '--use_git_ls_files']
       - id: proposal-list
         name: Update list of proposals
         description: Updates the list of proposals in proposals/README.md

bazel/testing/BUILD

@@ -2,4 +2,4 @@
 # Exceptions. See /LICENSE for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-exports_files(["golden_test.py"])
+exports_files(["lit_test.py"])

bazel/testing/golden_test.bzl

@@ -1,33 +0,0 @@
-# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
-# Exceptions. See /LICENSE for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-"""Rule for a golden test."""
-
-def golden_test(name, golden, cmd, data, env = None, golden_is_subset = False, **kwargs):
-    """Compares two files. Passes if they are identical.
-
-    Args:
-      name: Name of the build rule.
-      cmd: The command whose output is being tested.
-      golden: The golden file to be compared against the command output.
-      data: Data files.
-      env: Optional environment.
-      golden_is_subset: Set to True if the golden file should be a subset of
-        command output.
-      **kwargs: Any additional parameters for the generated py_test.
-    """
-    args = ["$(location %s)" % golden, cmd]
-    if not env:
-        env = {}
-    if golden_is_subset:
-        args.append("--golden_is_subset")
-    native.py_test(
-        name = name,
-        srcs = ["//bazel/testing:golden_test.py"],
-        main = "//bazel/testing:golden_test.py",
-        args = args,
-        data = [golden] + data,
-        env = env,
-        **kwargs
-    )

bazel/testing/golden_test.py

@@ -1,147 +0,0 @@
-#!/usr/bin/env python3
-
-"""Compare a command's output against an expected "golden" output file.
-
-Usage:
-
-golden_test.py <golden path> <command> [--update]
-
-<golden path> is the path to the golden file, and <command> is
-the command to run, including any arguments. If --update is specified,
-the command will be run and its output stored in the golden file.
-Otherwise, the command will be run and its output compared against
-the contents of the golden file.
-
-For these purposes, the command's output consists of the interleaved
-contents of stdout and stderr, as well as the command's exit code. Thus,
-golden tests can provide coverage of cases where the command is expected
-to fail, as well as cases where it's expected to succeed.
-
-This script is designed to be run by a `golden_test` Bazel rule,
-and may not work when run outside that context.
-"""
-
-__copyright__ = """
-Part of the Carbon Language project, under the Apache License v2.0 with LLVM
-Exceptions. See /LICENSE for license information.
-SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-"""
-
-import argparse
-import difflib
-import os
-import subprocess
-import sys
-
-
-_ERROR_MESSAGE = """When running under:
-  {dir}
-the golden contents of:
-  {golden_path}
-do not match generated output of:
-  {subject_cmd_args}
-"""
-
-_UPDATE_MESSAGE = """To update the golden file, run the following:
-
-  bazel run {test_target} -- --update
-"""
-
-
-def _parse_args():
-    """Parses command line arguments, returning the result."""
-    arg_parser = argparse.ArgumentParser(description=__doc__)
-    arg_parser.add_argument("golden_path", help="The path to the golden file.")
-    arg_parser.add_argument(
-        "subject_command", help="The command line to compare output with."
-    )
-    arg_parser.add_argument(
-        "--golden_is_subset",
-        action="store_true",
-        help="Indicates that the golden file will be a subset of output, "
-        "rather than full output.",
-    )
-    arg_parser.add_argument(
-        "--update",
-        action="store_true",
-        help="Whether to update the golden file.",
-    )
-    return arg_parser.parse_args()
-
-
-def _get_subject_output(args):
-    """Returns output from the subject command."""
-    subject_cmd = subprocess.run(
-        args=args.subject_command.split(),
-        stdout=subprocess.PIPE,  # Capture stdout as a string
-        stderr=subprocess.STDOUT,  # Send stderr to the same place as stdout
-        universal_newlines=True,
-    )
-
-    subject = subject_cmd.stdout
-    if subject_cmd.returncode != 0:
-        subject += "EXIT CODE: {0}\n".format(subject_cmd.returncode)
-
-    return subject
-
-
-def _check_diff(args, subject):
-    """Prints and checks the diff. Returns the appropriate exit code."""
-    subject_lines = subject.splitlines(keepends=True)
-    with open(args.golden_path) as golden:
-        golden_lines = list(golden.readlines())
-    if args.golden_is_subset:
-        golden_set = frozenset(golden_lines)
-        subject_lines = [line for line in subject_lines if line in golden_set]
-    context_diff = list(
-        difflib.context_diff(
-            subject_lines, golden_lines, fromfile="subject", tofile="golden"
-        )
-    )
-    if context_diff:
-        if args.golden_is_subset:
-            # Print subject output for context, because it may be useful in
-            # debugging.
-            print("=" * 80)
-            print("Subject output (including ignored lines)")
-            print("=" * 80)
-            print(subject)
-        print("=" * 80)
-        print("Output diff")
-        print("=" * 80)
-        sys.stdout.writelines(context_diff)
-        print("=" * 80)
-        print(
-            _ERROR_MESSAGE.format(
-                dir=os.getenv("TEST_SRCDIR"),
-                golden_path=args.golden_path,
-                subject_cmd_args=args.subject_command,
-            )
-        )
-        if not args.golden_is_subset:
-            print(
-                _UPDATE_MESSAGE.format(
-                    test_target=os.getenv("TEST_TARGET"),
-                )
-            )
-
-        return 1
-    else:
-        print("PASS")
-        return 0
-
-
-def main():
-    args = _parse_args()
-    subject = _get_subject_output(args)
-
-    if args.update:
-        with open(args.golden_path, "w") as golden:
-            golden.write(subject)
-            return 0
-
-    return _check_diff(args, subject)
-
-
-if __name__ == "__main__":
-    sys.exit(main())

bazel/testing/lit_test.bzl

@@ -0,0 +1,38 @@
+# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+# Exceptions. See /LICENSE for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+"""Rule for a lit test."""
+
+def lit_test(name, test_dir, tools = None, **kwargs):
+    """Runs `lit` on test_dir.
+
+    `lit` reference:
+      https://llvm.org/docs/CommandGuide/lit.html
+
+    To pass flags to `lit`, use `--test_arg`. For example:
+      bazel test :lit_test --test_arg=-v
+      bazel test :lit_test --test_arg=--filter=REGEXP
+
+    Args:
+      name: Name of the build rule.
+      test_dir: The directory with the lit tests.
+      tools: An optional list of tools to provide to the tests. These will be
+        aliased for execution.
+      **kwargs: Any additional parameters for the generated py_test.
+    """
+    if not tools:
+        tools = []
+    tools += [
+        "@llvm-project//llvm:lit",
+        "@llvm-project//llvm:not",
+        "@llvm-project//llvm:FileCheck",
+    ]
+    native.py_test(
+        name = name,
+        srcs = ["//bazel/testing:lit_test.py"],
+        main = "//bazel/testing:lit_test.py",
+        data = tools + native.glob([test_dir + "/**"]),
+        args = ["--tool=%s" % t for t in tools] + [test_dir, "--"],
+        **kwargs
+    )

bazel/testing/lit_test.py

@@ -0,0 +1,93 @@
+"""Runs `lit` for testing."""
+
+__copyright__ = """
+Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+Exceptions. See /LICENSE for license information.
+SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+"""
+
+import argparse
+import os
+import subprocess
+
+_PASSTHROUGH_FLAGS = ["filter", "filter-out"]
+
+
+def _parse_args():
+    """Parses command line arguments, returning the result."""
+    arg_parser = argparse.ArgumentParser(description=__doc__)
+    arg_parser.add_argument(
+        "test_dir", help="The directory containing tests to run."
+    )
+    arg_parser.add_argument(
+        "lit_args", nargs="*", help="Arguments to pass through to lit."
+    )
+    arg_parser.add_argument(
+        "--tool", action="append", help="A tool to add to the PATH."
+    )
+    return arg_parser.parse_args()
+
+
+def _normalize(relative_base, target):
+    """Given a target, normalizes it to a relative path."""
+    assert target
+    if target.startswith(":"):
+        # Local target; :foo -> my/dir/foo
+        return os.path.join(relative_base, target[1:])
+    elif target[0].isalpha():
+        # Local target; foo -> my/dir/foo
+        return os.path.join(relative_base, target)
+
+    if ":" in target:
+        # Specified target; //foo:bar -> //foo/bar
+        target = target.replace(":", "/")
+    else:
+        # Default target; //foo -> //foo/foo
+        target = os.path.join(target, os.path.basename(target))
+
+    if target.startswith("@"):
+        return os.path.join("external/", target[1:])
+    elif target.startswith("//"):
+        return target[2:]
+    else:
+        raise ValueError("Unhandled target path: %s" % target)
+
+
+def main():
+    parsed_args = _parse_args()
+
+    # A symlink directory is added to the PATH so that commands like `lit` and
+    # `not` can use the versions in the path.
+    symlink_dir = os.environ["TEST_TMPDIR"]
+
+    # Create symlinks to all the tools.
+    bin_dir = os.getcwd()
+    relative_base = os.path.dirname(_normalize("", os.environ["TEST_TARGET"]))
+    for tool in parsed_args.tool:
+        tool_path = _normalize(relative_base, tool)
+        symlink_loc = os.path.join(symlink_dir, os.path.basename(tool_path))
+        symlinked_file = os.path.join(bin_dir, tool_path)
+        if not os.path.exists(symlinked_file):
+            raise ValueError("Missing file: %s" % symlinked_file)
+        os.symlink(symlinked_file, symlink_loc)
+
+    # Figure out the actual path for the test_dir.
+    test_dir = os.path.join(
+        bin_dir, _normalize(relative_base, parsed_args.test_dir)
+    )
+
+    args = [
+        os.path.join(symlink_dir, "lit"),
+        "--path=%s" % symlink_dir,
+        test_dir,
+    ]
+
+    # Run lit.
+    p = subprocess.run(args=args + parsed_args.lit_args)
+    # Do this instead of check_call to hide stack traces.
+    if p.returncode != 0:
+        exit("lit failed, exit code %d" % p.returncode)
+
+
+if __name__ == "__main__":
+    exit(main())

executable_semantics/BUILD

@@ -2,8 +2,7 @@
 # Exceptions. See /LICENSE for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-load("//bazel/testing:golden_test.bzl", "golden_test")
-load("test_list.bzl", "TEST_LIST")
+load("//bazel/testing:lit_test.bzl", "lit_test")
 
 cc_binary(
     name = "executable_semantics",
@@ -16,38 +15,8 @@ cc_binary(
     ],
 )
 
-[golden_test(
-    name = "%s_test" % e,
-    cmd = "'$(location executable_semantics) $(location testdata/%s.carbon)'" % e,
-    data = [
-        ":executable_semantics",
-        "testdata/%s.carbon" % e,
-    ],
-    golden = "testdata/%s.golden" % e,
-) for e in TEST_LIST]
-
-# Convenience suite for running golden tests.
-test_suite(
-    name = "golden_tests",
-    tests = [":%s_test" % e for e in TEST_LIST],
-)
-
-# Test --trace by expecting golden output to be a *subset* of trace output. Note
-# the normal test must be used to update golden files.
-[golden_test(
-    name = "%s_trace_test" % e,
-    cmd = "'$(location executable_semantics) --trace " +
-          "$(location testdata/%s.carbon)'" % e,
-    data = [
-        ":executable_semantics",
-        "testdata/%s.carbon" % e,
-    ],
-    golden = "testdata/%s.golden" % e,
-    golden_is_subset = True,
-) for e in TEST_LIST]
-
-# Convenience suite for running trace tests.
-test_suite(
-    name = "trace_tests",
-    tests = [":%s_trace_test" % e for e in TEST_LIST],
+lit_test(
+    name = "executable_semantics_lit_test",
+    test_dir = "testdata",
+    tools = [":executable_semantics"],
 )

executable_semantics/test_list.bzl

@@ -1,115 +0,0 @@
-# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
-# Exceptions. See /LICENSE for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-"""Auto-generated list of tests. Run `./tests.py --update_list` to update."""
-
-TEST_LIST = [
-    "assignment_copy1",
-    "assignment_copy2",
-    "block1",
-    "block2",
-    "break1",
-    "choice1",
-    "class1",
-    "class2",
-    "class3",
-    "class_field_access_mismatch",
-    "class_field_mismatch",
-    "class_field_missing",
-    "continue1",
-    "experimental_continuation1",
-    "experimental_continuation2",
-    "experimental_continuation3",
-    "experimental_continuation4",
-    "experimental_continuation5",
-    "experimental_continuation6",
-    "experimental_continuation7",
-    "experimental_continuation9",
-    "fun1",
-    "fun2",
-    "fun3",
-    "fun4",
-    "fun5",
-    "fun6_fail_type",
-    "fun_named_params",
-    "fun_named_params2",
-    "fun_recur",
-    "funptr1",
-    "generic_function1",
-    "generic_function2",
-    "generic_function3",
-    "generic_function_apply",
-    "generic_function_fail1",
-    "generic_function_fail2",
-    "generic_function_fail3",
-    "generic_function_swap",
-    "generic_function_tuple_map",
-    "global_variable1",
-    "global_variable2",
-    "global_variable3",
-    "global_variable4",
-    "global_variable5",
-    "global_variable6",
-    "global_variable7",
-    "global_variable8",
-    "if_else",
-    "if_else_if",
-    "if_else_if_else",
-    "if_false",
-    "if_nesting",
-    "if_true",
-    "ignored_parameter",
-    "import_late",
-    "import_nonexistent_library",
-    "import_nonexistent_package",
-    "invalid_char",
-    "match_any_int",
-    "match_int",
-    "match_int_default",
-    "match_placeholder",
-    "next",
-    "no_match",
-    "package_library",
-    "package_missing",
-    "pattern_init",
-    "pattern_variable_fail",
-    "placeholder_variable",
-    "print1",
-    "record1",
-    "return_auto",
-    "return_empty_explicit",
-    "return_empty_explicit_fail",
-    "return_empty_implicit1",
-    "return_empty_implicit2",
-    "return_empty_implicit_fail1",
-    "return_empty_implicit_fail2",
-    "star",
-    "string1",
-    "string2",
-    "string3",
-    "string_fail1",
-    "string_fail2",
-    "string_fail3",
-    "string_fail4",
-    "string_fail5",
-    "string_fail6",
-    "tuple1",
-    "tuple2",
-    "tuple3",
-    "tuple4",
-    "tuple5",
-    "tuple_assign",
-    "tuple_equality",
-    "tuple_equality2",
-    "tuple_equality3",
-    "tuple_match",
-    "tuple_match2",
-    "tuple_match3",
-    "type_compute",
-    "type_compute2",
-    "type_compute3",
-    "type_match",
-    "while1",
-    "zero",
-]

executable_semantics/testdata/assignment_copy1.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 0
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/assignment_copy1.golden

@@ -1 +0,0 @@
-result: 0

executable_semantics/testdata/assignment_copy2.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 0
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/assignment_copy2.golden

@@ -1 +0,0 @@
-result: 0

executable_semantics/testdata/block1.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 0
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/block1.golden

@@ -1 +0,0 @@
-result: 0

executable_semantics/testdata/block2.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 0
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/block2.golden

@@ -1 +0,0 @@
-result: 0

executable_semantics/testdata/break1.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 0
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/break1.golden

@@ -1 +0,0 @@
-result: 0

executable_semantics/testdata/choice1.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 0
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/choice1.golden

@@ -1 +0,0 @@
-result: 0

executable_semantics/testdata/class1.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 0
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/class1.golden

@@ -1 +0,0 @@
-result: 0

executable_semantics/testdata/class2.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 0
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/class2.golden

@@ -1 +0,0 @@
-result: 0

executable_semantics/testdata/class3.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 0
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/class3.golden

@@ -1 +0,0 @@
-result: 0

executable_semantics/testdata/class_field_access_mismatch.carbon

@@ -1,6 +1,14 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: not executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: not executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: COMPILATION ERROR: {{.*}}/class_field_access_mismatch.carbon:21: class Point does not have a field named z
+
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/class_field_access_mismatch.golden

@@ -1,2 +0,0 @@
-COMPILATION ERROR: executable_semantics/testdata/class_field_access_mismatch.carbon:13: class Point does not have a field named z
-EXIT CODE: 255

executable_semantics/testdata/class_field_mismatch.carbon

@@ -1,6 +1,16 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: not executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: not executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: COMPILATION ERROR: {{.*}}/class_field_mismatch.carbon:23: type error in call
+// CHECK: expected: (x = i32, y = i32)
+// CHECK: actual: (x = i32, z = i32)
+
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/class_field_mismatch.golden

@@ -1,4 +0,0 @@
-COMPILATION ERROR: executable_semantics/testdata/class_field_mismatch.carbon:13: type error in call
-expected: (x = i32, y = i32)
-actual: (x = i32, z = i32)
-EXIT CODE: 255

executable_semantics/testdata/class_field_missing.carbon

@@ -1,6 +1,16 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: not executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: not executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: COMPILATION ERROR: {{.*}}/class_field_missing.carbon:23: type error in call
+// CHECK: expected: (x = i32, y = i32)
+// CHECK: actual: (x = i32)
+
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/class_field_missing.golden

@@ -1,4 +0,0 @@
-COMPILATION ERROR: executable_semantics/testdata/class_field_missing.carbon:13: type error in call
-expected: (x = i32, y = i32)
-actual: (x = i32)
-EXIT CODE: 255

executable_semantics/testdata/continue1.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 0
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/continue1.golden

@@ -1 +0,0 @@
-result: 0

executable_semantics/testdata/experimental_continuation1.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 0
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/experimental_continuation1.golden

@@ -1 +0,0 @@
-result: 0

executable_semantics/testdata/experimental_continuation2.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 1
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/experimental_continuation2.golden

@@ -1 +0,0 @@
-result: 1

executable_semantics/testdata/experimental_continuation3.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 3
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/experimental_continuation3.golden

@@ -1 +0,0 @@
-result: 3

executable_semantics/testdata/experimental_continuation4.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 3
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/experimental_continuation4.golden

@@ -1 +0,0 @@
-result: 3

executable_semantics/testdata/experimental_continuation5.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 3
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/experimental_continuation5.golden

@@ -1 +0,0 @@
-result: 3

executable_semantics/testdata/experimental_continuation6.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 10
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/experimental_continuation6.golden

@@ -1 +0,0 @@
-result: 10

executable_semantics/testdata/experimental_continuation7.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 3
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/experimental_continuation7.golden

@@ -1 +0,0 @@
-result: 3

executable_semantics/testdata/experimental_continuation9.carbon

@@ -1,6 +1,14 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: not executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: not executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: RUNTIME ERROR: {{.*}}/experimental_continuation9.carbon:22: undefined behavior: access to dead value 1
+
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/experimental_continuation9.golden

@@ -1,2 +0,0 @@
-RUNTIME ERROR: executable_semantics/testdata/experimental_continuation9.carbon:14: undefined behavior: access to dead value 1
-EXIT CODE: 255

executable_semantics/testdata/fun1.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 0
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/fun1.golden

@@ -1 +0,0 @@
-result: 0

executable_semantics/testdata/fun2.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 0
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/fun2.golden

@@ -1 +0,0 @@
-result: 0

executable_semantics/testdata/fun3.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 0
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/fun3.golden

@@ -1 +0,0 @@
-result: 0

executable_semantics/testdata/fun4.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 0
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/fun4.golden

@@ -1 +0,0 @@
-result: 0

executable_semantics/testdata/fun5.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 0
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/fun5.golden

@@ -1 +0,0 @@
-result: 0

executable_semantics/testdata/fun6_fail_type.carbon

@@ -1,6 +1,16 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: not executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: not executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: COMPILATION ERROR: {{.*}}/fun6_fail_type.carbon:22: type error in call
+// CHECK: expected: (0 = i32, 1 = i32)
+// CHECK: actual: (0 = (0 = i32, 1 = i32))
+
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/fun6_fail_type.golden

@@ -1,4 +0,0 @@
-COMPILATION ERROR: executable_semantics/testdata/fun6_fail_type.carbon:12: type error in call
-expected: (0 = i32, 1 = i32)
-actual: (0 = (0 = i32, 1 = i32))
-EXIT CODE: 255

executable_semantics/testdata/fun_named_params.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 0
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/fun_named_params.golden

@@ -1 +0,0 @@
-result: 0

executable_semantics/testdata/fun_named_params2.carbon

@@ -1,6 +1,14 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: not executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: not executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: PROGRAM ERROR: {{.*}}/fun_named_params2.carbon:15: positional members must come before named members
+
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/fun_named_params2.golden

@@ -1,2 +0,0 @@
-PROGRAM ERROR: executable_semantics/testdata/fun_named_params2.carbon:7: positional members must come before named members
-EXIT CODE: 255

executable_semantics/testdata/fun_recur.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 0
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/fun_recur.golden

@@ -1 +0,0 @@
-result: 0

executable_semantics/testdata/funptr1.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 0
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/funptr1.golden

@@ -1 +0,0 @@
-result: 0

executable_semantics/testdata/generic_function1.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 0
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/generic_function1.golden

@@ -1 +0,0 @@
-result: 0

executable_semantics/testdata/generic_function2.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 0
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/generic_function2.golden

@@ -1 +0,0 @@
-result: 0

executable_semantics/testdata/generic_function3.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 1
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/generic_function3.golden

@@ -1 +0,0 @@
-result: 1

executable_semantics/testdata/generic_function_apply.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: -2
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/generic_function_apply.golden

@@ -1 +0,0 @@
-result: -2

executable_semantics/testdata/generic_function_fail1.carbon

@@ -1,6 +1,16 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: not executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: not executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: COMPILATION ERROR: {{.*}}/generic_function_fail1.carbon:22: type error in argument deduction
+// CHECK: expected: i32
+// CHECK: actual: Bool
+
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/generic_function_fail1.golden

@@ -1,4 +0,0 @@
-COMPILATION ERROR: executable_semantics/testdata/generic_function_fail1.carbon:12: type error in argument deduction
-expected: i32
-actual: Bool
-EXIT CODE: 255

executable_semantics/testdata/generic_function_fail2.carbon

@@ -1,6 +1,14 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: not executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: not executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: COMPILATION ERROR: {{.*}}/generic_function_fail2.carbon:20: could not deduce type argument for type parameter T
+
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/generic_function_fail2.golden

@@ -1,2 +0,0 @@
-COMPILATION ERROR: executable_semantics/testdata/generic_function_fail2.carbon:12: could not deduce type argument for type parameter T
-EXIT CODE: 255

executable_semantics/testdata/generic_function_fail3.carbon

@@ -1,6 +1,16 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: not executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: not executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: COMPILATION ERROR: {{.*}}/generic_function_fail3.carbon:18: type error in addition(1)
+// CHECK: expected: i32
+// CHECK: actual: T
+
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/generic_function_fail3.golden

@@ -1,4 +0,0 @@
-COMPILATION ERROR: executable_semantics/testdata/generic_function_fail3.carbon:8: type error in addition(1)
-expected: i32
-actual: T
-EXIT CODE: 255

executable_semantics/testdata/generic_function_swap.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 0
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/generic_function_swap.golden

@@ -1 +0,0 @@
-result: 0

executable_semantics/testdata/generic_function_tuple_map.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 1
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/generic_function_tuple_map.golden

@@ -1 +0,0 @@
-result: 1

executable_semantics/testdata/global_variable1.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 0
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/global_variable1.golden

@@ -1 +0,0 @@
-result: 0

executable_semantics/testdata/global_variable2.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 0
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/global_variable2.golden

@@ -1 +0,0 @@
-result: 0

executable_semantics/testdata/global_variable3.carbon

@@ -1,6 +1,16 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: not executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: not executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: COMPILATION ERROR: {{.*}}/global_variable3.carbon:19: type error in initializer of variable
+// CHECK: expected: i32
+// CHECK: actual: Bool
+
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/global_variable3.golden

@@ -1,4 +0,0 @@
-COMPILATION ERROR: executable_semantics/testdata/global_variable3.carbon:9: type error in initializer of variable
-expected: i32
-actual: Bool
-EXIT CODE: 255

executable_semantics/testdata/global_variable4.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 0
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/global_variable4.golden

@@ -1 +0,0 @@
-result: 0

executable_semantics/testdata/global_variable5.carbon

@@ -1,6 +1,16 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: not executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: not executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: COMPILATION ERROR: {{.*}}/global_variable5.carbon:22: type error in return
+// CHECK: expected: ()
+// CHECK: actual: i32
+
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/global_variable5.golden

@@ -1,4 +0,0 @@
-COMPILATION ERROR: executable_semantics/testdata/global_variable5.carbon:12: type error in return
-expected: ()
-actual: i32
-EXIT CODE: 255

executable_semantics/testdata/global_variable6.carbon

@@ -1,6 +1,13 @@
 // Part of the Carbon Language project, under the Apache License v2.0 with LLVM
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: executable_semantics %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes=false %s
+// RUN: executable_semantics --trace %s 2>&1 | \
+// RUN:   FileCheck --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: executable_semantics %s
+// CHECK: result: 0
 
 package ExecutableSemanticsTest api;
 

executable_semantics/testdata/global_variable6.golden

@@ -1 +0,0 @@
-result: 0