Refactor lit testing to generate per-file tests. (#1063)

This switches to generating per-file targets such as `//executable_semantics/testdata:tuple/equality_false.carbon.test`, instead of one test covering all files. Essentially this relies on bazel instead of lit to handle test parallelization.

bazel/testing/lit.bzl

@@ -0,0 +1,35 @@
+# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+# Exceptions. See /LICENSE for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+"""Rule for a lit test."""
+
+def glob_lit_tests(driver, data, test_file_exts, **kwargs):
+    """Runs `lit` on test_dir.
+
+    `lit` reference:
+      https://llvm.org/docs/CommandGuide/lit.html
+
+    Args:
+      driver: The path to the lit config.
+      data: A list of tools to provide to the tests. These will be aliased for
+        execution.
+      test_file_exts: A list of extensions to use for tests.
+      **kwargs: Any additional parameters for the generated py_test.
+    """
+    test_files = native.glob(
+        ["**"],
+        exclude_directories = 1,
+    )
+    data.append("@llvm-project//llvm:lit")
+    for f in test_files:
+        if f.split(".")[-1] not in test_file_exts:
+            continue
+        native.py_test(
+            name = "%s.test" % f,
+            srcs = ["//bazel/testing:lit_test.py"],
+            main = "//bazel/testing:lit_test.py",
+            data = data + [driver, f],
+            args = ["--package_name=%s" % native.package_name(), "--"],
+            **kwargs
+        )

bazel/testing/lit_test.bzl

@@ -1,36 +0,0 @@
-# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
-# Exceptions. See /LICENSE for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-"""Rule for a lit test."""
-
-def lit_test(name, test_dir, data = None, **kwargs):
-    """Runs `lit` on test_dir.
-
-    `lit` reference:
-      https://llvm.org/docs/CommandGuide/lit.html
-
-    To pass flags to `lit`, use `--test_arg`. For example:
-      bazel test :lit_test --test_arg=-v
-      bazel test :lit_test --test_arg=--filter=REGEXP
-
-    Args:
-      name: Name of the build rule.
-      test_dir: The directory with the lit tests.
-      data: An optional list of tools to provide to the tests. These will be
-        aliased for execution.
-      **kwargs: Any additional parameters for the generated py_test.
-    """
-    if not data:
-        data = []
-    data += [
-        "@llvm-project//llvm:lit",
-    ]
-    native.py_test(
-        name = name,
-        srcs = ["//bazel/testing:lit_test.py"],
-        main = "//bazel/testing:lit_test.py",
-        data = data + native.glob([test_dir + "/**"]),
-        args = ["--package_name=%s" % native.package_name(), "--test_dir=%s" % test_dir, "--"],
-        **kwargs
-    )

bazel/testing/lit_test.py

@@ -18,9 +18,6 @@ def _parse_args():
     arg_parser.add_argument(
         "--package_name", help="The directory containing tests to run."
     )
-    arg_parser.add_argument(
-        "--test_dir", help="The directory containing tests to run."
-    )
     arg_parser.add_argument(
         "lit_args", nargs="*", help="Arguments to pass through to lit."
     )
@@ -32,10 +29,8 @@ def main():
 
     args = [
         str(Path(os.environ["TEST_SRCDIR"]).joinpath("llvm-project/llvm/lit")),
-        str(
-            Path.cwd().joinpath(parsed_args.package_name, parsed_args.test_dir)
-        ),
-        "-sv",
+        str(Path.cwd().joinpath(parsed_args.package_name)),
+        "-v",
     ]
 
     # Force tests to be explicit about command paths.

executable_semantics/BUILD

@@ -2,7 +2,6 @@
 # Exceptions. See /LICENSE for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-load("//bazel/testing:lit_test.bzl", "lit_test")
 load("@mypy_integration//:mypy.bzl", "mypy_test")
 
 package(default_visibility = [
@@ -23,16 +22,6 @@ cc_binary(
     ],
 )
 
-lit_test(
-    name = "executable_semantics_lit_test",
-    data = [
-        ":executable_semantics",
-        "@llvm-project//llvm:FileCheck",
-        "@llvm-project//llvm:not",
-    ],
-    test_dir = "testdata",
-)
-
 py_binary(
     name = "gen_rtti",
     srcs = ["gen_rtti.py"],

executable_semantics/README.md

@@ -111,11 +111,7 @@ To explain this boilerplate:
 ### Useful commands
 
 -   `./update_checks.py` -- Updates expected output.
--   `bazel test :executable_semantics_lit_test --test_output=errors` -- Runs
-    tests and prints any errors.
--   `bazel test :executable_semantics_lit_test --test_output=errors --test_arg=--filter=basic_syntax/.*`
-    -- Only runs tests in the `basic_syntax` directory; `--filter` is a regular
-    expression.
+-   `bazel test ... --test_output=errors` -- Runs tests and prints any errors.
 
 ## Experimental feature: Delimited Continuations
 

executable_semantics/testdata/BUILD

@@ -0,0 +1,15 @@
+# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+# Exceptions. See /LICENSE for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+load("//bazel/testing:lit.bzl", "glob_lit_tests")
+
+glob_lit_tests(
+    data = [
+        "//executable_semantics",
+        "@llvm-project//llvm:FileCheck",
+        "@llvm-project//llvm:not",
+    ],
+    driver = "lit.cfg.py",
+    test_file_exts = ["carbon"],
+)

executable_semantics/update_checks.py

@@ -29,7 +29,7 @@ def _get_tests() -> Set[str]:
     tests = set()
     for root, _, files in os.walk(_TESTDATA):
         for f in files:
-            if f == "lit.cfg.py":
+            if f in {"lit.cfg.py", "BUILD"}:
                 # Ignore the lit config.
                 continue
             if os.path.splitext(f)[1] == ".carbon":

toolchain/driver/BUILD

@@ -3,7 +3,6 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 load("//bazel/fuzzing:rules.bzl", "cc_fuzz_test")
-load("//bazel/testing:lit_test.bzl", "lit_test")
 
 package(default_visibility = ["//visibility:public"])
 
@@ -53,13 +52,3 @@ cc_binary(
         "@llvm-project//llvm:Support",
     ],
 )
-
-lit_test(
-    name = "carbon_lit_test",
-    data = [
-        ":carbon",
-        "@llvm-project//llvm:FileCheck",
-        "@llvm-project//llvm:not",
-    ],
-    test_dir = "testdata",
-)

toolchain/driver/testdata/BUILD

@@ -0,0 +1,15 @@
+# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+# Exceptions. See /LICENSE for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+load("//bazel/testing:lit.bzl", "glob_lit_tests")
+
+glob_lit_tests(
+    data = [
+        "//toolchain/driver:carbon",
+        "@llvm-project//llvm:FileCheck",
+        "@llvm-project//llvm:not",
+    ],
+    driver = "lit.cfg.py",
+    test_file_exts = ["carbon"],
+)