Merge toolchain file_test children in order to improve linking. (#3206)

Specifically this should improve linking by producing one large binary
instead of one-per-directory. The inclusion of the driver hits the size
issue. Separating out things which have more llvm deps has been
discussed, but I'm not doing that here because I think the semantics
layer will need to depend on clang for interop, and we'd lose a lot of
the benefits that way. Also, having just one place to look seems
simpler.

Includes supporting changes to file_test infrastructure, the most
significant of which is probably passing tests via file instead of a
really large args thing, using a custom rule to do that. That's because
dealing with the layered filegroups that allow the toolchain setup is
more complicated, and this approach scales well.

Combined test time is ~9s, so not sharding right now.

I wasn't sure if people would prefer having the autoupdate script under
testing, so I left it alone for now.

explorer/file_test.cpp

@@ -18,8 +18,8 @@ namespace {
 
 class ExplorerFileTest : public FileTestBase {
  public:
-  explicit ExplorerFileTest(std::filesystem::path path)
-      : FileTestBase(std::move(path)),
+  explicit ExplorerFileTest(llvm::StringRef test_name)
+      : FileTestBase(test_name),
         prelude_line_re_(R"(prelude.carbon:(\d+))"),
         timing_re_(R"((Time elapsed in \w+: )\d+(ms))") {
     CARBON_CHECK(prelude_line_re_.ok()) << prelude_line_re_.error();
@@ -100,7 +100,7 @@ class ExplorerFileTest : public FileTestBase {
  private:
   // Trace output is directly checked for a few tests.
   auto check_trace_output() -> bool {
-    return path().string().find("/trace/") != std::string::npos;
+    return test_name().find("/trace/") != std::string::npos;
   }
 
   TestRawOstream trace_stream_;

testing/file_test/file_test_base.cpp

@@ -6,6 +6,7 @@
 
 #include <filesystem>
 #include <fstream>
+#include <string>
 #include <utility>
 
 #include "absl/flags/flag.h"
@@ -18,7 +19,10 @@
 #include "llvm/Support/MemoryBuffer.h"
 
 ABSL_FLAG(std::vector<std::string>, file_tests, {},
-          "A comma-separated list of tests for file_test infrastructure.");
+          "A comma-separated list of repo-relative names of test files. "
+          "Overrides test_targets_file.");
+ABSL_FLAG(std::string, test_targets_file, "",
+          "A path to a file containing repo-relative names of test files.");
 ABSL_FLAG(bool, autoupdate, false,
           "Instead of verifying files match test output, autoupdate files "
           "based on test output.");
@@ -31,7 +35,7 @@ using ::testing::MatchesRegex;
 using ::testing::StrEq;
 
 // Reads a file to string.
-static auto ReadFile(std::filesystem::path path) -> std::string {
+static auto ReadFile(std::string_view path) -> std::string {
   std::ifstream proto_file(path);
   std::stringstream buffer;
   buffer << proto_file.rdbuf();
@@ -57,13 +61,14 @@ auto FileTestBase::TestBody() -> void {
   CARBON_CHECK(target);
   // This advice overrides the --file_tests flag provided by the file_test rule.
   llvm::errs() << "\nTo test this file alone, run:\n  bazel test " << target
-               << " --test_arg=--file_tests=" << GetTestFilename() << "\n\n";
+               << " --test_arg=--file_tests=" << test_name_ << "\n\n";
 
   TestContext context;
   auto run_result = ProcessTestFileAndRun(context);
   ASSERT_TRUE(run_result.ok()) << run_result.error();
   ValidateRun();
-  EXPECT_THAT(!llvm::StringRef(path().filename()).starts_with("fail_"),
+  auto test_filename = std::filesystem::path(test_name_.str()).filename();
+  EXPECT_THAT(!llvm::StringRef(test_filename).starts_with("fail_"),
               Eq(context.exit_with_success))
       << "Tests should be prefixed with `fail_` if and only if running them "
          "is expected to fail.";
@@ -87,7 +92,7 @@ auto FileTestBase::Autoupdate() -> ErrorOr<bool> {
   TestContext context;
   auto run_result = ProcessTestFileAndRun(context);
   if (!run_result.ok()) {
-    return ErrorBuilder() << "Error updating " << GetTestFilename() << ": "
+    return ErrorBuilder() << "Error updating " << test_name_ << ": "
                           << run_result.error();
   }
   if (!context.autoupdate_line_number) {
@@ -110,8 +115,9 @@ auto FileTestBase::Autoupdate() -> ErrorOr<bool> {
   }
 
   return AutoupdateFileTest(
-      path(), context.input_content, filenames, *context.autoupdate_line_number,
-      context.non_check_lines, context.stdout, context.stderr,
+      std::filesystem::absolute(test_name_.str()), context.input_content,
+      filenames, *context.autoupdate_line_number, context.non_check_lines,
+      context.stdout, context.stderr,
       GetLineNumberReplacement(filenames_for_line_number),
       [&](std::string& line) { DoExtraCheckReplacements(line); });
 }
@@ -127,7 +133,7 @@ auto FileTestBase::GetLineNumberReplacement(
 auto FileTestBase::ProcessTestFileAndRun(TestContext& context)
     -> ErrorOr<Success> {
   // Store the file so that test_files can use references to content.
-  context.input_content = ReadFile(path());
+  context.input_content = ReadFile(test_name_);
 
   // Load expected output.
   CARBON_RETURN_IF_ERROR(ProcessTestFile(context));
@@ -330,8 +336,9 @@ auto FileTestBase::ProcessTestFile(TestContext& context) -> ErrorOr<Success> {
                                  file_content.end() - current_file_start)));
   } else {
     // If no file splitting happened, use the main file as the test file.
-    context.test_files.push_back(
-        TestFile(path().filename().string(), file_content));
+    // There will always be a `/` unless tests are in the repo root.
+    context.test_files.push_back(TestFile(
+        test_name_.drop_front(test_name_.rfind("/") + 1).str(), file_content));
   }
 
   // Assume there is always a suffix `\n` in output.
@@ -431,16 +438,31 @@ auto FileTestBase::TransformExpectation(int line_index, llvm::StringRef in)
   return Matcher<std::string>{MatchesRegex(str)};
 }
 
-auto FileTestBase::GetTestFilename() -> std::string {
-  const char* src_dir = getenv("TEST_SRCDIR");
-  CARBON_CHECK(src_dir);
-  return path().lexically_relative(
-      std::filesystem::path(src_dir).append("carbon"));
-}
+// Returns the tests to run.
+static auto GetTests() -> llvm::SmallVector<std::string> {
+  // Prefer a user-specified list if present.
+  auto specific_tests = absl::GetFlag(FLAGS_file_tests);
+  if (!specific_tests.empty()) {
+    return llvm::SmallVector<std::string>(specific_tests.begin(),
+                                          specific_tests.end());
+  }
 
-}  // namespace Carbon::Testing
+  // Extracts tests from the target file.
+  CARBON_CHECK(!absl::GetFlag(FLAGS_test_targets_file).empty())
+      << "Missing --test_targets_file.";
+  auto content = ReadFile(absl::GetFlag(FLAGS_test_targets_file));
+  llvm::SmallVector<std::string> all_tests;
+  for (llvm::StringRef file_ref : llvm::split(content, "\n")) {
+    if (file_ref.empty()) {
+      continue;
+    }
+    all_tests.push_back(file_ref.str());
+  }
+  return all_tests;
+}
 
-auto main(int argc, char** argv) -> int {
+// Implements main() within the Carbon::Testing namespace for convenience.
+static auto Main(int argc, char** argv) -> int {
   absl::ParseCommandLine(argc, argv);
   testing::InitGoogleTest(&argc, argv);
   llvm::setBugReportMsg(
@@ -454,46 +476,29 @@ auto main(int argc, char** argv) -> int {
     return EXIT_FAILURE;
   }
 
-  // Configure the base directory for test names.
-  const char* target = getenv("TEST_TARGET");
-  CARBON_CHECK(target);
-  llvm::StringRef target_dir = target;
-  std::error_code ec;
-  std::filesystem::path working_dir = std::filesystem::current_path(ec);
-  CARBON_CHECK(!ec) << ec.message();
-  // Leaves one slash.
-  CARBON_CHECK(target_dir.consume_front("/"));
-  target_dir = target_dir.substr(0, target_dir.rfind(":"));
-  std::string base_dir = working_dir.string() + target_dir.str() + "/";
-
-  auto test_factory = Carbon::Testing::GetFileTestFactory();
-
-  for (const auto& file_test : absl::GetFlag(FLAGS_file_tests)) {
-    // Pass the absolute path to the factory function.
-    auto path = std::filesystem::absolute(file_test, ec);
-    CARBON_CHECK(!ec) << file_test << ": " << ec.message();
-    CARBON_CHECK(llvm::StringRef(path.string()).starts_with(base_dir))
-        << "\n  " << path << "\n  should start with\n  " << base_dir;
-    if (absl::GetFlag(FLAGS_autoupdate)) {
-      std::unique_ptr<Carbon::Testing::FileTestBase> test(
-          test_factory.factory_fn(path));
+  llvm::SmallVector<std::string> tests = GetTests();
+  auto test_factory = GetFileTestFactory();
+  if (absl::GetFlag(FLAGS_autoupdate)) {
+    for (const auto& test_name : tests) {
+      std::unique_ptr<FileTestBase> test(test_factory.factory_fn(test_name));
       auto result = test->Autoupdate();
       llvm::errs() << (result.ok() ? (*result ? "!" : ".")
                                    : result.error().message());
-    } else {
-      std::string test_name = path.string().substr(base_dir.size());
-      testing::RegisterTest(test_factory.name, test_name.c_str(), nullptr,
-                            test_name.c_str(), __FILE__, __LINE__,
-                            [=]() { return test_factory.factory_fn(path); });
     }
-  }
-  if (absl::GetFlag(FLAGS_autoupdate)) {
     llvm::errs() << "\nDone!\n";
-  }
-
-  if (absl::GetFlag(FLAGS_autoupdate)) {
     return EXIT_SUCCESS;
   } else {
+    for (llvm::StringRef test_name : tests) {
+      testing::RegisterTest(test_factory.name, test_name.data(), nullptr,
+                            test_name.data(), __FILE__, __LINE__,
+                            [&test_factory, test_name = test_name]() {
+                              return test_factory.factory_fn(test_name);
+                            });
+    }
     return RUN_ALL_TESTS();
   }
 }
+
+}  // namespace Carbon::Testing
+
+auto main(int argc, char** argv) -> int { Carbon::Testing::Main(argc, argv); }

testing/file_test/file_test_base.h

@@ -8,7 +8,6 @@
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
 
-#include <filesystem>
 #include <functional>
 
 #include "common/error.h"
@@ -43,7 +42,7 @@ class FileTestBase : public testing::Test {
   // Provided for child class convenience.
   using LineNumberReplacement = FileTestLineNumberReplacement;
 
-  explicit FileTestBase(std::filesystem::path path) : path_(std::move(path)) {}
+  explicit FileTestBase(llvm::StringRef test_name) : test_name_(test_name) {}
 
   // Implemented by children to run the test. For example, TestBody validates
   // stdout and stderr. Children should use fs for file content, and may add
@@ -82,8 +81,8 @@ class FileTestBase : public testing::Test {
   // Runs the test and autoupdates checks. Returns true if updated.
   auto Autoupdate() -> ErrorOr<bool>;
 
-  // Returns the full path of the file being tested.
-  auto path() -> const std::filesystem::path& { return path_; };
+  // Returns the name of the test (relative to the repo root).
+  auto test_name() const -> llvm::StringRef { return test_name_; }
 
  private:
   // Encapsulates test context generated by processing and running.
@@ -135,15 +134,12 @@ class FileTestBase : public testing::Test {
   // Processes the test input, producing test files and expected output.
   auto ProcessTestFile(TestContext& context) -> ErrorOr<Success>;
 
-  // Gets the test filename, relative to the target directory.
-  auto GetTestFilename() -> std::string;
-
   // Transforms an expectation on a given line from `FileCheck` syntax into a
   // standard regex matcher.
   static auto TransformExpectation(int line_index, llvm::StringRef in)
       -> ErrorOr<testing::Matcher<std::string>>;
 
-  const std::filesystem::path path_;
+  llvm::StringRef test_name_;
 };
 
 // Aggregate a name and factory function for tests using this framework.
@@ -152,7 +148,7 @@ struct FileTestFactory {
   const char* name;
 
   // A factory function for tests.
-  std::function<FileTestBase*(const std::filesystem::path& path)> factory_fn;
+  std::function<FileTestBase*(llvm::StringRef path)> factory_fn;
 };
 
 // Must be implemented by the individual file_test to initialize tests.
@@ -166,10 +162,9 @@ struct FileTestFactory {
 extern auto GetFileTestFactory() -> FileTestFactory;
 
 // Provides a standard GetFileTestFactory implementation.
-#define CARBON_FILE_TEST_FACTORY(Name)                                         \
-  auto GetFileTestFactory()->FileTestFactory {                                 \
-    return {(#Name),                                                           \
-            [](const std::filesystem::path& path) { return new Name(path); }}; \
+#define CARBON_FILE_TEST_FACTORY(Name)                                   \
+  auto GetFileTestFactory()->FileTestFactory {                           \
+    return {#Name, [](llvm::StringRef path) { return new Name(path); }}; \
   }
 
 }  // namespace Carbon::Testing

testing/file_test/file_test_base_test.cpp

@@ -31,7 +31,7 @@ class FileTestBaseTest : public FileTestBase {
     }
     stdout << "\n";
 
-    auto filename = path().filename();
+    auto filename = std::filesystem::path(test_name().str()).filename();
     if (filename == "args.carbon") {
       // 'args.carbon' has custom arguments, so don't do regular argument
       // validation for it.

testing/file_test/rules.bzl

@@ -2,9 +2,41 @@
 # Exceptions. See /LICENSE for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+"""Rules for building file tests.
+
+file_test uses the tests_as_input_file rule to transform test dependencies into
+a file which can be accessed as a list. This avoids long argument parsing.
+"""
+
 load("@rules_cc//cc:defs.bzl", "cc_test")
 
-"""Rules for building fuzz tests."""
+DataFilesInfo = provider(
+    "Data files for this target.",
+    fields = {
+        "data_files": "Data files for this target",
+    },
+)
+
+def _tests_as_input_file_rule_impl(ctx):
+    data_files = []
+    for tests in ctx.attr.data:
+        data_files.extend(
+            [f.path for f in tests[DefaultInfo].data_runfiles.files.to_list()],
+        )
+        data_files.extend(
+            [f.path for f in tests[DefaultInfo].files.to_list()],
+        )
+    ctx.actions.write(ctx.outputs.data_files, "\n".join(data_files) + "\n")
+
+_tests_as_input_file_rule = rule(
+    attrs = {
+        "data": attr.label_list(allow_files = True),
+    },
+    outputs = {
+        "data_files": "%{name}.txt",
+    },
+    implementation = _tests_as_input_file_rule_impl,
+)
 
 def file_test(name, tests, data = [], args = [], **kwargs):
     """Generates tests using the file_test base.
@@ -20,12 +52,17 @@ def file_test(name, tests, data = [], args = [], **kwargs):
       args: Passed to cc_test.
       **kwargs: Passed to cc_test.
     """
+
+    # Ensure tests are always a filegroup for tests_as_input_file_rule.
+    tests_file = "{0}.tests".format(name)
+    _tests_as_input_file_rule(
+        name = tests_file,
+        data = tests,
+        testonly = 1,
+    )
     cc_test(
         name = name,
-        data = tests + data,
-        args = ["--file_tests=" + ",".join([
-            "$(location {0})".format(x)
-            for x in tests
-        ])] + args,
+        data = [tests_file] + tests + data,
+        args = ["--test_targets_file=$(rootpath :{0})".format(tests_file)] + args,
         **kwargs
     )

toolchain/autoupdate_testdata.py

@@ -8,49 +8,38 @@ Exceptions. See /LICENSE for license information.
 SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 """
 
-import argparse
 import subprocess
-
-TARGETS = {
-    "check": "//toolchain/check:check_file_test",
-    "codegen": "//toolchain/codegen:codegen_file_test",
-    "driver": "//toolchain/driver:driver_file_test",
-    "lex": "//toolchain/lex:lex_file_test",
-    "lower": "//toolchain/lower:lower_file_test",
-    "parse": "//toolchain/parse:parse_file_test",
-}
+import sys
+from pathlib import Path
 
 
 def main() -> None:
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "dirs",
-        # We don't use `choices` because it seems to conflict with "*".
-        nargs="*",
-        default=TARGETS.keys(),
-        help="Optionally restrict directories to update. Defaults to all.",
-    )
-    parsed_args = parser.parse_args()
-
-    # Deduplicate and validate arguments.
-    dirs = set(parsed_args.dirs)
-    invalid_dirs = dirs.difference(TARGETS.keys())
-    if invalid_dirs:
-        exit(
-            f"Invalid dirs: {', '.join(invalid_dirs)}; "
-            f"allowed dirs are {', '.join(TARGETS.keys())}."
-        )
-
-    # Build the targets together if there's more than one. Otherwise, we may as
-    # well build and run together.
-    if len(dirs) > 1:
-        subprocess.check_call(
-            ["bazel", "build", "-c", "opt"] + [TARGETS[d] for d in dirs]
-        )
-    for d in dirs:
-        subprocess.check_call(
-            ["bazel", "run", "-c", "opt", TARGETS[d], "--", "--autoupdate"]
-        )
+    argv = [
+        "bazel",
+        "run",
+        "-c",
+        "opt",
+        "//toolchain/testing:file_test",
+        "--",
+        "--autoupdate",
+    ]
+    # Support specifying tests to update, such as:
+    # ./autoupdate_testdata.py lex/**/*
+    if len(sys.argv) > 1:
+        repo_root = Path(__file__).parent.parent
+        file_tests = []
+        # Filter down to just test files.
+        for f in sys.argv[1:]:
+            if f.endswith(".carbon"):
+                path = str(Path(f).absolute().relative_to(repo_root))
+                if path.find("/testdata/"):
+                    file_tests.append(path)
+        if not file_tests:
+            sys.exit(
+                f"Args do not seem to be test files; for example, {sys.argv[1]}"
+            )
+        argv.append("--file_tests=" + ",".join(file_tests))
+    subprocess.run(argv, check=True)
 
 
 if __name__ == "__main__":

toolchain/check/BUILD

@@ -4,11 +4,15 @@
 
 load("@rules_cc//cc:defs.bzl", "cc_library")
 load("//bazel/sh_run:rules.bzl", "glob_sh_run")
-load("//testing/file_test:rules.bzl", "file_test")
 load("//testing/fuzzing:rules.bzl", "cc_fuzz_test")
 
 package(default_visibility = ["//visibility:public"])
 
+filegroup(
+    name = "testdata",
+    data = glob(["testdata/**/*.carbon"]),
+)
+
 cc_library(
     name = "node_stack",
     srcs = ["node_stack.cpp"],
@@ -62,16 +66,6 @@ cc_library(
     ],
 )
 
-file_test(
-    name = "check_file_test",
-    srcs = ["check_file_test.cpp"],
-    tests = glob(["testdata/**/*.carbon"]),
-    deps = [
-        "//toolchain/driver:driver_file_test_base",
-        "@llvm-project//llvm:Support",
-    ],
-)
-
 cc_fuzz_test(
     name = "check_fuzzer",
     size = "small",

toolchain/check/check_file_test.cpp

@@ -1,26 +0,0 @@
-// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
-// Exceptions. See /LICENSE for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-#include <string>
-
-#include "llvm/ADT/SmallVector.h"
-#include "toolchain/driver/driver_file_test_base.h"
-
-namespace Carbon::Testing {
-namespace {
-
-class CheckFileTest : public DriverFileTestBase {
- public:
-  using DriverFileTestBase::DriverFileTestBase;
-
-  auto GetDefaultArgs() -> llvm::SmallVector<std::string> override {
-    return {"compile", "--phase=check", "--dump-semantics-ir", "%s"};
-  }
-};
-
-}  // namespace
-
-CARBON_FILE_TEST_FACTORY(CheckFileTest);
-
-}  // namespace Carbon::Testing

toolchain/codegen/BUILD

@@ -3,10 +3,14 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 load("@rules_cc//cc:defs.bzl", "cc_library")
-load("//testing/file_test:rules.bzl", "file_test")
 
 package(default_visibility = ["//visibility:public"])
 
+filegroup(
+    name = "testdata",
+    data = glob(["testdata/**/*.carbon"]),
+)
+
 cc_library(
     name = "codegen",
     srcs = ["codegen.cpp"],
@@ -21,13 +25,3 @@ cc_library(
         "@llvm-project//llvm:TargetParser",
     ],
 )
-
-file_test(
-    name = "codegen_file_test",
-    srcs = ["codegen_file_test.cpp"],
-    tests = glob(["testdata/**/*.carbon"]),
-    deps = [
-        "//toolchain/driver:driver_file_test_base",
-        "@llvm-project//llvm:Support",
-    ],
-)

toolchain/codegen/codegen_file_test.cpp

@@ -1,26 +0,0 @@
-// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
-// Exceptions. See /LICENSE for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-#include <string>
-
-#include "llvm/ADT/SmallVector.h"
-#include "toolchain/driver/driver_file_test_base.h"
-
-namespace Carbon::Testing {
-namespace {
-
-class CodeGenFileTest : public DriverFileTestBase {
- public:
-  using DriverFileTestBase::DriverFileTestBase;
-
-  auto GetDefaultArgs() -> llvm::SmallVector<std::string> override {
-    CARBON_FATAL() << "ARGS is always set in these tests";
-  }
-};
-
-}  // namespace
-
-CARBON_FILE_TEST_FACTORY(CodeGenFileTest);
-
-}  // namespace Carbon::Testing

toolchain/driver/BUILD

@@ -4,11 +4,15 @@
 
 load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test")
 load("//bazel/cc_toolchains:defs.bzl", "cc_env")
-load("//testing/file_test:rules.bzl", "file_test")
 load("//testing/fuzzing:rules.bzl", "cc_fuzz_test")
 
 package(default_visibility = ["//visibility:public"])
 
+filegroup(
+    name = "testdata",
+    data = glob(["testdata/**/*.carbon"]),
+)
+
 cc_library(
     name = "driver",
     srcs = ["driver.cpp"],
@@ -33,16 +37,6 @@ cc_library(
     ],
 )
 
-file_test(
-    name = "driver_file_test",
-    srcs = ["driver_file_test.cpp"],
-    tests = glob(["testdata/**/*.carbon"]),
-    deps = [
-        ":driver_file_test_base",
-        "@llvm-project//llvm:Support",
-    ],
-)
-
 cc_test(
     name = "driver_test",
     size = "small",
@@ -72,17 +66,6 @@ cc_fuzz_test(
     ],
 )
 
-cc_library(
-    name = "driver_file_test_base",
-    testonly = 1,
-    hdrs = ["driver_file_test_base.h"],
-    deps = [
-        ":driver",
-        "//testing/file_test:file_test_base",
-        "@llvm-project//llvm:Support",
-    ],
-)
-
 cc_binary(
     name = "carbon",
     srcs = ["driver_main.cpp"],

toolchain/driver/driver_file_test.cpp

@@ -1,33 +0,0 @@
-// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
-// Exceptions. See /LICENSE for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-#include <string>
-
-#include "llvm/ADT/SmallVector.h"
-#include "toolchain/driver/driver_file_test_base.h"
-
-namespace Carbon::Testing {
-namespace {
-
-class DriverFileTest : public DriverFileTestBase {
- public:
-  using DriverFileTestBase::DriverFileTestBase;
-
-  auto GetDefaultArgs() -> llvm::SmallVector<std::string> override {
-    CARBON_FATAL() << "ARGS is always set in these tests";
-  }
-
-  auto DoExtraCheckReplacements(std::string& check_line) -> void override {
-    // TODO: Disable token output, it's not interesting for these tests.
-    if (llvm::StringRef(check_line).starts_with("// CHECK:STDOUT: {")) {
-      check_line = "// CHECK:STDOUT: {{.*}}";
-    }
-  }
-};
-
-}  // namespace
-
-CARBON_FILE_TEST_FACTORY(DriverFileTest);
-
-}  // namespace Carbon::Testing

toolchain/driver/driver_file_test_base.h

@@ -1,34 +0,0 @@
-// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
-// Exceptions. See /LICENSE for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-#ifndef CARBON_TOOLCHAIN_DRIVER_DRIVER_FILE_TEST_BASE_H_
-#define CARBON_TOOLCHAIN_DRIVER_DRIVER_FILE_TEST_BASE_H_
-
-#include <cstdio>
-
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/Support/VirtualFileSystem.h"
-#include "testing/file_test/file_test_base.h"
-#include "toolchain/driver/driver.h"
-
-namespace Carbon::Testing {
-
-// Provides common test support for the driver. This is used by file tests in
-// phase subdirectories.
-class DriverFileTestBase : public FileTestBase {
- public:
-  using FileTestBase::FileTestBase;
-
-  auto Run(const llvm::SmallVector<llvm::StringRef>& test_args,
-           llvm::vfs::InMemoryFileSystem& fs, llvm::raw_pwrite_stream& stdout,
-           llvm::raw_pwrite_stream& stderr) -> ErrorOr<bool> override {
-    Driver driver(fs, stdout, stderr);
-    return driver.RunCommand(test_args);
-  }
-};
-
-}  // namespace Carbon::Testing
-
-#endif  // CARBON_TOOLCHAIN_DRIVER_DRIVER_FILE_TEST_BASE_H_

toolchain/lex/BUILD

@@ -4,11 +4,15 @@
 
 load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test")
 load("//bazel/sh_run:rules.bzl", "glob_sh_run")
-load("//testing/file_test:rules.bzl", "file_test")
 load("//testing/fuzzing:rules.bzl", "cc_fuzz_test")
 
 package(default_visibility = ["//visibility:public"])
 
+filegroup(
+    name = "testdata",
+    data = glob(["testdata/**/*.carbon"]),
+)
+
 cc_library(
     name = "token_kind",
     srcs = ["token_kind.cpp"],
@@ -249,17 +253,6 @@ cc_binary(
     ],
 )
 
-file_test(
-    name = "lex_file_test",
-    srcs = ["lex_file_test.cpp"],
-    tests = glob(["testdata/**/*.carbon"]),
-    deps = [
-        "//toolchain/driver:driver_file_test_base",
-        "@com_googlesource_code_re2//:re2",
-        "@llvm-project//llvm:Support",
-    ],
-)
-
 glob_sh_run(
     args = [
         "$(location //toolchain/driver:carbon)",

toolchain/lex/lex_file_test.cpp

@@ -1,46 +0,0 @@
-// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
-// Exceptions. See /LICENSE for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-#include <string>
-
-#include "llvm/ADT/SmallVector.h"
-#include "re2/re2.h"
-#include "toolchain/driver/driver_file_test_base.h"
-
-namespace Carbon::Testing {
-namespace {
-
-class LexerFileTest : public DriverFileTestBase {
- public:
-  explicit LexerFileTest(std::filesystem::path path)
-      : DriverFileTestBase(std::move(path)),
-        end_of_file_re_((R"((EndOfFile.*column: )( *\d+))")) {}
-
-  auto GetDefaultArgs() -> llvm::SmallVector<std::string> override {
-    return {"compile", "--phase=lex", "--dump-tokens", "%s"};
-  }
-
-  auto GetLineNumberReplacement(llvm::ArrayRef<llvm::StringRef> /*filenames*/)
-      -> LineNumberReplacement override {
-    return {.has_file = false,
-            .pattern = R"(line: (\s*\d+))",
-            // The `{{{{` becomes `{{`.
-            .line_formatv = "{{{{ *}}{0}"};
-  }
-
-  auto DoExtraCheckReplacements(std::string& check_line) -> void override {
-    // Ignore the resulting column of EndOfFile because it's often the end of
-    // the CHECK comment.
-    RE2::Replace(&check_line, end_of_file_re_, R"(\1{{ *\\d+}})");
-  }
-
- private:
-  RE2 end_of_file_re_;
-};
-
-}  // namespace
-
-CARBON_FILE_TEST_FACTORY(LexerFileTest);
-
-}  // namespace Carbon::Testing

toolchain/lower/BUILD

@@ -4,10 +4,14 @@
 
 load("@rules_cc//cc:defs.bzl", "cc_library")
 load("//bazel/sh_run:rules.bzl", "glob_sh_run")
-load("//testing/file_test:rules.bzl", "file_test")
 
 package(default_visibility = ["//visibility:public"])
 
+filegroup(
+    name = "testdata",
+    data = glob(["testdata/**/*.carbon"]),
+)
+
 cc_library(
     name = "lower",
     srcs = ["lower.cpp"],
@@ -45,16 +49,6 @@ cc_library(
     ],
 )
 
-file_test(
-    name = "lower_file_test",
-    srcs = ["lower_file_test.cpp"],
-    tests = glob(["testdata/**/*.carbon"]),
-    deps = [
-        "//toolchain/driver:driver_file_test_base",
-        "@llvm-project//llvm:Support",
-    ],
-)
-
 glob_sh_run(
     args = [
         "$(location //toolchain/driver:carbon)",

toolchain/lower/lower_file_test.cpp

@@ -1,26 +0,0 @@
-// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
-// Exceptions. See /LICENSE for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-#include <string>
-
-#include "llvm/ADT/SmallVector.h"
-#include "toolchain/driver/driver_file_test_base.h"
-
-namespace Carbon::Testing {
-namespace {
-
-class LowerFileTest : public DriverFileTestBase {
- public:
-  using DriverFileTestBase::DriverFileTestBase;
-
-  auto GetDefaultArgs() -> llvm::SmallVector<std::string> override {
-    return {"compile", "--phase=lower", "--dump-llvm-ir", "%s"};
-  }
-};
-
-}  // namespace
-
-CARBON_FILE_TEST_FACTORY(LowerFileTest);
-
-}  // namespace Carbon::Testing

toolchain/parse/BUILD

@@ -4,11 +4,15 @@
 
 load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test")
 load("//bazel/sh_run:rules.bzl", "glob_sh_run")
-load("//testing/file_test:rules.bzl", "file_test")
 load("//testing/fuzzing:rules.bzl", "cc_fuzz_test")
 
 package(default_visibility = ["//visibility:public"])
 
+filegroup(
+    name = "testdata",
+    data = glob(["testdata/**/*.carbon"]),
+)
+
 cc_library(
     name = "node_kind",
     srcs = ["node_kind.cpp"],
@@ -119,16 +123,6 @@ cc_test(
     ],
 )
 
-file_test(
-    name = "parse_file_test",
-    srcs = ["parse_file_test.cpp"],
-    tests = glob(["testdata/**/*.carbon"]),
-    deps = [
-        "//toolchain/driver:driver_file_test_base",
-        "@llvm-project//llvm:Support",
-    ],
-)
-
 glob_sh_run(
     args = [
         "$(location //toolchain/driver:carbon)",

toolchain/parse/parse_file_test.cpp

@@ -1,26 +0,0 @@
-// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
-// Exceptions. See /LICENSE for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-#include <string>
-
-#include "llvm/ADT/SmallVector.h"
-#include "toolchain/driver/driver_file_test_base.h"
-
-namespace Carbon::Testing {
-namespace {
-
-class ParseFileTest : public DriverFileTestBase {
- public:
-  using DriverFileTestBase::DriverFileTestBase;
-
-  auto GetDefaultArgs() -> llvm::SmallVector<std::string> override {
-    return {"compile", "--phase=parse", "--dump-parse-tree", "%s"};
-  }
-};
-
-}  // namespace
-
-CARBON_FILE_TEST_FACTORY(ParseFileTest);
-
-}  // namespace Carbon::Testing

toolchain/testing/BUILD

@@ -2,4 +2,24 @@
 # Exceptions. See /LICENSE for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+load("//testing/file_test:rules.bzl", "file_test")
+
 package(default_visibility = ["//visibility:public"])
+
+file_test(
+    name = "file_test",
+    srcs = ["file_test.cpp"],
+    tests = [
+        "//toolchain/check:testdata",
+        "//toolchain/codegen:testdata",
+        "//toolchain/driver:testdata",
+        "//toolchain/lex:testdata",
+        "//toolchain/lower:testdata",
+        "//toolchain/parse:testdata",
+    ],
+    deps = [
+        "//testing/file_test:file_test_base",
+        "//toolchain/driver",
+        "@llvm-project//llvm:Support",
+    ],
+)

toolchain/testing/file_test.cpp

@@ -0,0 +1,95 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef CARBON_TOOLCHAIN_DRIVER_DRIVER_FILE_TEST_BASE_H_
+#define CARBON_TOOLCHAIN_DRIVER_DRIVER_FILE_TEST_BASE_H_
+
+#include <string>
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/VirtualFileSystem.h"
+#include "testing/file_test/file_test_base.h"
+#include "toolchain/driver/driver.h"
+
+namespace Carbon::Testing {
+namespace {
+
+// Provides common test support for the driver. This is used by file tests in
+// phase subdirectories.
+class ToolchainFileTest : public FileTestBase {
+ public:
+  explicit ToolchainFileTest(llvm::StringRef test_name)
+      : FileTestBase(test_name), component_(GetComponent(test_name)) {}
+
+  auto Run(const llvm::SmallVector<llvm::StringRef>& test_args,
+           llvm::vfs::InMemoryFileSystem& fs, llvm::raw_pwrite_stream& stdout,
+           llvm::raw_pwrite_stream& stderr) -> ErrorOr<bool> override {
+    Driver driver(fs, stdout, stderr);
+    return driver.RunCommand(test_args);
+  }
+
+  auto GetDefaultArgs() -> llvm::SmallVector<std::string> override {
+    if (component_ == "check") {
+      return {"compile", "--phase=check", "--dump-semantics-ir", "%s"};
+    } else if (component_ == "lex") {
+      return {"compile", "--phase=lex", "--dump-tokens", "%s"};
+    } else if (component_ == "lower") {
+      return {"compile", "--phase=lower", "--dump-llvm-ir", "%s"};
+    } else if (component_ == "parse") {
+      return {"compile", "--phase=parse", "--dump-parse-tree", "%s"};
+    } else if (component_ == "codegen" || component_ == "driver") {
+      CARBON_FATAL() << "ARGS is always set in these tests";
+    } else {
+      CARBON_FATAL() << "Unexpected test component " << component_ << ": "
+                     << test_name();
+    }
+  }
+
+  auto GetLineNumberReplacement(llvm::ArrayRef<llvm::StringRef> filenames)
+      -> LineNumberReplacement override {
+    if (component_ == "lex") {
+      return {.has_file = false,
+              .pattern = R"(line: (\s*\d+))",
+              // The `{{{{` becomes `{{`.
+              .line_formatv = "{{{{ *}}{0}"};
+    } else {
+      return FileTestBase::GetLineNumberReplacement(filenames);
+    }
+  }
+
+  auto DoExtraCheckReplacements(std::string& check_line) -> void override {
+    if (component_ == "driver") {
+      // TODO: Disable token output, it's not interesting for these tests.
+      if (llvm::StringRef(check_line).starts_with("// CHECK:STDOUT: {")) {
+        check_line = "// CHECK:STDOUT: {{.*}}";
+      }
+    } else if (component_ == "lex") {
+      // Ignore the resulting column of EndOfFile because it's often the end of
+      // the CHECK comment.
+      static RE2 end_of_file_re(R"((EndOfFile.*column: )( *\d+))");
+      RE2::Replace(&check_line, end_of_file_re, R"(\1{{ *\\d+}})");
+    } else {
+      return FileTestBase::DoExtraCheckReplacements(check_line);
+    }
+  }
+
+ private:
+  // Returns the toolchain subdirectory being tested.
+  static auto GetComponent(llvm::StringRef test_name) -> llvm::StringRef {
+    CARBON_CHECK(test_name.consume_front("toolchain/"));
+    test_name = test_name.take_front(test_name.find("/"));
+    return test_name;
+  }
+
+  const llvm::StringRef component_;
+};
+
+}  // namespace
+
+CARBON_FILE_TEST_FACTORY(ToolchainFileTest);
+
+}  // namespace Carbon::Testing
+
+#endif  // CARBON_TOOLCHAIN_DRIVER_DRIVER_FILE_TEST_BASE_H_