Introduce a runtimes caching and management layer (#6002)

This layer allows runtimes to be built on-demand but cached in a
consistent and re-usable location on the system. It handles careful
filesystem operations to ensure consistency even in the face of multiple
versions and build configurations.

This addresses a number of TODOs from the initial runtimes building
on-demand, and sets the stage to scale up to more runtimes.

This doesn't switch on-demand runtimes to be on by default, I wanted to
wait and make that change as a separate step.

---------

Co-authored-by: Geoff Romer <gromer@google.com>

common/filesystem.cpp

@@ -62,7 +62,10 @@ auto PathError::Print(llvm::raw_ostream& out) const -> void {
   // The `format_` member is a `StringLiteral` that is null terminated, so
   // `.data()` is safe here.
   // NOLINTNEXTLINE(bugprone-suspicious-stringview-data-usage)
-  out << llvm::formatv(format_.data(), path_, dir_fd_) << " failed: ";
+  out << llvm::formatv(format_.data(), path_,
+                       dir_fd_ == AT_FDCWD ? std::string("AT_FDCWD")
+                                           : std::to_string(dir_fd_))
+      << " failed: ";
   PrintErrorNumber(out, unix_errnum());
 }
 

toolchain/diagnostics/diagnostic_kind.def

@@ -22,6 +22,8 @@
 // ============================================================================
 
 CARBON_DIAGNOSTIC_KIND(DriverInstallInvalid)
+CARBON_DIAGNOSTIC_KIND(DriverRuntimesCacheInvalid)
+CARBON_DIAGNOSTIC_KIND(DriverPrebuiltRuntimesInvalid)
 CARBON_DIAGNOSTIC_KIND(DriverCommandLineParseFailed)
 CARBON_DIAGNOSTIC_KIND(CompilePhaseFlagConflict)
 CARBON_DIAGNOSTIC_KIND(CompilePreludeManifestError)

toolchain/driver/BUILD

@@ -23,6 +23,7 @@ cc_library(
     hdrs = ["clang_runner.h"],
     deps = [
         ":llvm_runner",
+        ":runtimes_cache",
         ":tool_runner_base",
         "//common:error",
         "//common:filesystem",
@@ -130,6 +131,7 @@ cc_library(
         ":clang_runner",
         ":lld_runner",
         ":llvm_runner",
+        ":runtimes_cache",
         "//common:command_line",
         "//common:error",
         "//common:ostream",
@@ -270,6 +272,44 @@ cc_test(
     ],
 )
 
+cc_library(
+    name = "runtimes_cache",
+    srcs = ["runtimes_cache.cpp"],
+    hdrs = ["runtimes_cache.h"],
+    deps = [
+        "//common:check",
+        "//common:error",
+        "//common:filesystem",
+        "//common:ostream",
+        "//common:version",
+        "//common:vlog",
+        "//toolchain/install:install_paths",
+        "@llvm-project//llvm:Support",
+    ],
+)
+
+cc_test(
+    name = "runtimes_cache_test",
+    size = "small",
+    srcs = ["runtimes_cache_test.cpp"],
+    data = ["//toolchain/install:install_data"],
+    deps = [
+        ":runtimes_cache",
+        "//common:check",
+        "//common:error_test_helpers",
+        "//common:filesystem",
+        "//common:ostream",
+        "//common:raw_string_ostream",
+        "//common:version",
+        "//testing/base:capture_std_streams",
+        "//testing/base:file_helpers",
+        "//testing/base:global_exe_path",
+        "//testing/base:gtest_main",
+        "@googletest//:gtest",
+        "@llvm-project//llvm:Support",
+    ],
+)
+
 cc_library(
     name = "tool_runner_base",
     srcs = ["tool_runner_base.cpp"],

toolchain/driver/build_runtimes_subcommand.cpp

@@ -4,6 +4,8 @@
 
 #include "toolchain/driver/build_runtimes_subcommand.h"
 
+#include <variant>
+
 #include "llvm/TargetParser/Triple.h"
 #include "toolchain/driver/clang_runner.h"
 
@@ -43,9 +45,6 @@ BuildRuntimesSubcommand::BuildRuntimesSubcommand()
     : DriverSubcommand(SubcommandInfo) {}
 
 auto BuildRuntimesSubcommand::Run(DriverEnv& driver_env) -> DriverResult {
-  ClangRunner runner(driver_env.installation, driver_env.fs,
-                     driver_env.vlog_stream);
-
   // Don't run Clang when fuzzing, it is known to not be reliable under fuzzing
   // due to many unfixed issues.
   if (TestAndDiagnoseIfFuzzingExternalLibraries(driver_env, "clang")) {
@@ -56,39 +55,44 @@ auto BuildRuntimesSubcommand::Run(DriverEnv& driver_env) -> DriverResult {
   CARBON_DIAGNOSTIC(FailureBuildingRuntimes, Error,
                     "failure building runtimes: {0}", std::string);
 
-  auto tmp_result = Filesystem::MakeTmpDir();
-  if (!tmp_result.ok()) {
+  auto run_result = RunInternal(driver_env);
+  if (!run_result.ok()) {
     driver_env.emitter.Emit(FailureBuildingRuntimes,
-                            tmp_result.error().message());
+                            run_result.error().message());
     return {.success = false};
   }
-  Filesystem::RemovingDir tmp_dir = *std::move(tmp_result);
-
-  // TODO: Currently, the default location is just a subdirectory of the
-  // temporary directory used for the build. This allows the subcommand to be
-  // used to test and debug runtime building, but not for the results to be
-  // reused. Eventually, this should be connected to the same runtimes cache
-  // used by link commands.
-  std::filesystem::path output_path =
-      options_.directory.empty()
-          ? tmp_dir.abs_path() / "runtimes"
-          : std::filesystem::path(options_.directory.str());
-
-  // Hard code a subdirectory of the runtimes output for the Clang resource
-  // directory runtimes.
-  //
-  // TODO: This should be replaced with an abstraction that manages the layout
-  // of the generated runtimes rather than hardcoding it.
-  std::filesystem::path resource_dir_path = output_path / "clang_resource_dir";
-
-  auto build_result = runner.BuildTargetResourceDir(
-      options_.codegen_options.target, resource_dir_path, tmp_dir.abs_path());
-  if (!build_result.ok()) {
-    driver_env.emitter.Emit(FailureBuildingRuntimes,
-                            build_result.error().message());
+
+  llvm::outs() << "Built runtimes: " << *run_result << "\n";
+  return {.success = true};
+}
+
+auto BuildRuntimesSubcommand::RunInternal(DriverEnv& driver_env)
+    -> ErrorOr<std::filesystem::path> {
+  ClangRunner runner(driver_env.installation, &driver_env.runtimes_cache,
+                     driver_env.fs, driver_env.vlog_stream);
+
+  Runtimes::Cache::Features features = {
+      .target = options_.codegen_options.target.str()};
+
+  bool is_cache = options_.directory.empty();
+  std::filesystem::path explicit_output_path = options_.directory.str();
+  if (!is_cache) {
+    auto access_result = Filesystem::Cwd().Access(explicit_output_path);
+    if (access_result.ok()) {
+      return Error("output directory already exists");
+    }
+    if (!access_result.error().no_entity()) {
+      return std::move(access_result).error();
+    }
   }
 
-  return {.success = build_result.ok()};
+  CARBON_ASSIGN_OR_RETURN(
+      auto runtimes,
+      is_cache ? driver_env.runtimes_cache.Lookup(features)
+               : Runtimes::Make(explicit_output_path, driver_env.vlog_stream));
+  CARBON_ASSIGN_OR_RETURN(auto tmp_dir, Filesystem::MakeTmpDir());
+
+  return runner.BuildTargetResourceDir(features, runtimes, tmp_dir.abs_path());
 }
 
 }  // namespace Carbon

toolchain/driver/build_runtimes_subcommand.h

@@ -36,6 +36,8 @@ class BuildRuntimesSubcommand : public DriverSubcommand {
   auto Run(DriverEnv& driver_env) -> DriverResult override;
 
  private:
+  auto RunInternal(DriverEnv& driver_env) -> ErrorOr<std::filesystem::path>;
+
   BuildRuntimesOptions options_;
 };
 

toolchain/driver/clang_runner.cpp

@@ -14,6 +14,7 @@
 #include <string>
 #include <system_error>
 #include <utility>
+#include <variant>
 
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/DiagnosticOptions.h"
@@ -51,10 +52,12 @@ auto clang_main(int Argc, char** Argv, const llvm::ToolContext& ToolContext)
 namespace Carbon {
 
 ClangRunner::ClangRunner(const InstallPaths* install_paths,
+                         Runtimes::Cache* runtimes_cache,
                          llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fs,
                          llvm::raw_ostream* vlog_stream,
                          bool build_runtimes_on_demand)
     : ToolRunnerBase(install_paths, vlog_stream),
+      runtimes_cache_(runtimes_cache),
       fs_(std::move(fs)),
       diagnostic_ids_(new clang::DiagnosticIDs()),
       build_runtimes_on_demand_(build_runtimes_on_demand) {}
@@ -130,47 +133,44 @@ static auto IsNonLinkCommand(llvm::ArrayRef<llvm::StringRef> args) -> bool {
   });
 }
 
-auto ClangRunner::Run(
-    llvm::ArrayRef<llvm::StringRef> args,
-    std::optional<std::filesystem::path> prebuilt_resource_dir_path)
-    -> ErrorOr<bool> {
+auto ClangRunner::Run(llvm::ArrayRef<llvm::StringRef> args,
+                      Runtimes* prebuilt_runtimes) -> ErrorOr<bool> {
   // Check the args to see if we have a known target-independent command. If so,
   // directly dispatch it to avoid the cost of building the target resource
   // directory.
   // TODO: Maybe handle response file expansion similar to the Clang CLI?
   if (args.empty() || args[0].starts_with("-cc1") || IsNonLinkCommand(args) ||
-      (!build_runtimes_on_demand_ && !prebuilt_resource_dir_path)) {
+      (!build_runtimes_on_demand_ && !prebuilt_runtimes)) {
     return RunTargetIndependentCommand(args);
   }
 
   std::string target = ComputeClangTarget(args);
 
   // If we have pre-built runtimes, use them rather than building on demand.
-  if (prebuilt_resource_dir_path) {
-    return RunInternal(args, target, prebuilt_resource_dir_path->native());
+  if (prebuilt_runtimes) {
+    CARBON_ASSIGN_OR_RETURN(std::filesystem::path prebuilt_resource_dir_path,
+                            prebuilt_runtimes->Get(Runtimes::ClangResourceDir));
+    return RunInternal(args, target, prebuilt_resource_dir_path.native());
   }
   CARBON_CHECK(build_runtimes_on_demand_);
 
   // Otherwise, we need to build a target resource directory.
-  //
-  // TODO: Currently, this builds the runtimes in a temporary directory that is
-  // removed after the Clang invocation. That requires building them on each
-  // execution which is expensive and slow. Eventually, we want to replace this
-  // with using an on-disk cache so that only the first execution has to build
-  // the runtimes and subsequently the cached build can be used.
   CARBON_VLOG("Building target resource dir...\n");
-  CARBON_ASSIGN_OR_RETURN(Filesystem::RemovingDir tmp_dir,
-                          Filesystem::MakeTmpDir());
-
-  // Hard code the subdirectory for the resource-dir runtimes.
-  //
-  // TODO: This should be replaced with an abstraction that manages the layout
-  // of a built runtimes tree.
-  std::filesystem::path resource_dir_path =
-      tmp_dir.abs_path() / "clang_resource_dir";
-
-  CARBON_RETURN_IF_ERROR(
-      BuildTargetResourceDir(target, resource_dir_path, tmp_dir.abs_path()));
+  Runtimes::Cache::Features features = {.target = target};
+  CARBON_ASSIGN_OR_RETURN(Runtimes runtimes, runtimes_cache_->Lookup(features));
+
+  // We need to build the Clang resource directory for these runtimes. This
+  // requires a temporary directory as well as the destination directory for
+  // the build. The temporary directory should only be used during the build,
+  // not once we are running Clang with the built runtime.
+  std::filesystem::path resource_dir_path;
+  {
+    CARBON_ASSIGN_OR_RETURN(Filesystem::RemovingDir tmp_dir,
+                            Filesystem::MakeTmpDir());
+    CARBON_ASSIGN_OR_RETURN(
+        resource_dir_path,
+        BuildTargetResourceDir(features, runtimes, tmp_dir.abs_path()));
+  }
 
   // Note that this function always successfully runs `clang` and returns a bool
   // to indicate whether `clang` itself succeeded, not whether the runner was
@@ -186,32 +186,37 @@ auto ClangRunner::RunTargetIndependentCommand(
 }
 
 auto ClangRunner::BuildTargetResourceDir(
-    llvm::StringRef target, const std::filesystem::path& resource_dir_path,
-    const std::filesystem::path& tmp_path) -> ErrorOr<Success> {
+    const Runtimes::Cache::Features& features, Runtimes& runtimes,
+    const std::filesystem::path& tmp_path) -> ErrorOr<std::filesystem::path> {
   // Disable any leaking of memory while building the target resource dir, and
   // restore the previous setting at the end.
   auto restore_leak_flag = llvm::make_scope_exit(
       [&, orig_flag = enable_leaking_] { enable_leaking_ = orig_flag; });
   enable_leaking_ = false;
 
-  // Create the destination directory if needed.
-  CARBON_ASSIGN_OR_RETURN(
-      Filesystem::Dir resource_dir,
-      Filesystem::Cwd().CreateDirectories(resource_dir_path));
+  CARBON_ASSIGN_OR_RETURN(auto build_dir,
+                          runtimes.Build(Runtimes::ClangResourceDir));
+  if (std::holds_alternative<std::filesystem::path>(build_dir)) {
+    // Found cached build.
+    return std::get<std::filesystem::path>(std::move(build_dir));
+  }
+
+  auto builder = std::get<Runtimes::Builder>(std::move(build_dir));
+  std::string target = features.target;
 
   // Symlink the installation's `include` and `share` directories.
   std::filesystem::path install_resource_path =
       installation_->clang_resource_path();
   CARBON_RETURN_IF_ERROR(
-      resource_dir.Symlink("include", install_resource_path / "include"));
+      builder.dir().Symlink("include", install_resource_path / "include"));
   CARBON_RETURN_IF_ERROR(
-      resource_dir.Symlink("share", install_resource_path / "share"));
+      builder.dir().Symlink("share", install_resource_path / "share"));
 
   // Create the target's `lib` directory.
   std::filesystem::path lib_path =
       std::filesystem::path("lib") / std::string_view(target);
   CARBON_ASSIGN_OR_RETURN(Filesystem::Dir lib_dir,
-                          resource_dir.CreateDirectories(lib_path));
+                          builder.dir().CreateDirectories(lib_path));
 
   llvm::Triple target_triple(target);
   if (target_triple.isOSWindows()) {
@@ -222,15 +227,15 @@ auto ClangRunner::BuildTargetResourceDir(
   // provide the CRT begin/end files, and so we need to build them.
   if (target_triple.isOSLinux()) {
     BuildCrtFile(target, RuntimeSources::CrtBegin,
-                 resource_dir_path / lib_path / "clang_rt.crtbegin.o");
+                 builder.path() / lib_path / "clang_rt.crtbegin.o");
     BuildCrtFile(target, RuntimeSources::CrtEnd,
-                 resource_dir_path / lib_path / "clang_rt.crtend.o");
+                 builder.path() / lib_path / "clang_rt.crtend.o");
   }
 
   CARBON_RETURN_IF_ERROR(
       BuildBuiltinsLib(target, target_triple, tmp_path, lib_dir));
 
-  return Success();
+  return std::move(builder).Commit();
 }
 
 auto ClangRunner::RunInternal(
@@ -508,11 +513,13 @@ auto ClangRunner::BuildBuiltinsLib(llvm::StringRef target,
     objs.push_back(std::move(*obj));
   }
 
-  // Now build an archive out of the `.o` files for the builtins.
+  // Now build an archive out of the `.o` files for the builtins. Note that we
+  // build this directly into the `lib_dir` as this is expected to be a staging
+  // directory and cleaned up on errors.
   std::filesystem::path builtins_a_path = "libclang_rt.builtins.a";
   CARBON_ASSIGN_OR_RETURN(
       Filesystem::WriteFile builtins_a_file,
-      tmp_dir.OpenWriteOnly(builtins_a_path, Filesystem::CreateAlways));
+      lib_dir.OpenWriteOnly(builtins_a_path, Filesystem::CreateAlways));
   {
     llvm::raw_fd_ostream builtins_a_os = builtins_a_file.WriteStream();
 
@@ -528,10 +535,6 @@ auto ClangRunner::BuildBuiltinsLib(llvm::StringRef target,
   }
   CARBON_RETURN_IF_ERROR(std::move(builtins_a_file).Close());
 
-  // Move it into the lib directory.
-  CARBON_RETURN_IF_ERROR(
-      tmp_dir.Rename(builtins_a_path, lib_dir, builtins_a_path));
-
   return Success();
 }
 

toolchain/driver/clang_runner.h

@@ -14,6 +14,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include "llvm/TargetParser/Triple.h"
+#include "toolchain/driver/runtimes_cache.h"
 #include "toolchain/driver/tool_runner_base.h"
 #include "toolchain/install/install_paths.h"
 
@@ -48,6 +49,7 @@ class ClangRunner : ToolRunnerBase {
   // If `verbose` is passed as true, will enable verbose logging to the
   // `err_stream` both from the runner and Clang itself.
   ClangRunner(const InstallPaths* install_paths,
+              Runtimes::Cache* on_demand_runtimes_cache,
               llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fs,
               llvm::raw_ostream* vlog_stream = nullptr,
               bool build_runtimes_on_demand = false);
@@ -66,8 +68,7 @@ class ClangRunner : ToolRunnerBase {
   // TODO: Eventually, this will need to accept an abstraction that can
   // represent multiple different pre-built runtimes.
   auto Run(llvm::ArrayRef<llvm::StringRef> args,
-           std::optional<std::filesystem::path> prebuilt_resource_dir_path = {})
-      -> ErrorOr<bool>;
+           Runtimes* prebuilt_runtimes = nullptr) -> ErrorOr<bool>;
 
   // Run Clang with the provided arguments and without any target-dependent
   // resources.
@@ -85,10 +86,10 @@ class ClangRunner : ToolRunnerBase {
   // contains all the target independent files such as headers. However, for
   // target-specific files like runtimes, we build those on demand here and
   // return the path.
-  auto BuildTargetResourceDir(llvm::StringRef target,
-                              const std::filesystem::path& resource_dir_path,
+  auto BuildTargetResourceDir(const Runtimes::Cache::Features& features,
+                              Runtimes& runtimes,
                               const std::filesystem::path& tmp_path)
-      -> ErrorOr<Success>;
+      -> ErrorOr<std::filesystem::path>;
 
   // Enable leaking memory.
   //
@@ -126,6 +127,8 @@ class ClangRunner : ToolRunnerBase {
                         const std::filesystem::path& tmp_path,
                         Filesystem::DirRef lib_dir) -> ErrorOr<Success>;
 
+  Runtimes::Cache* runtimes_cache_;
+
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fs_;
   llvm::IntrusiveRefCntPtr<clang::DiagnosticIDs> diagnostic_ids_;
 

toolchain/driver/clang_runner_test.cpp

@@ -62,13 +62,15 @@ class ClangRunnerTest : public ::testing::Test {
  public:
   InstallPaths install_paths_ =
       InstallPaths::MakeForBazelRunfiles(Testing::GetExePath());
+  Runtimes::Cache runtimes_cache_ =
+      *Runtimes::Cache::MakeSystem(install_paths_);
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> vfs_ =
       llvm::vfs::getRealFileSystem();
 };
 
 TEST_F(ClangRunnerTest, Version) {
   RawStringOstream test_os;
-  ClangRunner runner(&install_paths_, vfs_, &test_os);
+  ClangRunner runner(&install_paths_, &runtimes_cache_, vfs_, &test_os);
 
   std::string out;
   std::string err;
@@ -99,7 +101,7 @@ TEST_F(ClangRunnerTest, DashC) {
   std::filesystem::path test_output = *Testing::WriteTestFile("test.o", "");
 
   RawStringOstream verbose_out;
-  ClangRunner runner(&install_paths_, vfs_, &verbose_out);
+  ClangRunner runner(&install_paths_, &runtimes_cache_, vfs_, &verbose_out);
   std::string out;
   std::string err;
   EXPECT_TRUE(Testing::CallWithCapturedOutput(
@@ -128,7 +130,7 @@ TEST_F(ClangRunnerTest, BuitinHeaders) {
   std::filesystem::path test_output = *Testing::WriteTestFile("test.o", "");
 
   RawStringOstream verbose_out;
-  ClangRunner runner(&install_paths_, vfs_, &verbose_out);
+  ClangRunner runner(&install_paths_, &runtimes_cache_, vfs_, &verbose_out);
   std::string out;
   std::string err;
   EXPECT_TRUE(Testing::CallWithCapturedOutput(
@@ -155,7 +157,7 @@ TEST_F(ClangRunnerTest, CompileMultipleFiles) {
     std::filesystem::path output = *Testing::WriteTestFile(output_file, "");
 
     RawStringOstream verbose_out;
-    ClangRunner runner(&install_paths_, vfs_, &verbose_out);
+    ClangRunner runner(&install_paths_, &runtimes_cache_, vfs_, &verbose_out);
     std::string out;
     std::string err;
     EXPECT_TRUE(Testing::CallWithCapturedOutput(
@@ -178,7 +180,7 @@ TEST_F(ClangRunnerTest, CompileMultipleFiles) {
 }
 
 TEST_F(ClangRunnerTest, BuildResourceDir) {
-  ClangRunner runner(&install_paths_, vfs_, &llvm::errs(),
+  ClangRunner runner(&install_paths_, &runtimes_cache_, vfs_, &llvm::errs(),
                      /*build_runtimes_on_demand=*/true);
 
   // Note that we can't test arbitrary targets here as we need to be able to
@@ -186,12 +188,13 @@ TEST_F(ClangRunnerTest, BuildResourceDir) {
   // the most likely to pass.
   std::string target = llvm::sys::getDefaultTargetTriple();
   llvm::Triple target_triple(target);
+  Runtimes::Cache::Features features = {.target = target};
+  auto runtimes = *runtimes_cache_.Lookup(features);
   auto tmp_dir = *Filesystem::MakeTmpDir();
-  std::filesystem::path resource_dir_path = tmp_dir.abs_path() / "clang";
-
-  auto build_result = runner.BuildTargetResourceDir(target, resource_dir_path,
-                                                    tmp_dir.abs_path());
+  auto build_result =
+      runner.BuildTargetResourceDir(features, runtimes, tmp_dir.abs_path());
   ASSERT_TRUE(build_result.ok()) << build_result.error();
+  std::filesystem::path resource_dir_path = std::move(*build_result);
 
   // For Linux we can directly check the CRT begin/end object files.
   if (target_triple.isOSLinux()) {
@@ -267,7 +270,7 @@ TEST_F(ClangRunnerTest, LinkCommandEcho) {
   std::filesystem::path bar_file = *Testing::WriteTestFile("bar.o", "");
 
   RawStringOstream verbose_out;
-  ClangRunner runner(&install_paths_, vfs_, &verbose_out);
+  ClangRunner runner(&install_paths_, &runtimes_cache_, vfs_, &verbose_out);
   std::string out;
   std::string err;
   EXPECT_TRUE(Testing::CallWithCapturedOutput(

toolchain/driver/clang_subcommand.cpp

@@ -12,18 +12,6 @@
 namespace Carbon {
 
 auto ClangOptions::Build(CommandLine::CommandBuilder& b) -> void {
-  b.AddStringOption(
-      {
-          .name = "prebuilt-runtimes",
-          .value_name = "PATH",
-          .help = R"""(
-Path to prebuilt target runtimes for Clang.
-
-If this option is provided, runtimes will not be built on demand and this path
-will be used instead.
-)""",
-      },
-      [&](auto& arg_b) { arg_b.Set(&prebuilt_runtimes_path); });
   b.AddFlag(
       {
           .name = "build-runtimes",
@@ -82,7 +70,8 @@ ClangSubcommand::ClangSubcommand() : DriverSubcommand(SubcommandInfo) {}
 // https://github.com/llvm/llvm-project/blob/main/clang/tools/driver/driver.cpp
 auto ClangSubcommand::Run(DriverEnv& driver_env) -> DriverResult {
   ClangRunner runner(
-      driver_env.installation, driver_env.fs, driver_env.vlog_stream,
+      driver_env.installation, &driver_env.runtimes_cache, driver_env.fs,
+      driver_env.vlog_stream,
       /*build_runtimes_on_demand=*/options_.build_runtimes_on_demand);
 
   // Don't run Clang when fuzzing, it is known to not be reliable under fuzzing
@@ -96,16 +85,9 @@ auto ClangSubcommand::Run(DriverEnv& driver_env) -> DriverResult {
     runner.EnableLeakingMemory();
   }
 
-  std::optional<std::filesystem::path> prebuilt_resource_dir_path;
-  if (!options_.prebuilt_runtimes_path.empty()) {
-    prebuilt_resource_dir_path = options_.prebuilt_runtimes_path.str();
-    // TODO: Replace the hard coded `clang_resource_dir` subdirectory here with
-    // an abstraction that manages the layout of the built runtimes.
-    *prebuilt_resource_dir_path /= "clang_resource_dir";
-  }
-
-  ErrorOr<bool> run_result =
-      runner.Run(options_.args, prebuilt_resource_dir_path);
+  ErrorOr<bool> run_result = runner.Run(
+      options_.args,
+      driver_env.prebuilt_runtimes ? &*driver_env.prebuilt_runtimes : nullptr);
   if (!run_result.ok()) {
     // This is not a Clang failure, but a failure to even run Clang, so we need
     // to diagnose it here.

toolchain/driver/clang_subcommand.h

@@ -19,7 +19,6 @@ namespace Carbon {
 struct ClangOptions {
   auto Build(CommandLine::CommandBuilder& b) -> void;
 
-  llvm::StringRef prebuilt_runtimes_path;
   bool build_runtimes_on_demand = false;
 
   llvm::SmallVector<llvm::StringRef> args;

toolchain/driver/driver.cpp

@@ -5,6 +5,7 @@
 #include "toolchain/driver/driver.h"
 
 #include <algorithm>
+#include <filesystem>
 #include <memory>
 #include <optional>
 
@@ -32,6 +33,9 @@ struct Options {
   bool fuzzing = false;
   bool include_diagnostic_kind = false;
 
+  llvm::StringRef runtimes_cache_path;
+  llvm::StringRef prebuilt_runtimes_path;
+
   BuildRuntimesSubcommand runtimes;
   ClangSubcommand clang;
   CompileSubcommand compile;
@@ -74,6 +78,35 @@ auto Options::Build(CommandLine::CommandBuilder& b) -> void {
       },
       [&](CommandLine::FlagBuilder& arg_b) { arg_b.Set(&verbose); });
 
+  b.AddStringOption(
+      {
+          .name = "runtimes-cache",
+          .value_name = "PATH",
+          .help = R"""(
+Specify a custom runtimes cache location.
+
+By default, the runtimes cache is located in the `carbon_runtimes` subdirectory
+of `$XDG_CACHE_HOME` (or `$HOME/.cache` if not set). If unable to use either, it
+will be placed in a temporary directory that is removed when the command
+completes. This flag overrides that logic with a specific path. It has no effect
+if --prebuilt-runtimes is set.
+)""",
+      },
+      [&](auto& arg_b) { arg_b.Set(&runtimes_cache_path); });
+
+  b.AddStringOption(
+      {
+          .name = "prebuilt-runtimes",
+          .value_name = "PATH",
+          .help = R"""(
+Path to prebuilt runtimes tree.
+
+If this option is provided, runtimes will not be built on demand and this path
+will be used instead.
+)""",
+      },
+      [&](auto& arg_b) { arg_b.Set(&prebuilt_runtimes_path); });
+
   b.AddFlag(
       {
           .name = "fuzzing",
@@ -135,6 +168,38 @@ auto Driver::RunCommand(llvm::ArrayRef<llvm::StringRef> args) -> DriverResult {
     return {.success = true};
   }
 
+  auto cache_result =
+      options.runtimes_cache_path.empty()
+          ? Runtimes::Cache::MakeSystem(*driver_env_.installation,
+                                        driver_env_.vlog_stream)
+          : Runtimes::Cache::MakeCustom(
+                *driver_env_.installation,
+                std::filesystem::absolute(options.runtimes_cache_path.str()),
+                driver_env_.vlog_stream);
+  if (!cache_result.ok()) {
+    // TODO: We should provide a better diagnostic than the raw error.
+    CARBON_DIAGNOSTIC(DriverRuntimesCacheInvalid, Error, "{0}", std::string);
+    driver_env_.emitter.Emit(DriverRuntimesCacheInvalid,
+                             cache_result.error().message());
+    return {.success = false};
+  }
+  driver_env_.runtimes_cache = std::move(*cache_result);
+
+  if (!options.prebuilt_runtimes_path.empty()) {
+    auto result = Runtimes::Make(
+        std::filesystem::absolute(options.prebuilt_runtimes_path.str()),
+        driver_env_.vlog_stream);
+    if (!result.ok()) {
+      // TODO: We should provide a better diagnostic than the raw error.
+      CARBON_DIAGNOSTIC(DriverPrebuiltRuntimesInvalid, Error, "{0}",
+                        std::string);
+      driver_env_.emitter.Emit(DriverPrebuiltRuntimesInvalid,
+                               result.error().message());
+      return {.success = false};
+    }
+    driver_env_.prebuilt_runtimes = *std::move(result);
+  }
+
   if (options.verbose) {
     // Note this implies streamed output in order to interleave.
     driver_env_.vlog_stream = driver_env_.error_stream;

toolchain/driver/driver_env.h

@@ -11,6 +11,7 @@
 #include "common/ostream.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include "toolchain/diagnostics/diagnostic_emitter.h"
+#include "toolchain/driver/runtimes_cache.h"
 #include "toolchain/install/install_paths.h"
 
 namespace Carbon {
@@ -63,6 +64,12 @@ struct DriverEnv {
 
   // For CARBON_VLOG.
   llvm::raw_pwrite_stream* vlog_stream = nullptr;
+
+  // Cached runtimes.
+  Runtimes::Cache runtimes_cache;
+
+  // Prebuilt runtimes.
+  std::optional<Runtimes> prebuilt_runtimes;
 };
 
 }  // namespace Carbon

toolchain/driver/link_subcommand.cpp

@@ -118,8 +118,8 @@ auto LinkSubcommand::Run(DriverEnv& driver_env) -> DriverResult {
   clang_args.append(options_.object_filenames.begin(),
                     options_.object_filenames.end());
 
-  ClangRunner runner(driver_env.installation, driver_env.fs,
-                     driver_env.vlog_stream);
+  ClangRunner runner(driver_env.installation, &driver_env.runtimes_cache,
+                     driver_env.fs, driver_env.vlog_stream);
   ErrorOr<bool> run_result = runner.Run(clang_args);
   if (!run_result.ok()) {
     // This is not a Clang failure, but a failure to even run Clang, so we need

toolchain/driver/lld_runner_test.cpp

@@ -85,7 +85,8 @@ static auto CompileTwoSources(const InstallPaths& install_paths,
   // First compile the two source files to `.o` files with Clang.
   RawStringOstream verbose_out;
   auto vfs = llvm::vfs::getRealFileSystem();
-  ClangRunner clang(&install_paths, vfs, &verbose_out);
+  ClangRunner clang(&install_paths, /*on_demand_runtimes_cache=*/nullptr, vfs,
+                    &verbose_out);
   std::string target_arg = llvm::formatv("--target={0}", target).str();
   std::string out;
   std::string err;

toolchain/driver/runtimes_cache.cpp

@@ -0,0 +1,514 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "toolchain/driver/runtimes_cache.h"
+
+#include <algorithm>
+#include <chrono>
+#include <filesystem>
+#include <memory>
+#include <numeric>
+#include <optional>
+#include <string>
+#include <system_error>
+#include <utility>
+#include <variant>
+
+#include "common/filesystem.h"
+#include "common/version.h"
+#include "common/vlog.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/ScopeExit.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/FormatAdapters.h"
+#include "llvm/Support/Program.h"
+#include "llvm/Support/SHA256.h"
+
+namespace Carbon {
+
+auto Runtimes::Make(std::filesystem::path path, llvm::raw_ostream* vlog_stream)
+    -> ErrorOr<Runtimes> {
+  if (!path.is_absolute()) {
+    return Error("Runtimes require an absolute path");
+  }
+
+  CARBON_ASSIGN_OR_RETURN(
+      Filesystem::Dir dir,
+      Filesystem::Cwd().OpenDir(path, Filesystem::OpenExisting));
+  return Runtimes(std::move(path), std::move(dir), {}, {}, vlog_stream);
+}
+
+auto Runtimes::Destroy() -> void {
+  // Release the lock on the runtimes and close the lock file.
+  flock_ = {};
+  auto close_result = std::move(lock_file_).Close();
+  if (!close_result.ok()) {
+    // Log and continue on close errors.
+    CARBON_VLOG("Error closing lock file for runtimes '{0}': {1}", base_path_,
+                close_result.error());
+  }
+}
+
+auto Runtimes::Get(Component component) -> ErrorOr<std::filesystem::path> {
+  std::filesystem::path path = base_path_ / ComponentPath(component);
+  auto open_result =
+      base_dir_.OpenDir(ComponentPath(component), Filesystem::OpenExisting);
+  if (open_result.ok()) {
+    return path;
+  }
+  return open_result.error().ToError();
+}
+
+auto Runtimes::Build(Component component)
+    -> ErrorOr<std::variant<std::filesystem::path, Builder>> {
+  return BuildImpl(component, BuildLockDeadline, BuildLockPollInterval);
+}
+
+auto Runtimes::BuildImpl(Component component, Filesystem::Duration deadline,
+                         Filesystem::Duration poll_interval)
+    -> ErrorOr<std::variant<std::filesystem::path, Builder>> {
+  // Try to get an existing resource directory first.
+  auto existing_result = Get(component);
+  if (existing_result.ok()) {
+    return {*std::move(existing_result)};
+  }
+
+  // Otherwise, we will need to build the runtimes and commit them into this
+  // directory once ready. Try and acquire an advisory lock to avoid redundant
+  // computation.
+  std::string_view component_path = ComponentPath(component);
+  CARBON_ASSIGN_OR_RETURN(
+      Filesystem::ReadWriteFile lock_file,
+      base_dir_.OpenReadWrite(
+          llvm::formatv(LockFileFormat, component_path).str(),
+          Filesystem::OpenAlways, /*creation_mode=*/0700));
+  CARBON_VLOG("PID {0} locking cache path: {1}\n", getpid(),
+              base_path_ / component_path);
+  Filesystem::FileLock flock;
+  auto flock_result = lock_file.TryLock(Filesystem::FileLock::Exclusive,
+                                        deadline, poll_interval);
+  if (flock_result.ok()) {
+    flock = *std::move(flock_result);
+    CARBON_VLOG("Successfully locked cache path\n");
+    // As a debugging aid, write our PID into the lock file when we
+    // successfully acquire it. Ignore errors here though.
+    (void)lock_file.WriteFileFromString(std::to_string(getpid()));
+  } else if (!flock_result.error().would_block()) {
+    // Some unexpected filesystem error, report that rather than trying to
+    // continue.
+    return std::move(flock_result).error();
+  } else {
+    CARBON_VLOG("Unable to lock cache path, held by: {1}\n",
+                *lock_file.ReadFileToString());
+    (void)std::move(lock_file).Close();
+  }
+
+  // See if another process has built the runtimes while we waited on the lock.
+  // We do this even if we didn't successfully acquire the lock because we
+  // ensure that a successful build atomically creates a viable directory.
+  existing_result = Get(component);
+  if (existing_result.ok()) {
+    // Clear and close the lock file.
+    (void)lock_file.WriteFileFromString("");
+    flock = {};
+    (void)std::move(lock_file).Close();
+    return {*std::move(existing_result)};
+  }
+
+  // Whether we hold the lock file or not, we're going to now build these
+  // runtimes. Create a temporary directory where we can do that safely
+  // regardless of what else is happening.
+  std::filesystem::path tmp_path =
+      base_path_ / llvm::formatv(".{0}.tmp", component_path).str();
+  CARBON_ASSIGN_OR_RETURN(Filesystem::RemovingDir tmp_dir,
+                          Filesystem::MakeTmpDirWithPrefix(tmp_path));
+  return {Builder(*this, std::move(lock_file), std::move(flock),
+                  std::move(tmp_dir), component_path)};
+}
+
+auto Runtimes::Cache::FindXdgCachePath()
+    -> std::optional<std::filesystem::path> {
+  if (const char* xdg_cache_home = getenv("XDG_CACHE_HOME");
+      xdg_cache_home != nullptr) {
+    std::filesystem::path path = xdg_cache_home;
+    if (path.is_absolute()) {
+      CARBON_VLOG("Using '$XDG_CACHE_HOME' cache: {0}", path);
+      return path;
+    }
+  }
+
+  // Unable to use the standard environment variable. Try the designated
+  // fallback of `$HOME/.cache`.
+  const char* home = getenv("HOME");
+  if (home == nullptr) {
+    return std::nullopt;
+  }
+
+  std::filesystem::path path = home;
+  if (!path.is_absolute()) {
+    return std::nullopt;
+  }
+  path /= ".cache";
+  CARBON_VLOG("Using '$HOME/.cache' cache: {0}", path);
+  return path;
+}
+
+auto Runtimes::Cache::InitTmpSystemCache() -> ErrorOr<Success> {
+  CARBON_ASSIGN_OR_RETURN(dir_owner_, Filesystem::MakeTmpDir());
+  path_ = std::get<Filesystem::RemovingDir>(dir_owner_).abs_path();
+  dir_ = std::get<Filesystem::RemovingDir>(dir_owner_);
+  CARBON_VLOG("Using temporary cache: {0}", path_);
+  return Success();
+}
+
+auto Runtimes::Cache::InitSystemCache(const InstallPaths& install)
+    -> ErrorOr<Success> {
+  constexpr llvm::StringLiteral CachePath = "carbon_runtimes";
+
+  // If we have a digest to use as the cache key, save it and we can try to
+  // use persistent caches.
+  auto read_digest_result =
+      Filesystem::Cwd().ReadFileToString(install.digest_path());
+  if (!read_digest_result.ok()) {
+    return InitTmpSystemCache();
+  }
+  cache_key_ = *std::move(read_digest_result);
+
+  auto xdg_path_result = FindXdgCachePath();
+  if (!xdg_path_result) {
+    return InitTmpSystemCache();
+  }
+
+  // We have a candidate XDG-based cache path. Try to open that, and a
+  // directory below it for Carbon's runtimes. Note that we don't error on a
+  // missing directory, we fall through to using a temporary directory.
+  auto open_result = Filesystem::Cwd().OpenDir(*xdg_path_result);
+  if (!open_result.ok()) {
+    if (!open_result.error().no_entity()) {
+      // Some other unexpected error in the filesystem, propagate that.
+      return std::move(open_result).error();
+    }
+    // Otherwise we fall back to a temporary system cache.
+    return InitTmpSystemCache();
+  }
+
+  path_ = *std::move(xdg_path_result);
+  // Now open a subdirectory of the cache for Carbon's usage. This will
+  // create a subdirectory if one doesn't yet exist.
+  path_ /= std::string_view(CachePath);
+  CARBON_ASSIGN_OR_RETURN(
+      dir_owner_, open_result->OpenDir(CachePath.str(), Filesystem::OpenAlways,
+                                       /*creation_mode=*/0700));
+  dir_ = std::get<Filesystem::Dir>(dir_owner_);
+
+  // Ensure the directory has narrow permissions so runtimes can't be
+  // overwritten.
+  CARBON_ASSIGN_OR_RETURN(auto dir_stat, dir_.Stat());
+  if (dir_stat.permissions() != 0700 || dir_stat.unix_uid() != geteuid()) {
+    return Error(llvm::formatv(
+        "Found runtimes cache path '{0}' with excessive permissions ({1}) "
+        "or an invalid owning UID ({2})",
+        path_, dir_stat.permissions(), dir_stat.unix_uid()));
+  }
+
+  return Success();
+}
+
+auto Runtimes::Cache::InitCachePath(const InstallPaths& install,
+                                    std::filesystem::path cache_path)
+    -> ErrorOr<Success> {
+  auto read_digest_result =
+      Filesystem::Cwd().ReadFileToString(install.digest_path());
+  if (read_digest_result.ok()) {
+    // If we have a digest to use as the cache key, save it and we can try to
+    // use persistent caches.
+    cache_key_ = *std::move(read_digest_result);
+  } else {
+    // Without a digest, use the path itself as the key.
+    cache_key_ = cache_path.string();
+  }
+
+  CARBON_ASSIGN_OR_RETURN(dir_owner_, Filesystem::Cwd().OpenDir(cache_path));
+  dir_ = std::get<Filesystem::Dir>(dir_owner_);
+  path_ = std::move(cache_path);
+  CARBON_VLOG("Using custom cache: {0}", path_);
+  return Success();
+}
+
+auto Runtimes::Cache::Lookup(const Features& features) -> ErrorOr<Runtimes> {
+  // Compute the hash of the features. We'll use this to build the subdirectory
+  // within the cache.
+
+  llvm::SHA256 entry_hasher;
+  // First incorporate our cache key that comes from the installation's digest.
+  // This ensures we don't share a cache entry with any other Carbon
+  // installations using different inputs.
+  entry_hasher.update(cache_key_);
+  // Then incorporate the specific features that are enabled in this entry.
+  entry_hasher.update(features.target);
+
+  std::array<uint8_t, 32> entry_digest = entry_hasher.final();
+  std::filesystem::path entry_path =
+      llvm::formatv("runtimes-{0}-{1}", Version::String,
+                    llvm::toHex(entry_digest, /*LowerCase=*/true))
+          .str();
+
+  Filesystem::Dir entry_dir;
+  auto open_result = dir_.OpenDir(entry_path, Filesystem::OpenExisting);
+  if (open_result.ok()) {
+    entry_dir = *std::move(open_result);
+  } else {
+    if (!open_result.error().no_entity()) {
+      return std::move(open_result).error();
+    }
+
+    // We're going to potentially create a new set of runtimes, prune the
+    // existing runtimes first to provide a bound on the total size of runtimes.
+    PruneStaleRuntimes(entry_path);
+
+    // Now we can create or open, we don't care if a racing process created the
+    // same runtime directory.
+    CARBON_ASSIGN_OR_RETURN(entry_dir,
+                            dir_.OpenDir(entry_path, Filesystem::OpenAlways));
+  }
+
+  CARBON_ASSIGN_OR_RETURN(
+      auto lock_file, entry_dir.OpenWriteOnly(".lock", Filesystem::OpenAlways));
+  CARBON_RETURN_IF_ERROR(lock_file.UpdateTimes());
+  CARBON_ASSIGN_OR_RETURN(
+      Filesystem::FileLock flock,
+      lock_file.TryLock(Filesystem::FileLock::Shared, RuntimesLockDeadline,
+                        RuntimesLockPollInterval));
+
+  return Runtimes(path_ / entry_path, std::move(entry_dir),
+                  std::move(lock_file), std::move(flock), vlog_stream_);
+}
+
+auto Runtimes::Cache::ComputeEntryAges(
+    llvm::SmallVector<std::filesystem::path> entry_paths)
+    -> llvm::SmallVector<Entry> {
+  llvm::SmallVector<Entry> entries;
+
+  Filesystem::TimePoint now = Filesystem::Clock::now();
+  for (auto& path : entry_paths) {
+    // We use the `mtime` from the lock file in the directory rather than the
+    // directory itself to avoid any oddities with `mtime` on directories.
+    //
+    // Note that we also ignore errors here as if we can't read the stamp file
+    // we will pick an arbitrary old time stamp, and we want pruning to be
+    // maximally resilient to partially deleted or corrupted caches in order to
+    // prune them back into a healthy state.
+    auto stat_result = dir_.Lstat(path / ".lock");
+    auto mtime = stat_result.ok()
+                     ? stat_result->mtime()
+                     : Filesystem::TimePoint(Filesystem::Duration(0));
+    entries.push_back({.path = std::move(path), .age = now - mtime});
+  }
+  return entries;
+}
+
+auto Runtimes::Cache::PruneStaleRuntimes(
+    const std::filesystem::path& new_entry_path) -> void {
+  llvm::SmallVector<std::filesystem::path> dir_entries;
+  llvm::SmallVector<std::filesystem::path> non_dir_entries;
+  auto read_result = dir_.AppendEntriesIf(
+      dir_entries, non_dir_entries,
+      [](llvm::StringRef name) { return name.starts_with("runtimes-"); });
+  if (!read_result.ok()) {
+    CARBON_VLOG("Unable to read cache directory to prune stale entries: {0}",
+                read_result.error());
+    return;
+  }
+
+  // Directly attempt to remove non-directory and bad directory entries.
+  for (const auto& name : non_dir_entries) {
+    CARBON_VLOG("Unlinking non-directory entry '{0}'", name);
+    auto result = dir_.Unlink(name);
+    if (!result.ok()) {
+      CARBON_VLOG("Error unlinking non-directory entry '{0}': {1}", name,
+                  result.error());
+    }
+  }
+
+  // If we only have a small number of entries, no need to prune.
+  if (dir_entries.size() < MinNumEntries) {
+    return;
+  }
+
+  llvm::SmallVector<Entry> entries = ComputeEntryAges(std::move(dir_entries));
+
+  auto rm_entry = [&](const std::filesystem::path& entry_name) {
+    // Note that we don't propagate errors here because we want to prune as much
+    // as possible. We do log them.
+    CARBON_VLOG("Removing cache entry '{0}'", entry_name);
+    auto rm_result = dir_.Rmtree(entry_name);
+    if (!rm_result.ok() && !rm_result.error().no_entity()) {
+      CARBON_VLOG("Unable to remove old runtimes '{0}': {1}", entry_name,
+                  rm_result.error());
+      return false;
+    }
+    return true;
+  };
+
+  // Remove entries older than our max first. We don't need to check for locking
+  // or other issues here given the age.
+  llvm::erase_if(entries, [&](const Entry& entry) {
+    return entry.age > MaxEntryAge && rm_entry(entry.path);
+  });
+
+  // Sort the entries so that the oldest is first.
+  llvm::sort(entries, [](const Entry& lhs, const Entry& rhs) {
+    return lhs.age > rhs.age;
+  });
+
+  // Now try to get the number of entries below our max target by removing the
+  // least-recently used entries that are either more than our max locked age or
+  // unlocked.
+  auto rm_unlocked_entry = [&](const std::filesystem::path& name,
+                               Filesystem::Duration age) {
+    // Past a certain age, bypass the locking for efficiency and to avoid
+    // retaining entries with stale locks.
+    if (age > MaxLockedEntryAge) {
+      return rm_entry(name);
+    }
+
+    CARBON_VLOG("Attempting to lock cache entry '{0}'", name);
+    auto lock_file_open_result =
+        dir_.OpenReadOnly(name / ".lock", Filesystem::OpenAlways);
+    if (!lock_file_open_result.ok()) {
+      if (lock_file_open_result.error().no_entity() ||
+          lock_file_open_result.error().not_dir()) {
+        // The only way these failures should be possible is if something
+        // removed the cache directory between our read above and here. Assume
+        // the entry is gone and continue.
+        return true;
+      }
+
+      // For other errors, assume locked.
+      CARBON_VLOG("Error opening lock file for cache entry '{0}': {1}", name,
+                  lock_file_open_result.error());
+      return false;
+    }
+
+    Filesystem::ReadFile lock_file = *std::move(lock_file_open_result);
+    auto lock_result =
+        lock_file.TryLock(Filesystem::FileLock::Exclusive, RuntimesLockDeadline,
+                          RuntimesLockPollInterval);
+    if (!lock_result.ok()) {
+      // The normal case is when locking would block, log anything else.
+      if (!lock_result.error().would_block()) {
+        CARBON_VLOG("Error locking cache entry '{0}': {1}", name,
+                    lock_result.error());
+      }
+      // However, don't try to remove it as we didn't acquire the lock.
+      return false;
+    }
+
+    // The lock is held, remove the entry.
+    return rm_entry(name);
+  };
+
+  int num_entries = entries.size();
+  for (const auto& [name, age] : entries) {
+    if (num_entries < MaxNumEntries) {
+      break;
+    }
+
+    // Don't prune the currently being built entry. We should only reach here
+    // when some other process created this entry in a race, and we don't want
+    // to remove it or trigger rebuilds.
+    if (name == new_entry_path) {
+      continue;
+    }
+
+    if (rm_unlocked_entry(name, age)) {
+      --num_entries;
+    }
+  }
+
+  if (num_entries >= MaxNumEntries) {
+    CARBON_VLOG(
+        "Unable to prune cache to our target size due to held locks on recent "
+        "cache entries or removal errors, leaving {0} entries in the cache",
+        num_entries);
+  }
+}
+
+auto Runtimes::Builder::Commit() && -> ErrorOr<std::filesystem::path> {
+  std::filesystem::path dest_path = runtimes_->base_path() / dest_;
+
+  // First, try to do the atomic commit of the built runtimes into the final
+  // location.
+  CARBON_CHECK(dir_.abs_path().parent_path() == runtimes_->base_path(),
+               "Building a temporary directory '{0}' that is not in the "
+               "runtimes tree '{1}'",
+               dir_.abs_path(), runtimes_->base_path());
+  auto rename_result = runtimes_->base_dir().Rename(
+      dir_.abs_path().filename(), runtimes_->base_dir(), dest_);
+  // If the rename was successful, then we don't need to remove anything so
+  // release that state.
+  if (rename_result.ok()) {
+    std::move(dir_).Release();
+  } else if (rename_result.error().not_empty()) {
+    // Some other runtimes were successfully committed before ours, so we want
+    // to discard ours. We report errors cleaning up here as we don't want to
+    // pollute the filesystem excessively.
+    //
+    // TODO: Consider instead being more resilient to errors here and just log
+    // them.
+    CARBON_VLOG("PID {0} found racily built runtimes in cache path: {1}",
+                getpid(), dest_path);
+    CARBON_RETURN_IF_ERROR(std::move(dir_).Remove());
+  } else {
+    // An unexpected error occurred, propagate it and let the normal cleanup
+    // occur.
+    //
+    // TODO: It's possible we need to handle `EBUSY` here, likely by ensuring it
+    // is the *destination* that is busy and an existing, valid directory built
+    // concurrently.
+    return std::move(rename_result).error();
+  }
+
+  // Now that we've got a final path in place successfully, clear the flock if
+  // it is currently held.
+  ReleaseFileLock();
+
+  // Finally, the build is committed so finish putting this into the moved-from
+  // state by clearing the runtimes pointer.
+  runtimes_ = nullptr;
+  return dest_path;
+}
+
+auto Runtimes::Builder::ReleaseFileLock() -> void {
+  CARBON_CHECK(runtimes_ != nullptr);
+
+  if (flock_.is_locked()) {
+    std::filesystem::path dest_path = runtimes_->base_path() / dest_;
+    CARBON_VLOG("PID {0} releasing lock on cache path: {1}", getpid(),
+                dest_path);
+    (void)lock_file_.WriteFileFromString("");
+    flock_ = {};
+    (void)std::move(lock_file_).Close();
+  } else {
+    CARBON_CHECK(!lock_file_.is_valid());
+  }
+}
+
+auto Runtimes::Builder::Destroy() -> void {
+  // If the runtimes are null, no in-flight build is owned so nothing to do.
+  if (runtimes_ == nullptr) {
+    CARBON_CHECK(
+        !lock_file_.is_valid() && !flock_.is_locked() && !dir_.is_valid(),
+        "Builder left in a partially cleared state!");
+    return;
+  }
+
+  // Otherwise we need to abandon an in-flight build. First release the lock.
+  ReleaseFileLock();
+
+  // The rest of the cleanup is handled by the `RemovingDir` destructor.
+}
+
+}  // namespace Carbon

toolchain/driver/runtimes_cache.h

@@ -0,0 +1,421 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef CARBON_TOOLCHAIN_DRIVER_RUNTIMES_CACHE_H_
+#define CARBON_TOOLCHAIN_DRIVER_RUNTIMES_CACHE_H_
+
+#include <chrono>
+#include <filesystem>
+#include <utility>
+
+#include "common/check.h"
+#include "common/error.h"
+#include "common/filesystem.h"
+#include "common/ostream.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringRef.h"
+#include "toolchain/install/install_paths.h"
+
+namespace Carbon {
+
+// Manages a runtimes directory.
+//
+// Carbon (including Clang) relies on a set of runtime libraries and data files.
+// These are organized into a directory modeled by a `Runtimes` object, with
+// different components of those runtimes built into subdirectories. The
+// `Runtimes` object in turn provides access to each of these subdirectories.
+//
+// Different components are managed in separate directories based on how they
+// are _built_, and each one may contain a mixture of one or more runtime
+// libraries or data files. The separation of each component is so that we don't
+// force building all of them in a configuration where only one makes sense or
+// is typically needed.
+//
+// Beyond providing access, a `Runtimes` object also supports orchestrating the
+// build of these components into their designated subdirectories, including
+// synchronizing between different threads or processes trying to build the same
+// component.
+//
+// TODO: Add libc++ to the runtimes tree.
+// TODO: Add the Core library to the runtimes tree.
+class Runtimes {
+ public:
+  class Builder;
+  class Cache;
+
+  enum Component {
+    ClangResourceDir,
+
+    NumComponents,
+  };
+
+  // Creates a `Runtimes` object for a specific existing directory.
+  //
+  // Requires that the `path` is an absolute path that is an existing directory.
+  static auto Make(std::filesystem::path path,
+                   llvm::raw_ostream* vlog_stream = nullptr)
+      -> ErrorOr<Runtimes>;
+
+  // Default construction produces an unformed runtimes that can only be
+  // assigned to or destroyed.
+  Runtimes() = default;
+  // A specific runtimes object is move-only as it owns the relevant filesystem
+  // resources.
+  Runtimes(Runtimes&& arg) noexcept
+      : base_path_(std::move(arg.base_path_)),
+        base_dir_(std::exchange(arg.base_dir_, {})),
+        lock_file_(std::exchange(arg.lock_file_, {})),
+        flock_(std::exchange(arg.flock_, {})),
+        vlog_stream_(arg.vlog_stream_) {}
+  auto operator=(Runtimes&& arg) noexcept -> Runtimes& {
+    Destroy();
+    base_path_ = std::move(arg.base_path_);
+    base_dir_ = std::exchange(arg.base_dir_, {});
+    lock_file_ = std::exchange(arg.lock_file_, {});
+    flock_ = std::exchange(arg.flock_, {});
+    vlog_stream_ = arg.vlog_stream_;
+    return *this;
+  }
+  ~Runtimes() { Destroy(); }
+
+  // The base path for the runtimes.
+  auto base_path() const -> const std::filesystem::path& { return base_path_; }
+
+  // The base directory for the runtimes.
+  auto base_dir() const -> Filesystem::DirRef { return base_dir_; }
+
+  // Gets the path to an _existing_ Clang resource directory.
+  //
+  // Clang's resource directory contains all of the compiler-builtin runtime
+  // libraries, headers, and data files.
+  //
+  // This will return the path to the Clang resource directory if it exists in
+  // the runtimes tree. Otherwise, it will return an error.
+  auto Get(Component component) -> ErrorOr<std::filesystem::path>;
+
+  // Builds or returns a Clang resource directory.
+  //
+  // If there is an existing, built Clang resource directory, this will return
+  // its path, the same as `GetExistingClangResourceDir` would. However, if
+  // there is not yet a Clang resource directory in this runtimes tree, returns
+  // a `Builder` object that can be used to build and commit a Clang resource
+  // directory to this runtimes tree.
+  auto Build(Component component)
+      -> ErrorOr<std::variant<std::filesystem::path, Builder>>;
+
+ private:
+  friend Builder;
+  friend Cache;
+  friend class RuntimesTestPeer;
+
+  // The deadline for acquiring a lock to build a new component of the runtimes.
+  // This needs to be quite large as this is how long racing processes or
+  // threads will wait to allow some other process to complete building the
+  // component. The result is that this should be significantly longer than the
+  // expected slowest-to-build component.
+  //
+  // Note, nothing goes _wrong_ if this deadline is exceeded, but multiple
+  // copies of the component may end up being built and all but one thrown away.
+  static constexpr Filesystem::Duration BuildLockDeadline =
+      std::chrono::seconds(200);
+  // The interval at which to poll for a build lock. This needs to be small
+  // enough that we don't waste an excessive amount of time if a build of the
+  // component completes *just* after a poll. Typically, that means we want this
+  // to be significant lower than the expected time it would take to build the
+  // component. Note that we don't poll if the component has been completely
+  // built prior to the query coming in, so this doesn't form the _minimum_ time
+  // to find a component of the runtimes tree.
+  static constexpr Filesystem::Duration BuildLockPollInterval =
+      std::chrono::milliseconds(200);
+
+  // The path to the clang resource directory within the runtimes tree.
+  //
+  // This uses `std::string_view` to simply using with paths.
+  static constexpr auto ComponentPath(Component component) -> std::string_view {
+    switch (component) {
+      case ClangResourceDir:
+        return "clang_resource_dir";
+      case NumComponents:
+        CARBON_FATAL("Invalid component");
+    }
+  }
+
+  // A format string used to form the lock file for a given directory in the
+  // runtimes tree. This needs to be C-string, so directly expose the character
+  // array.
+  static constexpr char LockFileFormat[] = ".{0}.lock";
+
+  explicit Runtimes(std::filesystem::path base_path, Filesystem::Dir base_dir,
+                    Filesystem::WriteFile lock_file, Filesystem::FileLock flock,
+                    llvm::raw_ostream* vlog_stream = nullptr)
+      : base_path_(std::move(base_path)),
+        base_dir_(std::move(base_dir)),
+        lock_file_(std::move(lock_file)),
+        flock_(std::move(flock)),
+        vlog_stream_(vlog_stream) {
+    CARBON_CHECK(base_path_.is_absolute(),
+                 "The base path must be absolute: {0}", base_path_);
+  }
+
+  // Implementation of building the Clang resource directory. This exposes the
+  // deadline and poll interval to allow testing with artificial values.
+  auto BuildImpl(Component component, Filesystem::Duration deadline,
+                 Filesystem::Duration poll_interval)
+      -> ErrorOr<std::variant<std::filesystem::path, Builder>>;
+
+  auto Destroy() -> void;
+
+  std::filesystem::path base_path_;
+  Filesystem::Dir base_dir_;
+  Filesystem::WriteFile lock_file_;
+  Filesystem::FileLock flock_;
+  llvm::raw_ostream* vlog_stream_ = nullptr;
+};
+
+// A managed cache of `Runtimes` directories.
+//
+// This class manages and provides access to a cache of runtimes. Each entry in
+// the cache is a runtimes directory for a specific set of `Feature`s,
+// represented by an object of the `Runtimes` type. An entry is sometimes
+// referred to simply as the "runtimes" in a specific context. Each of these
+// entries can consist of one or more components that together make up a
+// collection of runtime libraries, runtime data files, or other runtime
+// resources. However, entries are never combined -- each entry represents a
+// distinct target environment, potentially ABI, and set of runtimes that could
+// be used.
+//
+// The cache looks up entries based on the set of `Feature`s and the input
+// sources used to build them (including the compiler itself). Whenever looking
+// up an entry not already present in the cache, the cache will evict old
+// entries before creating the new one. The eviction strategy is to remove any
+// entries more than a year old, as well as the least-recently used entries
+// until there will only be a maximum of 50 entries in the cache. The goal is to
+// allow multiple versions and build features to stay resident in the cache
+// while providing a stable upper bound on the disk space used.
+//
+// The cache can be formed around a specific directory, or it can search for a
+// system-default directory. The system default directory follows the guidance
+// of the XDG Base Directory Specification:
+// https://specifications.freedesktop.org/basedir-spec/latest/
+//
+// This tries to place the system cache in
+// `$XDG_CACHE_HOME/carbon_runtimes_cache`, followed by
+// `$HOME/.cache/carbon_runtimes_cache`. A fallback if neither works is to
+// create a temporary directory for the cache. This temporary directory is owned
+// by the `Cache` object and will be removed when it is destroyed.
+//
+// These system-wide paths are only used if the installation contains a digest
+// file that can be used to ensure different builds and installs of Carbon don't
+// incorrectly share cache entries built from different sources. When missing, a
+// temporary directory is used.
+class Runtimes::Cache {
+ public:
+  // The features of a cached runtimes directory.
+  //
+  // TODO: Add support for more build flags that we want to enable when building
+  // runtimes such as sanitizers and CPU-specific optimizations.
+  struct Features {
+    std::string target;
+  };
+
+  Cache() = default;
+
+  // The cache is move-only as it owns open resources for the cache directory.
+  Cache(Cache&& arg) noexcept
+      : vlog_stream_(arg.vlog_stream_),
+        cache_key_(std::move(arg.cache_key_)),
+        path_(std::move(arg.path_)),
+        dir_owner_(std::exchange(arg.dir_owner_, {})),
+        dir_(arg.dir_) {}
+  auto operator=(Cache&& arg) noexcept -> Cache& {
+    vlog_stream_ = arg.vlog_stream_;
+    cache_key_ = std::move(arg.cache_key_);
+    path_ = std::move(arg.path_);
+    dir_owner_ = std::exchange(arg.dir_owner_, {});
+    dir_ = arg.dir_;
+    return *this;
+  }
+
+  // Creates a cache object for the current system.
+  //
+  // This will try to locate and use a persistent cache on the system if it can,
+  // and otherwise fall back to creating a temporary cache. If either of these
+  // hit unrecoverable errors, that error is returned instead. See the class
+  // comment for more details about the overall strategy.
+  static auto MakeSystem(const InstallPaths& install,
+                         llvm::raw_ostream* vlog_stream = nullptr)
+      -> ErrorOr<Cache> {
+    Cache cache(vlog_stream);
+    CARBON_RETURN_IF_ERROR(cache.InitSystemCache(install));
+    return cache;
+  }
+
+  // Creates a cache object referencing an explicit cache path.
+  //
+  // The path must be an existing, writable directory.
+  static auto MakeCustom(const InstallPaths& install,
+                         std::filesystem::path cache_path,
+                         llvm::raw_ostream* vlog_stream = nullptr)
+      -> ErrorOr<Cache> {
+    Cache cache(vlog_stream);
+    CARBON_RETURN_IF_ERROR(cache.InitCachePath(install, cache_path));
+    return cache;
+  }
+
+  // The path to the cache directory.
+  auto path() const -> const std::filesystem::path& { return path_; }
+
+  // Looks up a runtimes directory in the cache.
+  //
+  // This will return a `Runtimes` object for the given features. If an entry
+  // for these features does not exist in the cache, any stale cache entries
+  // will be pruned if needed, and then a new entry will be created and
+  // returned.
+  auto Lookup(const Features& features) -> ErrorOr<Runtimes>;
+
+ private:
+  friend class RuntimesTestPeer;
+
+  static constexpr int MinNumEntries = 10;
+  static constexpr int MaxNumEntries = 50;
+
+  // The maximum age of a cache entry. Cache entries older than this will always
+  // evicted if there are more than the minimum number of entries.
+  static constexpr auto MaxEntryAge = std::chrono::years(1);
+  // The maximum age of a locked cache entry. Cache entries older than this will
+  // be evicted if needed without regard to any held lock from a process
+  // currently using that entry.
+  static constexpr auto MaxLockedEntryAge = std::chrono::days(10);
+
+  // Entries are locked while in use to avoid them being removed concurrently,
+  // but the lock will be disregarded for entries older than
+  // `MaxLockedEntryAge`. We use a relatively short deadline and fast poll
+  // interval here as this is on the critical path even for an existing, built
+  // runtimes entry.
+  static constexpr auto RuntimesLockDeadline = std::chrono::milliseconds(100);
+  static constexpr auto RuntimesLockPollInterval = std::chrono::milliseconds(1);
+
+  struct Entry {
+    std::filesystem::path path;
+    Filesystem::Duration age;
+  };
+
+  explicit Cache(llvm::raw_ostream* vlog_stream) : vlog_stream_(vlog_stream) {}
+
+  // Tries to find a viable cache root.
+  //
+  // This must be an existing directory, not one we create. We use the XDG base
+  // directory specification as the basis for these directories:
+  // https://specifications.freedesktop.org/basedir-spec/
+  //
+  // Note that there is a concept of a "runtimes" directory in this spec, but it
+  // uses a different meaning of the term "runtimes" than ours. Runtimes for
+  // Carbon are cached, persistent built library data, not something that only
+  // exists during the running of the Carbon tool like a socket.
+  auto FindXdgCachePath() -> std::optional<std::filesystem::path>;
+
+  // Initializes a system cache in a temporary directory.
+  //
+  // The cache will create and own a temporary directory, removing it on
+  // destruction. This limits the caching lifetime but is used as a fallback
+  // when unable to create a persistent cache.
+  auto InitTmpSystemCache() -> ErrorOr<Success>;
+
+  // Helper function implementing the logic for `MakeSystem`.
+  auto InitSystemCache(const InstallPaths& install) -> ErrorOr<Success>;
+
+  // Helper function implementing the logic for `MakeCustom`.
+  auto InitCachePath(const InstallPaths& install,
+                     std::filesystem::path cache_path) -> ErrorOr<Success>;
+
+  // Computes the ages for each input path, and combines the path and age into
+  // the returned vector of `Entry` objects. This consumes the input paths when
+  // building the output `Entry` structs, and so accepts the vector of paths by
+  // value.
+  auto ComputeEntryAges(llvm::SmallVector<std::filesystem::path> entry_paths)
+      -> llvm::SmallVector<Entry>;
+
+  // Prunes stale cache entries sufficiently to insert the provided new entry
+  // path into the cache without growing it beyond the thresholds for the cache
+  // size.
+  //
+  // Errors during pruning are logged rather than returned as this is expected
+  // to be a background operation and not something we can always recover from.
+  auto PruneStaleRuntimes(const std::filesystem::path& new_entry_path) -> void;
+
+  llvm::raw_ostream* vlog_stream_ = nullptr;
+  std::string cache_key_;
+  std::filesystem::path path_;
+  std::variant<Filesystem::Dir, Filesystem::RemovingDir> dir_owner_;
+  // A reference to whichever form of `dir_owner_` is in use.
+  Filesystem::DirRef dir_;
+};
+
+// Builder for a new directory in a runtimes tree.
+//
+// This manages a staging directory for the build that will then be committed
+// into the destination once fully built.
+class Runtimes::Builder : public Printable<Builder> {
+ public:
+  Builder(Builder&& arg) noexcept
+      : runtimes_(std::exchange(arg.runtimes_, nullptr)),
+        lock_file_(std::exchange(arg.lock_file_, {})),
+        flock_(std::exchange(arg.flock_, {})),
+        dir_(std::exchange(arg.dir_, {})),
+        dest_(arg.dest_) {}
+  auto operator=(Builder&& arg) noexcept -> Builder& {
+    Destroy();
+    runtimes_ = std::exchange(arg.runtimes_, nullptr);
+    lock_file_ = std::exchange(arg.lock_file_, {});
+    flock_ = std::exchange(arg.flock_, {});
+    dir_ = std::exchange(arg.dir_, {});
+    dest_ = arg.dest_;
+    return *this;
+  }
+  ~Builder() { Destroy(); }
+
+  // The build's staging directory.
+  auto dir() const -> Filesystem::DirRef { return dir_; }
+  // The build's staging directory path.
+  auto path() const -> const std::filesystem::path& { return dir_.abs_path(); }
+
+  // Commits the new runtime to the cache.
+  //
+  // This will move the contents of the temporary directory to the final
+  // destination in the cache.
+  auto Commit() && -> ErrorOr<std::filesystem::path>;
+
+  auto Print(llvm::raw_ostream& out) const -> void {
+    out << "Runtimes::Builder{.path = '" << path() << "'}";
+  }
+
+ private:
+  friend Runtimes;
+  friend class RuntimesTestPeer;
+
+  Builder() = default;
+  explicit Builder(Runtimes& runtimes, Filesystem::ReadWriteFile lock_file,
+                   Filesystem::FileLock flock, Filesystem::RemovingDir tmp_dir,
+                   std::string_view dest)
+      : runtimes_(&runtimes),
+        vlog_stream_(runtimes.vlog_stream_),
+        lock_file_(std::move(lock_file)),
+        flock_(std::move(flock)),
+        dir_(std::move(tmp_dir)),
+        dest_(dest) {}
+
+  auto ReleaseFileLock() -> void;
+  auto Destroy() -> void;
+
+  Runtimes* runtimes_ = nullptr;
+  llvm::raw_ostream* vlog_stream_ = nullptr;
+  Filesystem::ReadWriteFile lock_file_;
+  Filesystem::FileLock flock_;
+  Filesystem::RemovingDir dir_;
+  std::string_view dest_;
+};
+
+}  // namespace Carbon
+
+#endif  // CARBON_TOOLCHAIN_DRIVER_RUNTIMES_CACHE_H_

toolchain/driver/runtimes_cache_test.cpp

@@ -0,0 +1,718 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "toolchain/driver/runtimes_cache.h"
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include <chrono>
+#include <filesystem>
+#include <fstream>
+#include <limits>
+#include <mutex>
+#include <ratio>
+#include <string>
+#include <thread>
+#include <utility>
+#include <variant>
+
+#include "common/check.h"
+#include "common/error_test_helpers.h"
+#include "common/filesystem.h"
+#include "common/ostream.h"
+#include "common/raw_string_ostream.h"
+#include "common/version.h"
+#include "llvm/ADT/ScopeExit.h"
+#include "llvm/Support/SHA256.h"
+#include "testing/base/capture_std_streams.h"
+#include "testing/base/file_helpers.h"
+#include "testing/base/global_exe_path.h"
+
+namespace Carbon {
+
+class RuntimesTestPeer {
+ public:
+  static auto LockFilePath(Runtimes::Component component) -> std::string {
+    return llvm::formatv(Runtimes::LockFileFormat,
+                         Runtimes::ComponentPath(component))
+        .str();
+  }
+
+  static auto BuildImpl(Runtimes& runtimes, Runtimes::Component component,
+                        Filesystem::Duration deadline,
+                        Filesystem::Duration poll_interval)
+      -> ErrorOr<std::variant<std::filesystem::path, Runtimes::Builder>> {
+    return runtimes.BuildImpl(component, deadline, poll_interval);
+  }
+
+  static auto CacheMinNumEntries() -> int {
+    return Runtimes::Cache::MinNumEntries;
+  }
+  static auto CacheMaxNumEntries() -> int {
+    return Runtimes::Cache::MaxNumEntries;
+  }
+};
+
+namespace {
+
+using ::testing::_;
+using ::testing::AllOf;
+using ::testing::AnyOf;
+using ::testing::Eq;
+using ::testing::Gt;
+using Testing::IsError;
+using Testing::IsSuccess;
+using ::testing::Lt;
+using ::testing::Ne;
+using ::testing::Not;
+using ::testing::StartsWith;
+using ::testing::StrEq;
+using ::testing::VariantWith;
+
+class RuntimesCacheTest : public ::testing::Test {
+ public:
+  RuntimesCacheTest()
+      : cache_(*Runtimes::Cache::MakeCustom(install_, tmp_dir_.abs_path())) {}
+
+  auto LookupNRuntimes(int n) -> llvm::SmallVector<Runtimes> {
+    llvm::SmallVector<Runtimes> runtimes;
+    for (int i : llvm::seq(n)) {
+      runtimes.push_back(*cache_.Lookup(
+          {.target = llvm::formatv("aarch64-unknown-unknown{0}", i).str()}));
+    }
+    return runtimes;
+  }
+
+  InstallPaths install_ =
+      InstallPaths::MakeForBazelRunfiles(Testing::GetExePath());
+  Filesystem::RemovingDir tmp_dir_ = *Filesystem::MakeTmpDir();
+  std::string cache_key_ = "test cache";
+  Runtimes::Cache cache_;
+};
+
+TEST_F(RuntimesCacheTest, BuildSystemCache) {
+  // Create an install with a missing digest.
+  auto bad_install_dir = *tmp_dir_.CreateDirectories("bad_install/lib/carbon");
+  bad_install_dir.WriteFileFromString("carbon_install.txt", "no digest")
+      .Check();
+  InstallPaths bad_install =
+      InstallPaths::Make((tmp_dir_.abs_path() / "bad_install").native());
+
+  // Create directories to use in various environment variables.
+  auto xdg_dir = *tmp_dir_.CreateDirectories("xdg_cache_home");
+  std::filesystem::path xdg_path = tmp_dir_.abs_path() / "xdg_cache_home";
+  auto test_home = *tmp_dir_.CreateDirectories("test_home");
+  std::filesystem::path home_path = tmp_dir_.abs_path() / "test_home";
+  auto home_cache_dir = *test_home.CreateDirectories(".cache");
+  std::filesystem::path home_cache_path = home_path / ".cache";
+
+  // Save the environment variables we'll override for testing and restore them
+  // afterward to avoid test-to-test oddities.
+  constexpr const char* XdgCacheEnv = "XDG_CACHE_HOME";
+  constexpr const char* HomeEnv = "HOME";
+  const char* orig_xdg_cache = getenv(XdgCacheEnv);
+  const char* orig_home = getenv(HomeEnv);
+  auto restore_env = llvm::make_scope_exit([&] {
+    for (const auto [env, orig] : {std::pair{XdgCacheEnv, orig_xdg_cache},
+                                   std::pair{HomeEnv, orig_home}}) {
+      if (orig) {
+        setenv(env, orig, /*overwrite*/ true);
+      } else {
+        unsetenv(env);
+      }
+    }
+  });
+
+  // Begin testing the basic logic of selecting different roots for the cache.
+  setenv(XdgCacheEnv, xdg_path.c_str(), /*overwrite*/ true);
+  setenv(HomeEnv, home_path.c_str(), /*overwrite*/ true);
+
+  // First check that even with all the environment set up, when we don't have a
+  // digest file available, we bypass those options and use a temporary cache
+  // path. This is the only safe approach as without a digest file we can't
+  // track whether it is correct to reuse a persistently cached entry.
+  auto result = Runtimes::Cache::MakeSystem(bad_install);
+  ASSERT_THAT(result, IsSuccess(_));
+  EXPECT_THAT(result->path(), Not(StartsWith(home_cache_path)));
+  EXPECT_THAT(result->path(), Not(StartsWith(xdg_path)));
+
+  // Once we have a digest, the main XDG cache logic should work.
+  result = Runtimes::Cache::MakeSystem(install_);
+  ASSERT_THAT(result, IsSuccess(_));
+  EXPECT_THAT(result->path(), StartsWith(xdg_path));
+  // Destruction shouldn't remove system cache directories.
+  result = Error("nothing");
+  EXPECT_TRUE(*tmp_dir_.Access("xdg_cache_home"));
+
+  // Remove the XDG cache directory, but leave the environment set. We want to
+  // be robust against this, but it isn't important *how* the fallback occurs,
+  // it could go to `$HOME/.cache`, or to a temporary directory.
+  tmp_dir_.Rmtree("xdg_cache_home").Check();
+  EXPECT_THAT(Runtimes::Cache::MakeSystem(install_), IsSuccess(_));
+
+  // Set the XDG environment to the empty string which should trigger using the
+  // home directory.
+  setenv(XdgCacheEnv, "", /*overwrite*/ true);
+  result = Runtimes::Cache::MakeSystem(install_);
+  ASSERT_THAT(result, IsSuccess(_));
+  EXPECT_THAT(result->path(), StartsWith(home_cache_path));
+  // Destruction shouldn't remove system cache directories.
+  result = Error("nothing");
+  EXPECT_TRUE(*tmp_dir_.Access("test_home"));
+  EXPECT_TRUE(*test_home.Access(".cache"));
+
+  // Same as with an empty string, but with a relative path instead.
+  setenv(XdgCacheEnv, "relative/cache/home", /*overwrite*/ true);
+  result = Runtimes::Cache::MakeSystem(install_);
+  ASSERT_THAT(result, IsSuccess(_));
+  EXPECT_THAT(result->path(), StartsWith(home_cache_path));
+  // Destruction shouldn't remove system cache directories.
+  result = Error("nothing");
+  EXPECT_TRUE(*tmp_dir_.Access("test_home"));
+  EXPECT_TRUE(*test_home.Access(".cache"));
+
+  // Same as with an empty string, but this time with an unset environment
+  // variable.
+  unsetenv(XdgCacheEnv);
+  result = Runtimes::Cache::MakeSystem(install_);
+  ASSERT_THAT(result, IsSuccess(_));
+  EXPECT_THAT(result->path(), StartsWith(home_cache_path));
+  // Destruction shouldn't remove system cache directories.
+  result = Error("nothing");
+  EXPECT_TRUE(*tmp_dir_.Access("test_home"));
+  EXPECT_TRUE(*test_home.Access(".cache"));
+
+  // Now check a bunch of different failure modes for the home directory
+  // fallback. These should all end up creating temporary directories which
+  // we'll test functionally at the end.
+  setenv(HomeEnv, "", /*overwrite*/ true);
+  EXPECT_THAT(Runtimes::Cache::MakeSystem(install_), IsSuccess(_));
+  setenv(HomeEnv, "relative/home", /*overwrite*/ true);
+  EXPECT_THAT(Runtimes::Cache::MakeSystem(install_), IsSuccess(_));
+
+  // Correct the path and make sure it works again.
+  setenv(HomeEnv, home_path.c_str(), /*overwrite*/ true);
+  result = Runtimes::Cache::MakeSystem(install_);
+  ASSERT_THAT(result, IsSuccess(_));
+  EXPECT_THAT(result->path(), StartsWith(home_cache_path));
+
+  // Now try removing directories around home.
+  test_home.Rmtree(".cache").Check();
+  EXPECT_THAT(Runtimes::Cache::MakeSystem(install_), IsSuccess(_));
+  tmp_dir_.Rmtree("test_home").Check();
+  EXPECT_THAT(Runtimes::Cache::MakeSystem(install_), IsSuccess(_));
+
+  // Finally, double check that these temporary caches still produce a writable
+  // directory.
+  result = Runtimes::Cache::MakeSystem(install_);
+  ASSERT_THAT(result, IsSuccess(_));
+  EXPECT_THAT(result->path(), Not(StartsWith(home_cache_path)));
+  EXPECT_THAT(result->path(), Not(StartsWith(xdg_path)));
+  ASSERT_THAT(Filesystem::Cwd().WriteFileFromString(
+                  result->path() / "test_file", "test"),
+              IsSuccess(_));
+  ASSERT_THAT(Filesystem::Cwd().ReadFileToString(result->path() / "test_file"),
+              IsSuccess(StrEq("test")));
+}
+
+TEST_F(RuntimesCacheTest, BasicBuild) {
+  llvm::SmallVector<std::string> targets = {"aarch64-unknown-unknown",
+                                            "x86_64-unknown-unknown"};
+  llvm::SmallVector<std::filesystem::path> built_runtimes_paths;
+
+  for (const std::string& target : targets) {
+    SCOPED_TRACE(target);
+    auto lookup_result = cache_.Lookup({.target = target});
+    ASSERT_THAT(lookup_result, IsSuccess(_));
+    auto runtimes = *std::move(lookup_result);
+
+    auto build_result = runtimes.Build(Runtimes::ClangResourceDir);
+    ASSERT_THAT(build_result, IsSuccess(VariantWith<Runtimes::Builder>(_)));
+    auto builder = std::get<Runtimes::Builder>(*std::move(build_result));
+    EXPECT_TRUE(builder.path().is_absolute()) << builder.path();
+
+    // Create a file as our "runtime".
+    builder.dir().WriteFileFromString("runtime_file", target).Check();
+    // Make sure the builder's path finds this file.
+    EXPECT_THAT(
+        Filesystem::Cwd().ReadFileToString(builder.path() / "runtime_file"),
+        IsSuccess(StrEq(target)));
+
+    auto commit_result = std::move(builder).Commit();
+    ASSERT_THAT(commit_result, IsSuccess(_));
+    std::filesystem::path clang_runtimes_path = *std::move(commit_result);
+
+    EXPECT_THAT(
+        runtimes.Build(Runtimes::ClangResourceDir),
+        IsSuccess(VariantWith<std::filesystem::path>(Eq(clang_runtimes_path))));
+    built_runtimes_paths.push_back(clang_runtimes_path);
+  }
+
+  for (const auto& [target, built_runtimes_path] :
+       llvm::zip(targets, built_runtimes_paths)) {
+    SCOPED_TRACE(target);
+    auto lookup_result = cache_.Lookup({.target = target});
+    ASSERT_THAT(lookup_result, IsSuccess(_));
+    auto runtimes = *std::move(lookup_result);
+
+    EXPECT_THAT(
+        runtimes.Build(Runtimes::ClangResourceDir),
+        IsSuccess(VariantWith<std::filesystem::path>(Eq(built_runtimes_path))));
+  }
+}
+
+TEST_F(RuntimesCacheTest, DifferentKeys) {
+  const std::string target = "aarch64-unknown-unknown";
+  auto runtimes1 = *cache_.Lookup({.target = target});
+
+  // Build a second cache with a different key but pointing at the same
+  // directory and target to simulate two versions or builds of the Carbon
+  // toolchain.
+  auto custom_install_dir =
+      *tmp_dir_.CreateDirectories("custom_install/lib/carbon");
+  custom_install_dir.WriteFileFromString("carbon_install.txt", "diff digest")
+      .Check();
+  custom_install_dir.WriteFileFromString("install_digest.txt", "abcd").Check();
+  InstallPaths install2 =
+      InstallPaths::Make((tmp_dir_.abs_path() / "custom_install").native());
+  auto cache2 = *Runtimes::Cache::MakeCustom(install2, tmp_dir_.abs_path());
+  auto runtimes2 = *cache2.Lookup({.target = target});
+
+  // The parent paths of these runtimes should be the same.
+  EXPECT_THAT(runtimes1.base_path().parent_path(),
+              Eq(runtimes2.base_path().parent_path()));
+
+  // But the base paths for these two runtimes should differ due to cache key
+  // differences.
+  EXPECT_THAT(runtimes1.base_path(), Ne(runtimes2.base_path()));
+}
+
+TEST_F(RuntimesCacheTest, ConcurrentBuilds) {
+  const std::string target = "aarch64-unknown-unknown";
+  auto runtimes1 = *cache_.Lookup({.target = target});
+
+  // Build a second cache and runtimes pointing at the same directory and target
+  // to simulate concurrent processes.
+  auto cache2 = *Runtimes::Cache::MakeCustom(install_, tmp_dir_.abs_path());
+  auto runtimes2 = *cache2.Lookup({.target = target});
+
+  // Start the first build, this will lock the directory.
+  auto build_result1 = runtimes1.Build(Runtimes::ClangResourceDir);
+  ASSERT_THAT(build_result1, IsSuccess(VariantWith<Runtimes::Builder>(_)));
+  auto builder1 = std::get<Runtimes::Builder>(*std::move(build_result1));
+  EXPECT_THAT(builder1.dir().WriteFileFromString("runtime_file", "build1"),
+              IsSuccess(_));
+
+  // Start the second build in a separate thread so that it can block while we
+  // finish the first build. The only result we'll need at the end is the built
+  // path.
+  std::filesystem::path build2_path;
+  auto build2_lambda = [&build2_path, &runtimes2] {
+    // Typically building here will try to acquire the same file lock acquired
+    // with the first build. However, the file locking is always _advisory_ and
+    // may fail. As a consequence we can't make assumptions about whether this
+    // blocks or not.
+    auto build_result2 = runtimes2.Build(Runtimes::ClangResourceDir);
+    ASSERT_THAT(build_result2, IsSuccess(_));
+    if (std::holds_alternative<std::filesystem::path>(*build_result2)) {
+      // In the common case, we blocked on a file lock and find the first built
+      // result directly. Save it.
+      build2_path = std::get<std::filesystem::path>(*std::move(build_result2));
+    } else {
+      // In rare cases, the initial build will fail to acquire the file lock.
+      // The entire build process is designed specifically to be resilient to
+      // that so we should still succeed, but now we need to handle building in
+      // this thread as well. Note that a true failure here may only
+      // show up intermittently.
+      auto builder2 = std::get<Runtimes::Builder>(*std::move(build_result2));
+      builder2.dir().WriteFileFromString("runtime_file", "build2").Check();
+      auto commit2_result = std::move(builder2).Commit();
+      ASSERT_THAT(commit2_result, IsSuccess(_));
+      build2_path = *std::move(commit2_result);
+    }
+  };
+  std::thread build2_thread(build2_lambda);
+  // Use a scoped join to avoid leaking the thread as some platforms don't have
+  // `std::jthread`.
+  auto scoped_join =
+      llvm::make_scope_exit([&build2_thread] { build2_thread.join(); });
+
+  // Commit the first built runtime.
+  auto commit_result = std::move(builder1).Commit();
+  ASSERT_THAT(commit_result, IsSuccess(_));
+  std::filesystem::path build1_path = *std::move(commit_result);
+
+  // Even though there may be is another thread running, we should now get
+  // non-blocking access directly to the built runtime.
+  EXPECT_THAT(runtimes1.Build(Runtimes::ClangResourceDir),
+              IsSuccess(VariantWith<std::filesystem::path>(Eq(build1_path))));
+
+  // Now join the second cache's build thread to ensure it completes and verify
+  // that it produces the same path fully-built path.
+  build2_thread.join();
+  scoped_join.release();
+  EXPECT_THAT(build2_path, Eq(build1_path));
+
+  // Note that we don't know which build actually ended up committed here so
+  // accept either. The first one is much more common, but in rare cases it will
+  // fail to acquire its file lock and we will have racing builds. In that case
+  // the second build may commit first.
+  EXPECT_THAT(*Filesystem::Cwd().ReadFileToString(build1_path / "runtime_file"),
+              AnyOf(StrEq("build1"), StrEq("build2")));
+}
+
+TEST_F(RuntimesCacheTest, ConcurrentBuildsWithFailedLocking) {
+  // This test is very similar to `ConcurrentBuild` in terms of what can happen.
+  // But here, we intentionally subvert the file locking and even us
+  // synchronization to maximize the chance of racing commits.
+  //
+  // The goal here is to do two things:
+  // 1) Provide more direct stress testing of lock-file-failure modes and racing
+  //    commits to catch any consistent bugs that emerge.
+  // 2) Ensure that a removed lock file specifically is handled gracefully, both
+  // by a build with the file open and locked, and by a racing build.
+  const std::string target = "aarch64-unknown-unknown";
+  auto runtimes1 = *cache_.Lookup({.target = target});
+
+  // Build a second cache and runtimes pointing at the same directory and target
+  // to simulate concurrent processes.
+  auto cache2 = *Runtimes::Cache::MakeCustom(install_, tmp_dir_.abs_path());
+  auto runtimes2 = *cache2.Lookup({.target = target});
+
+  // Start the first build, this will lock the directory.
+  auto build_result1 = runtimes1.Build(Runtimes::ClangResourceDir);
+  ASSERT_THAT(build_result1, IsSuccess(VariantWith<Runtimes::Builder>(_)));
+  auto builder1 = std::get<Runtimes::Builder>(*std::move(build_result1));
+  builder1.dir().WriteFileFromString("runtime_file", "build1").Check();
+
+  // Now sneakily remove the lock file from the runtimes directory in the cache.
+  // This is something that could happen, for example from temporary directories
+  // being cleaned. The cache should be resilient against this and it gives us a
+  // good way to have two racing builds of the same directory.
+  std::filesystem::path lock_file_path =
+      RuntimesTestPeer::LockFilePath(Runtimes::ClangResourceDir);
+  ASSERT_THAT(runtimes1.base_dir().Unlink(lock_file_path), IsSuccess(_));
+
+  // We will synchronize with the thread to ensure we _actually_ have two
+  // parallel builds rather than accidentally having a fully serial execution.
+  std::mutex m;
+  std::condition_variable cv;
+  bool build_started = false;
+
+  // Start the second build in a separate thread. The only result we'll need at
+  // the end is the built path.
+  std::filesystem::path build2_path;
+  auto build2_lambda = [&build2_path, &runtimes2, target, &m, &cv,
+                        &build_started] {
+    auto build_result2 = runtimes2.Build(Runtimes::ClangResourceDir);
+    ASSERT_THAT(build_result2, IsSuccess(VariantWith<Runtimes::Builder>(_)));
+    auto builder2 = std::get<Runtimes::Builder>(*std::move(build_result2));
+    builder2.dir().WriteFileFromString("runtime_file", "build2").Check();
+
+    // Notify the first thread to commit its build and concurrently commit this
+    // built runtime. The goal is to get as close as we can to having these
+    // commits actually race so that a failure in that mode would emerge as a
+    // flake of the test. None of this is providing correctness.
+    {
+      std::unique_lock lock(m);
+      build_started = true;
+      cv.notify_one();
+    }
+    auto commit_result = std::move(builder2).Commit();
+    ASSERT_THAT(commit_result, IsSuccess(_));
+    build2_path = *std::move(commit_result);
+
+    // Even though there may be another thread running, and even holding a lock
+    // file, we should now get non-blocking access directly to the built
+    // runtime. This is mostly added for completeness, a held lock is more
+    // directly tested in `CurrentBuildsLockTimeout`.
+    EXPECT_THAT(runtimes2.Build(Runtimes::ClangResourceDir),
+                IsSuccess(VariantWith<std::filesystem::path>(Eq(build2_path))));
+  };
+  std::thread build2_thread(build2_lambda);
+  // Use a scoped join to avoid leaking the thread as some platforms don't have
+  // `std::jthread`.
+  auto scoped_join =
+      llvm::make_scope_exit([&build2_thread] { build2_thread.join(); });
+
+  // As soon as the second thread notifies that its build is started and ready
+  // to commit, also commit the first built runtime.
+  {
+    std::unique_lock lock(m);
+    cv.wait(lock, [&build_started] { return build_started; });
+  }
+  auto commit_result = std::move(builder1).Commit();
+  ASSERT_THAT(commit_result, IsSuccess(_));
+  std::filesystem::path build1_path = *std::move(commit_result);
+
+  // Even though there may be another thread running, we should now get
+  // non-blocking access directly to the built runtime.
+  EXPECT_THAT(runtimes1.Build(Runtimes::ClangResourceDir),
+              IsSuccess(VariantWith<std::filesystem::path>(Eq(build1_path))));
+
+  // Now join the second cache's build thread to ensure it completes and verify
+  // that it produces the same path fully-built path.
+  build2_thread.join();
+  scoped_join.release();
+  EXPECT_THAT(build2_path, Eq(build1_path));
+
+  // Much like the simple concurrent build, we can't know which build finished
+  // first so we need to accept either build's runtime file.
+  EXPECT_THAT(*Filesystem::Cwd().ReadFileToString(build1_path / "runtime_file"),
+              AnyOf(StrEq("build1"), StrEq("build2")));
+}
+
+TEST_F(RuntimesCacheTest, ConcurrentBuildsLockTimeout) {
+  // Another test designed to be similar to `ConcurrentBuilds` but stressing a
+  // failure path. Here, we want to reliably exercise the code path where a lock
+  // file is held when a second build begins and it polls and times out. This
+  // can happen naturally, even with very large timeouts under sufficient system
+  // load. Here, we artificially make it as likely as possible for better stress
+  // testing and easier debugging of problems with this situation.
+  const std::string target = "aarch64-unknown-unknown";
+  auto runtimes1 = *cache_.Lookup({.target = target});
+
+  // Build a second cache and runtimes pointing at the same directory and target
+  // to simulate concurrent processes.
+  auto cache2 = *Runtimes::Cache::MakeCustom(install_, tmp_dir_.abs_path());
+  auto runtimes2 = *cache2.Lookup({.target = target});
+
+  // Start the first build, this will lock the directory.
+  auto build_result1 = runtimes1.Build(Runtimes::ClangResourceDir);
+  ASSERT_THAT(build_result1, IsSuccess(VariantWith<Runtimes::Builder>(_)));
+  auto builder1 = std::get<Runtimes::Builder>(*std::move(build_result1));
+  builder1.dir().WriteFileFromString("runtime_file", "build1").Check();
+
+  // Directly simulate a second thread or process timing out on acquiring the
+  // file-based advisory lock by giving it an artificially short timeout and
+  // running it in the same thread. This should only poll for 50ms before
+  // proceeding without the lock.
+  //
+  // However, note that this is not *guaranteed* -- the first build may have
+  // exhausted the much higher default poll timeout and failed to acquire a file
+  // lock at all. When that happens, this path may in turn succeed at acquiring
+  // the file lock. All of that is fine, and the test even remains effective as
+  // either way we have successfully exercised the code path with lock file
+  // timeout. The lowered time here just ensures that the test finishes promptly
+  // relative to the system load.
+  auto build_result2 = RuntimesTestPeer::BuildImpl(
+      runtimes2, Runtimes::ClangResourceDir, std::chrono::milliseconds(50),
+      std::chrono::milliseconds(10));
+  ASSERT_THAT(build_result2, IsSuccess(VariantWith<Runtimes::Builder>(_)));
+  auto builder2 = std::get<Runtimes::Builder>(*std::move(build_result2));
+  builder2.dir().WriteFileFromString("runtime_file", "build2").Check();
+
+  // Commit the second runtime, as this one *doesn't* hold any lock. This leaves
+  // the lock present and held, but creates a valid runtimes directory.
+  auto commit2_result = std::move(builder2).Commit();
+  ASSERT_THAT(commit2_result, IsSuccess(_));
+  std::filesystem::path build2_path = *std::move(commit2_result);
+
+  // Now, even though we still have the lock file held, repeatedly building
+  // proceeds without blocking.
+  EXPECT_THAT(runtimes2.Build(Runtimes::ClangResourceDir),
+              IsSuccess(VariantWith<std::filesystem::path>(Eq(build2_path))));
+
+  // Finally, commit the lock-holding build to ensure it also succeeds, even
+  // though it will reliably discard its built cache.
+  auto commit1_result = std::move(builder1).Commit();
+  ASSERT_THAT(commit1_result, IsSuccess(_));
+  std::filesystem::path build1_path = *std::move(commit1_result);
+
+  // And ensure that we got the same path and the second build's contents.
+  EXPECT_THAT(build1_path, Eq(build2_path));
+  EXPECT_THAT(*Filesystem::Cwd().ReadFileToString(build1_path / "runtime_file"),
+              StrEq("build2"));
+}
+
+TEST_F(RuntimesCacheTest, Lookup) {
+  // Basic successful lookup of a new runtimes.
+  auto lookup_result = cache_.Lookup({.target = "aarch64-unknown-unknown"});
+  ASSERT_THAT(lookup_result, IsSuccess(_));
+  auto runtimes = *std::move(lookup_result);
+
+  auto lock_stat = runtimes.base_dir().Stat(".lock");
+  ASSERT_THAT(lock_stat, IsSuccess(_));
+  EXPECT_TRUE(lock_stat->is_file());
+
+  // Looking up the same target should return the same runtimes.
+  lookup_result = cache_.Lookup({.target = "aarch64-unknown-unknown"});
+  ASSERT_THAT(lookup_result, IsSuccess(_));
+  auto runtimes2 = *std::move(lookup_result);
+  EXPECT_THAT(runtimes2.base_path(), Eq(runtimes.base_path()));
+
+  EXPECT_THAT(runtimes.base_dir().Stat()->unix_inode(),
+              Eq(runtimes.base_dir().Stat()->unix_inode()));
+}
+
+TEST_F(RuntimesCacheTest, LookupFailsIfCannotCreateDir) {
+  // Create a read-only directory with the cache in it to cause failures.
+  std::filesystem::path ro_cache_path = tmp_dir_.abs_path() / "ro_cache";
+  tmp_dir_.CreateDirectories("ro_cache", /*creation_mode=*/0500).Check();
+  auto ro_cache = *Runtimes::Cache::MakeCustom(install_, ro_cache_path);
+
+  auto lookup_result = ro_cache.Lookup({.target = "aarch64-unknown-unknown"});
+  EXPECT_THAT(lookup_result, IsError(_));
+}
+
+TEST_F(RuntimesCacheTest, LookupWithSmallNumberOfStaleRuntimes) {
+  // Lookup two runtimes to populate the cache.
+  auto runtimes1 = *cache_.Lookup({.target = "aarch64-unknown-unknown1"});
+  auto runtimes2 = *cache_.Lookup({.target = "aarch64-unknown-unknown2"});
+
+  // Get the Unix-like inode of the directories so we can check whether
+  // subsequent lookups create a new directory.
+  auto runtimes1_inode = runtimes1.base_dir().Stat()->unix_inode();
+  auto runtimes2_inode = runtimes2.base_dir().Stat()->unix_inode();
+
+  // Now adjust their age backwards in time by two years to make them very, very
+  // stale.
+  auto now = Filesystem::Clock::now();
+  auto two_years_ago = now - std::chrono::years(2);
+  runtimes1.base_dir().UpdateTimes(".lock", two_years_ago).Check();
+  runtimes2.base_dir().UpdateTimes(".lock", two_years_ago).Check();
+
+  // Close the runtimes, releasing any locks.
+  runtimes1 = {};
+  runtimes2 = {};
+
+  // Lookup a new runtime, potentially pruning stale ones.
+  auto runtimes3 = *cache_.Lookup({.target = "aarch64-unknown-unknown3"});
+
+  // Redo the previous lookups and ensure they found the original directories as
+  // we don't have enough runtimes to prune.
+  runtimes1 = *cache_.Lookup({.target = "aarch64-unknown-unknown1"});
+  runtimes2 = *cache_.Lookup({.target = "aarch64-unknown-unknown2"});
+  EXPECT_THAT(runtimes1.base_dir().Stat()->unix_inode(), Eq(runtimes1_inode));
+  EXPECT_THAT(runtimes2.base_dir().Stat()->unix_inode(), Eq(runtimes2_inode));
+
+  // The timestamp on the lock file should also be updated. We can't assume the
+  // filesystem clock is monotonic, so it is possible an adjustment occurs while
+  // this test is running. We check that the updated time is within 2 days of
+  // `now` to minimize flake risks, which should be completely fine to detect
+  // bugs as we set the time to 2 years in the past above.
+  EXPECT_THAT(
+      runtimes1.base_dir().Stat(".lock")->mtime(),
+      AllOf(Gt(now - std::chrono::days(2)), Lt(now + std::chrono::days(2))));
+  EXPECT_THAT(
+      runtimes2.base_dir().Stat(".lock")->mtime(),
+      AllOf(Gt(now - std::chrono::days(2)), Lt(now + std::chrono::days(2))));
+}
+
+TEST_F(RuntimesCacheTest, LookupWithManyStaleRuntimes) {
+  auto runtimes1 = *cache_.Lookup({.target = "aarch64-unknown-unknown-fresh1"});
+  auto stale_runtimes = LookupNRuntimes(RuntimesTestPeer::CacheMinNumEntries());
+  auto runtimes2 = *cache_.Lookup({.target = "aarch64-unknown-unknown-fresh2"});
+
+  // Get the Unix-like inode of the directories so we can check whether
+  // subsequent lookups create a new directory.
+  auto runtimes1_inode = runtimes1.base_dir().Stat()->unix_inode();
+  auto stale_runtimes_0_inode =
+      stale_runtimes[0].base_dir().Stat()->unix_inode();
+  auto runtimes2_inode = runtimes2.base_dir().Stat()->unix_inode();
+
+  // Now adjust their age backwards in time by two years to make them very, very
+  // stale.
+  auto now = Filesystem::Clock::now();
+  auto two_years_ago = now - std::chrono::years(2);
+  for (auto& stale_runtime : stale_runtimes) {
+    stale_runtime.base_dir().UpdateTimes(".lock", two_years_ago).Check();
+  }
+
+  // Close the runtimes, releasing any locks.
+  runtimes1 = {};
+  stale_runtimes.clear();
+  runtimes2 = {};
+
+  // Lookup a new runtime, potentially pruning stale ones.
+  auto runtimes3 = *cache_.Lookup({.target = "aarch64-unknown-unknown-fresh3"});
+
+  // Re-lookup three of the original runtimes.
+  runtimes1 = *cache_.Lookup({.target = "aarch64-unknown-unknown-fresh1"});
+  auto stale_runtimes_0 =
+      *cache_.Lookup({.target = "aarch64-unknown-unknown0"});
+  runtimes2 = *cache_.Lookup({.target = "aarch64-unknown-unknown-fresh2"});
+
+  // The first and last should have been preserved as they were not stale.
+  EXPECT_THAT(runtimes1.base_dir().Stat()->unix_inode(), Eq(runtimes1_inode));
+  EXPECT_THAT(runtimes2.base_dir().Stat()->unix_inode(), Eq(runtimes2_inode));
+
+  // One of the stale runtimes should be freshly created though.
+  EXPECT_THAT(stale_runtimes_0.base_dir().Stat()->unix_inode(),
+              Ne(stale_runtimes_0_inode));
+}
+
+TEST_F(RuntimesCacheTest, LookupWithTooManyRuntimes) {
+  auto runtimes1 = *cache_.Lookup({.target = "aarch64-unknown-unknown-fresh1"});
+  auto runtimes2 = *cache_.Lookup({.target = "aarch64-unknown-unknown-fresh2"});
+  int n = RuntimesTestPeer::CacheMaxNumEntries();
+  auto stale_runtimes = LookupNRuntimes(n);
+
+  // Compute stale target strings.
+  auto stale_runtimes_n_1_target =
+      llvm::formatv("aarch64-unknown-unknown{0}", n - 1).str();
+  auto stale_runtimes_n_2_target =
+      llvm::formatv("aarch64-unknown-unknown{0}", n - 2).str();
+
+  // Get the Unix-like inode of the directories so we can check whether
+  // subsequent lookups create a new directory.
+  auto runtimes1_inode = runtimes1.base_dir().Stat()->unix_inode();
+  auto runtimes2_inode = runtimes2.base_dir().Stat()->unix_inode();
+  auto stale_runtimes_0_inode =
+      stale_runtimes[0].base_dir().Stat()->unix_inode();
+  auto stale_runtimes_n_inode =
+      stale_runtimes.back().base_dir().Stat()->unix_inode();
+  auto stale_runtimes_n_1_inode =
+      std::prev(stale_runtimes.end(), 2)->base_dir().Stat()->unix_inode();
+
+  // Now manually set all the timestamps. We do this manually to avoid any
+  // reliance on the clock behavior or the amount of time passing between lookup
+  // calls.
+  auto now = Filesystem::Clock::now();
+  runtimes1.base_dir().UpdateTimes(".lock", now).Check();
+  runtimes2.base_dir().UpdateTimes(".lock", now).Check();
+  // Now set the stale runtimes to times further and further in the past.
+  now -= std::chrono::milliseconds(1);
+  for (auto [i, stale_runtime] : llvm::enumerate(stale_runtimes)) {
+    stale_runtime.base_dir()
+        .UpdateTimes(".lock", now - std::chrono::milliseconds(i * i))
+        .Check();
+  }
+
+  // Close most of the runtimes to release the locks, but keep the oldest stale
+  // runtime locked along with a fresh one to exercise the locking path.
+  runtimes1 = {};
+  auto stale_runtime_n_orig = stale_runtimes.pop_back_val();
+  stale_runtimes.clear();
+
+  // Lookup a new runtime, potentially pruning stale ones.
+  auto runtimes3 = *cache_.Lookup({.target = "aarch64-unknown-unknown-fresh3"});
+
+  // Re-lookup three of the original runtimes.
+  runtimes1 = *cache_.Lookup({.target = "aarch64-unknown-unknown-fresh1"});
+  runtimes2 = *cache_.Lookup({.target = "aarch64-unknown-unknown-fresh2"});
+  auto stale_runtimes_0 =
+      *cache_.Lookup({.target = "aarch64-unknown-unknown0"});
+  auto stale_runtimes_n = *cache_.Lookup({.target = stale_runtimes_n_1_target});
+  auto stale_runtimes_n_1 =
+      *cache_.Lookup({.target = stale_runtimes_n_2_target});
+
+  // The fresh runtimes should be preserved.
+  EXPECT_THAT(runtimes1.base_dir().Stat()->unix_inode(), Eq(runtimes1_inode));
+  EXPECT_THAT(runtimes2.base_dir().Stat()->unix_inode(), Eq(runtimes2_inode));
+  EXPECT_THAT(stale_runtimes_0.base_dir().Stat()->unix_inode(),
+              Eq(stale_runtimes_0_inode));
+
+  // THe last stale runtime should have been locked and so should remain.
+  EXPECT_THAT(stale_runtimes_n.base_dir().Stat()->unix_inode(),
+              Eq(stale_runtimes_n_inode));
+
+  // The next to last should have been pruned and re-created though.
+  EXPECT_THAT(stale_runtimes_n.base_dir().Stat()->unix_inode(),
+              Ne(stale_runtimes_n_1_inode));
+}
+
+}  // namespace
+}  // namespace Carbon

toolchain/driver/testdata/fail_bad_prebuilt_runtimes.carbon

@@ -0,0 +1,14 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// ARGS: --include-diagnostic-kind --prebuilt-runtimes=/a/b/c/does/not/exist compile --output=%t %s
+//
+// Diagnostic contains non-portable filesystem error text.
+// NOAUTOUPDATE
+// TIP: To test this file alone, run:
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/driver/testdata/fail_bad_prebuilt_runtimes.carbon
+// TIP: To dump output, run:
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/driver/testdata/fail_bad_prebuilt_runtimes.carbon
+// CHECK:STDERR: error: Calling `openat` on '/a/b/c/does/not/exist' relative to 'AT_FDCWD' during DirRef::OpenDir failed: {{.*}} [DriverPrebuiltRuntimesInvalid]
+// CHECK:STDERR:

toolchain/driver/testdata/fail_bad_runtimes_cache.carbon

@@ -0,0 +1,14 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// ARGS: --include-diagnostic-kind --runtimes-cache=/a/b/c/does/not/exist compile --output=%t %s
+//
+// Diagnostic contains non-portable filesystem error text.
+// NOAUTOUPDATE
+// TIP: To test this file alone, run:
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/driver/testdata/fail_bad_runtimes_cache.carbon
+// TIP: To dump output, run:
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/driver/testdata/fail_bad_runtimes_cache.carbon
+// CHECK:STDERR: error: Calling `openat` on '/a/b/c/does/not/exist' relative to 'AT_FDCWD' during DirRef::OpenDir failed: {{.*}} [DriverRuntimesCacheInvalid]
+// CHECK:STDERR:

toolchain/install/BUILD

@@ -181,6 +181,13 @@ install_dirs = {
     ],
     "lib/carbon": [
         install_target("carbon_install.txt", "carbon_install.txt"),
+        install_target(
+            "install_digest.txt",
+            ":install_digest.txt",
+            executable = False,
+            is_digest = True,
+            is_driver = True,
+        ),
         install_target(
             "carbon-busybox",
             ":carbon-busybox",
@@ -210,6 +217,7 @@ install_dirs = {
 make_install_filegroups(
     name = "install_data",
     install_dirs = install_dirs,
+    no_digest_name = "install_data.no_digest",
     no_driver_name = "install_data.no_driver",
     pkg_name = "pkg_data",
     prefix = "prefix_root",
@@ -227,6 +235,39 @@ manifest(
     srcs = [":install_data"],
 )
 
+cc_binary(
+    name = "make-installation-digest",
+    srcs = ["make_installation_digest.cpp"],
+    deps = [
+        ":install_paths",
+        "//common:bazel_working_dir",
+        "//common:error",
+        "//common:exe_path",
+        "//common:filesystem",
+        "//common:init_llvm",
+        "//common:map",
+        "//common:vlog",
+        "@llvm-project//llvm:Support",
+    ],
+)
+
+manifest(
+    name = "install_digest_manifest.txt",
+    srcs = [":install_data.no_digest"],
+)
+
+genrule(
+    name = "gen_digest",
+    srcs = [
+        ":install_data.no_digest",
+        "install_digest_manifest.txt",
+    ],
+    outs = [":install_digest.txt"],
+    cmd = "$(location :make-installation-digest) " +
+          "$(location install_digest_manifest.txt) $@",
+    tools = [":make-installation-digest"],
+)
+
 # A list of clang's installed builtin header files.
 # This is consumed by //toolchain/testing:file_test.
 manifest(

toolchain/install/install_filegroups.bzl

@@ -22,6 +22,7 @@ def install_filegroup(name, filegroup_target, remove_prefix = "", label = None):
     """
     return {
         "filegroup": filegroup_target,
+        "is_digest": False,
         "is_driver": False,
         "label": label,
         "name": name,
@@ -40,12 +41,13 @@ def install_symlink(name, symlink_to, is_driver = False):
         filegroup.
     """
     return {
+        "is_digest": False,
         "is_driver": is_driver,
         "name": name,
         "symlink": symlink_to,
     }
 
-def install_target(name, target, executable = False, is_driver = False):
+def install_target(name, target, executable = False, is_driver = False, is_digest = False):
     """Adds a target for install.
 
     Used in the `install_dirs` dict.
@@ -56,19 +58,24 @@ def install_target(name, target, executable = False, is_driver = False):
       executable: True if executable.
       is_driver: False if it should be included in the `no_driver_name`
         filegroup.
+      is_digest: False if it should be included in the `no_digest_name`
+        filegroup.
     """
     return {
         "executable": executable,
+        "is_digest": is_digest,
         "is_driver": is_driver,
         "name": name,
         "target": target,
     }
 
-def make_install_filegroups(name, no_driver_name, pkg_name, install_dirs, prefix):
+def make_install_filegroups(name, no_digest_name, no_driver_name, pkg_name, install_dirs, prefix):
     """Makes filegroups of install data.
 
     Args:
       name: The name of the main filegroup, that contains all install_data.
+      no_digest_name: The name of a filegroup which excludes the digest. This is
+        used to compute the digest itself.
       no_driver_name: The name of a filegroup which excludes the driver. This is
         for the driver to depend on and get other files, without a circular
         dependency.
@@ -79,6 +86,7 @@ def make_install_filegroups(name, no_driver_name, pkg_name, install_dirs, prefix
     """
     all_srcs = []
     no_driver_srcs = []
+    no_digest_srcs = []
     pkg_srcs = []
 
     for dir, entries in install_dirs.items():
@@ -89,6 +97,8 @@ def make_install_filegroups(name, no_driver_name, pkg_name, install_dirs, prefix
             all_srcs.append(prefixed_path)
             if not entry["is_driver"]:
                 no_driver_srcs.append(prefixed_path)
+            if not entry["is_digest"]:
+                no_digest_srcs.append(prefixed_path)
 
             pkg_label = entry.get("label") or path + ".pkg"
             pkg_srcs.append(pkg_label)
@@ -152,4 +162,5 @@ def make_install_filegroups(name, no_driver_name, pkg_name, install_dirs, prefix
                 fail("Unrecognized structure: {0}".format(entry))
     native.filegroup(name = name, srcs = all_srcs)
     native.filegroup(name = no_driver_name, srcs = no_driver_srcs)
+    native.filegroup(name = no_digest_name, srcs = no_digest_srcs)
     pkg_filegroup(name = pkg_name, srcs = pkg_srcs)

toolchain/install/install_paths.cpp

@@ -225,4 +225,9 @@ auto InstallPaths::llvm_runtime_srcs() const -> std::filesystem::path {
                    "/src";
 }
 
+auto InstallPaths::digest_path() const -> std::filesystem::path {
+  // TODO: Adjust this to work equally well on Windows.
+  return prefix_ / "lib/carbon/install_digest.txt";
+}
+
 }  // namespace Carbon

toolchain/install/install_paths.h

@@ -111,6 +111,11 @@ class InstallPaths {
   // The path to the root of LLVM runtime sources.
   auto llvm_runtime_srcs() const -> std::filesystem::path;
 
+  // The installation digest path.
+  //
+  // This file contains a digest of the installation.
+  auto digest_path() const -> std::filesystem::path;
+
  private:
   friend class InstallPathsTestPeer;
 

toolchain/install/make_installation_digest.cpp

@@ -0,0 +1,149 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <unistd.h>
+
+#include <cstdlib>
+#include <string>
+
+#include "common/bazel_working_dir.h"
+#include "common/error.h"
+#include "common/exe_path.h"
+#include "common/filesystem.h"
+#include "common/init_llvm.h"
+#include "common/map.h"
+#include "common/vlog.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/SHA256.h"
+
+namespace Carbon {
+namespace {
+
+// A class implementing our digest program.
+//
+// The program is started with a call to `Run`, and either returns an error or
+// an exit code for `main`. It has a very simple command line interface:
+// - An optional flag `--verbose` that must be the first argument if provided.
+// - A required positional argument of a manifest file of all the files in a
+//   Carbon installation that should be added to the digest.
+// - A required positional argument of an output file for the digest.
+//
+// The program reads the manifest of all the files in the Carbon installation,
+// and adds each of those files to a running cryptographic digest. Once
+// complete, it writes this cryptographic digest to the provided output digest
+// file.
+//
+// The exact digest format is unspecified, but should provide a strong guarantee
+// that changes to any of the files in the manifest of the install produce
+// different digests.
+class DigestProgram {
+ public:
+  auto Run(int argc, char** argv) -> ErrorOr<int>;
+
+ private:
+  auto ComputeFileDigest(Filesystem::ReadFileRef file)
+      -> std::array<uint8_t, 32>;
+
+  llvm::raw_ostream* vlog_stream_ = nullptr;
+  Map<uint64_t, std::array<uint8_t, 32>> file_digests_;
+};
+
+auto DigestProgram::Run(int argc, char** argv) -> ErrorOr<int> {
+  InitLLVM init_llvm(argc, argv);
+  SetWorkingDirForBazel();
+  llvm::SHA256 sha256;
+
+  // If the first argument is `--verbose`, enable verbose logging.
+  int num_args = 2;
+  if (argc > 1 && llvm::StringRef(argv[1]) == "--verbose") {
+    vlog_stream_ = &llvm::errs();
+    ++num_args;
+  }
+  // The last two arguments are required and positional.
+  if (argc <= num_args) {
+    return Error(
+        "Usage: make-installation-digest [--verbose] MANIFEST_FILE "
+        "OUTPUT_FILE");
+  }
+  std::filesystem::path manifest_path = argv[num_args - 1];
+  std::filesystem::path digest_path = argv[num_args];
+
+  CARBON_ASSIGN_OR_RETURN(std::string manifest,
+                          Filesystem::Cwd().ReadFileToString(manifest_path));
+  llvm::SmallVector<llvm::StringRef> manifest_lines;
+  llvm::StringRef(manifest).split(manifest_lines, '\n');
+
+  // Walk all the install data files in the manifest.
+  for (llvm::StringRef manifest_line : manifest_lines) {
+    if (manifest_line.empty()) {
+      continue;
+    }
+    // Compute the full path and installed path for each file. The installed
+    // path comes from the path components below the `prefix_root` component.
+    std::filesystem::path full_path = manifest_line.trim().str();
+    std::filesystem::path install_path;
+    bool append = false;
+    for (const auto& component : full_path) {
+      if (append) {
+        install_path /= component;
+        continue;
+      }
+
+      if (component == "prefix_root") {
+        append = true;
+      }
+    }
+
+    CARBON_VLOG("Digesting file: {0}\n", install_path);
+    // Add the install path itself to the digest to track the layout of the
+    // installation data.
+    sha256.update(install_path.native());
+
+    // Open the file and compute its digest to add as well. We use a memoizing
+    // helper here to avoid re-examining the same file even if there are
+    // multiple paths to reach that file.
+    CARBON_ASSIGN_OR_RETURN(
+        Filesystem::ReadFile file,
+        Filesystem::Cwd().OpenReadOnly(full_path, Filesystem::OpenExisting));
+    sha256.update(ComputeFileDigest(file));
+  }
+
+  auto digest = sha256.final();
+  CARBON_VLOG("Digest: {0}\n", llvm::toHex(digest));
+
+  CARBON_RETURN_IF_ERROR(Filesystem::Cwd().WriteFileFromString(
+      digest_path, llvm::toHex(digest, /*LowerCase=*/true) + "\n"));
+
+  return EXIT_SUCCESS;
+}
+
+auto DigestProgram::ComputeFileDigest(Filesystem::ReadFileRef file)
+    -> std::array<uint8_t, 32> {
+  // Filesystem errors are unlikely here, and the library will check them just
+  // in case.
+  Filesystem::FileStatus stat = *file.Stat();
+  auto result = file_digests_.Insert(stat.unix_inode(), [file]() mutable {
+    llvm::SHA256 sha256;
+    // TODO: We could do this more efficiently by using a fixed buffer.
+    sha256.update(*file.ReadFileToString());
+    return sha256.final();
+  });
+  return result.value();
+}
+
+}  // namespace
+}  // namespace Carbon
+
+auto main(int argc, char** argv) -> int {
+  Carbon::DigestProgram program;
+  auto result = program.Run(argc, argv);
+  if (result.ok()) {
+    return *result;
+  } else {
+    llvm::errs() << "error: " << result.error() << "\n";
+    return EXIT_FAILURE;
+  }
+}