Add support for forcing a rebuild of runtimes (#6537)

This is useful during development, testing, and will also be useful for
a more bazel-integrated build step.

Also clean up the path management when creating runtimes:

- Teach the main runtimes code to handle making a relative path absolute
- Separate out methods for _creating_ a runtimes tree vs. opening an
existing one. Teach the creation path to create intervening directories
as needed. This provides a more useful and less surprising set of
behaviors.

Last but not least, also clean up a bunch of comments in the runtimes
cache code to talk generically about components -- these APIs are no
longer specific to the resource directory.

toolchain/driver/build_runtimes_subcommand.cpp

@@ -22,6 +22,17 @@ generation options.
       },
       [&](auto& arg_b) { arg_b.Set(&directory); });
 
+  b.AddFlag(
+      {
+          .name = "force",
+          .help = R"""(
+Force re-creating the provided output path from scratch
+
+This will **remove** the provided output path and re-create it from scratch.
+)""",
+      },
+      [&](CommandLine::FlagBuilder& arg_b) { arg_b.Set(&force); });
+
   codegen_options.Build(b);
 }
 
@@ -74,22 +85,19 @@ auto BuildRuntimesSubcommand::RunInternal(DriverEnv& driver_env)
       .target = options_.codegen_options.target.str()};
 
   bool is_cache = options_.directory.empty();
-  std::filesystem::path explicit_output_path = options_.directory.str();
-  if (!is_cache) {
-    auto access_result = Filesystem::Cwd().Access(explicit_output_path);
-    if (access_result.ok()) {
-      return Error("output directory already exists");
-    }
-    if (!access_result.error().no_entity()) {
-      return std::move(access_result).error();
-    }
-  }
+  std::filesystem::path output_path = options_.directory.str();
 
   CARBON_ASSIGN_OR_RETURN(
-      auto runtimes,
-      is_cache ? driver_env.runtimes_cache.Lookup(features)
-               : Runtimes::Make(explicit_output_path, driver_env.vlog_stream));
-  CARBON_ASSIGN_OR_RETURN(auto tmp_dir, Filesystem::MakeTmpDir());
+      auto runtimes, is_cache
+                         ? driver_env.runtimes_cache.Lookup(features)
+                         : Runtimes::Make(output_path, driver_env.vlog_stream));
+
+  if (options_.force) {
+    // Remove existing runtimes to force a rebuild.
+    CARBON_RETURN_IF_ERROR(runtimes.Remove(Runtimes::ClangResourceDir));
+    CARBON_RETURN_IF_ERROR(runtimes.Remove(Runtimes::LibUnwind));
+    CARBON_RETURN_IF_ERROR(runtimes.Remove(Runtimes::Libcxx));
+  }
 
   ClangResourceDirBuilder resource_dir_builder(&runner, driver_env.thread_pool,
                                                llvm::Triple(features.target),

toolchain/driver/build_runtimes_subcommand.h

@@ -22,6 +22,7 @@ struct BuildRuntimesOptions {
 
   CodegenOptions codegen_options;
   llvm::StringRef directory;
+  bool force;
 };
 
 // Implements the link subcommand of the driver.

toolchain/driver/driver.cpp

@@ -206,9 +206,8 @@ auto Driver::RunCommand(llvm::ArrayRef<llvm::StringRef> args) -> DriverResult {
   driver_env_.runtimes_cache = std::move(*cache_result);
 
   if (!options.prebuilt_runtimes_path.empty()) {
-    auto result = Runtimes::Make(
-        std::filesystem::absolute(options.prebuilt_runtimes_path.str()),
-        driver_env_.vlog_stream);
+    auto result = Runtimes::OpenExisting(options.prebuilt_runtimes_path.str(),
+                                         driver_env_.vlog_stream);
     if (!result.ok()) {
       // TODO: We should provide a better diagnostic than the raw error.
       CARBON_DIAGNOSTIC(DriverPrebuiltRuntimesInvalid, Error, "{0}",

toolchain/driver/runtimes_cache.cpp

@@ -28,11 +28,23 @@
 
 namespace Carbon {
 
-auto Runtimes::Make(std::filesystem::path path, llvm::raw_ostream* vlog_stream)
-    -> ErrorOr<Runtimes> {
+static auto MakeAbsolute(std::filesystem::path path)
+    -> ErrorOr<std::filesystem::path> {
   if (!path.is_absolute()) {
-    return Error("Runtimes require an absolute path");
+    std::error_code ec;
+    path = std::filesystem::absolute(path, ec);
+    if (ec) {
+      return Error(llvm::formatv("Unable to compute an absolute path: {0}",
+                                 ec.message()));
+    }
   }
+  return std::move(path);
+}
+
+auto Runtimes::OpenExisting(std::filesystem::path path,
+                            llvm::raw_ostream* vlog_stream)
+    -> ErrorOr<Runtimes> {
+  CARBON_ASSIGN_OR_RETURN(path, MakeAbsolute(std::move(path)));
 
   CARBON_ASSIGN_OR_RETURN(
       Filesystem::Dir dir,
@@ -40,6 +52,15 @@ auto Runtimes::Make(std::filesystem::path path, llvm::raw_ostream* vlog_stream)
   return Runtimes(std::move(path), std::move(dir), {}, {}, vlog_stream);
 }
 
+auto Runtimes::Make(std::filesystem::path path, llvm::raw_ostream* vlog_stream)
+    -> ErrorOr<Runtimes> {
+  CARBON_ASSIGN_OR_RETURN(path, MakeAbsolute(std::move(path)));
+
+  CARBON_ASSIGN_OR_RETURN(Filesystem::Dir dir,
+                          Filesystem::Cwd().CreateDirectories(path));
+  return Runtimes(std::move(path), std::move(dir), {}, {}, vlog_stream);
+}
+
 auto Runtimes::Destroy() -> void {
   // Release the lock on the runtimes and close the lock file.
   flock_ = {};
@@ -66,6 +87,11 @@ auto Runtimes::Build(Component component)
   return BuildImpl(component, BuildLockDeadline, BuildLockPollInterval);
 }
 
+auto Runtimes::Remove(Component component) -> ErrorOr<Success> {
+  CARBON_RETURN_IF_ERROR(base_dir_.Rmtree(ComponentPath(component)));
+  return Success();
+}
+
 auto Runtimes::BuildImpl(Component component, Filesystem::Duration deadline,
                          Filesystem::Duration poll_interval)
     -> ErrorOr<std::variant<std::filesystem::path, Builder>> {

toolchain/driver/runtimes_cache.h

@@ -52,9 +52,22 @@ class Runtimes {
     NumComponents,
   };
 
-  // Creates a `Runtimes` object for a specific existing directory.
+  // Creates a `Runtimes` object for a specific, existing directory.
   //
-  // Requires that the `path` is an absolute path that is an existing directory.
+  // If `path` is a relative path, it will be resolved to an absolute path
+  // relative to the current working directory.
+  //
+  // If there is not a directory at `path`, this will return an error.
+  static auto OpenExisting(std::filesystem::path path,
+                           llvm::raw_ostream* vlog_stream = nullptr)
+      -> ErrorOr<Runtimes>;
+
+  // Creates a `Runtimes` object for a specific directory.
+  //
+  // Opens or creates `path`, including any intervening directories needed.
+  //
+  // If `path` is a relative path, it will be resolved to an absolute path
+  // relative to the current working directory.
   static auto Make(std::filesystem::path path,
                    llvm::raw_ostream* vlog_stream = nullptr)
       -> ErrorOr<Runtimes>;
@@ -87,25 +100,29 @@ class Runtimes {
   // The base directory for the runtimes.
   auto base_dir() const -> Filesystem::DirRef { return base_dir_; }
 
-  // Gets the path to an _existing_ Clang resource directory.
-  //
-  // Clang's resource directory contains all of the compiler-builtin runtime
-  // libraries, headers, and data files.
-  //
-  // This will return the path to the Clang resource directory if it exists in
-  // the runtimes tree. Otherwise, it will return an error.
+  // Gets the path to an _existing_ built component subdirectory of the
+  // runtimes. If the component subdirectory doesn't exist, returns an error.
   auto Get(Component component) -> ErrorOr<std::filesystem::path>;
 
-  // Builds or returns a Clang resource directory.
+  // Builds or returns a component subdirectory of the runtimes.
   //
-  // If there is an existing, built Clang resource directory, this will return
-  // its path, the same as `GetExistingClangResourceDir` would. However, if
-  // there is not yet a Clang resource directory in this runtimes tree, returns
-  // a `Builder` object that can be used to build and commit a Clang resource
-  // directory to this runtimes tree.
+  // If there is an existing, built component subdirectory, this will return its
+  // path, the same as `Get` would. However, if there is not yet a built
+  // subdirectory in this runtimes tree, returns a `Builder` object that can be
+  // used to build and commit a component subdirectory to this runtimes tree.
   auto Build(Component component)
       -> ErrorOr<std::variant<std::filesystem::path, Builder>>;
 
+  // Removes a component subdirectory of the runtimes.
+  //
+  // This can be used to force a subsequent `Build` invocation to actually build
+  // the relevant component. It is a separate operation though because there is
+  // no way to atomically and reliably replace a component with a fresh build --
+  // there is always a chance that a racing build occurs instead. The separate
+  // API surfaces this, but is still useful in inherently non-racing cases like
+  // direct build requests for a given set of runtimes.
+  auto Remove(Component component) -> ErrorOr<Success>;
+
  private:
   friend Builder;
   friend Cache;