Many improvements to the filesystem library (#6000)

This is a collection of improvements to the filesystem library motivated
by using it to build a runtimes cache. It adds several core features:

- Advisory file locking
- Renaming of entries
- Testing for things being open
- File timestamp querying and updating

It also makes several more minor improvements such as improving the
names of functions and making them work in a more predictable fashion.
For example, the functions to read and write an entire file to/from
strings now actually handle the entire file rather than potentially
composing with other reads or writes, and adding the word `File` to
their name makes that more clear. Similarly, directory reading is more
robust in the face of repeatedly reading the same directory, and several
convenience functions were added to handle common patterns of reading
directories.

There is also a small fix to `ostream` uncovered by the tests added
here.

---------

Co-authored-by: Dana Jansens <danakj@orodu.net>

common/filesystem.cpp

@@ -5,8 +5,11 @@
 #include "common/filesystem.h"
 
 #include <fcntl.h>
+#include <time.h>  // NOLINT(modernize-deprecated-headers)
 #include <unistd.h>
 
+#include <chrono>
+
 #include "common/build_data.h"
 #include "llvm/Support/MathExtras.h"
 
@@ -63,7 +66,8 @@ auto PathError::Print(llvm::raw_ostream& out) const -> void {
   PrintErrorNumber(out, unix_errnum());
 }
 
-auto Internal::FileRefBase::ReadToString() -> ErrorOr<std::string, FdError> {
+auto Internal::FileRefBase::ReadFileToString()
+    -> ErrorOr<std::string, FdError> {
   std::string result;
 
   // Read a buffer at a time until we reach the end. We use the pipe buffer
@@ -76,6 +80,7 @@ auto Internal::FileRefBase::ReadToString() -> ErrorOr<std::string, FdError> {
   // be any faster, but it will be much more friendly to callers with
   // constrained stack sizes and use less memory overall.
   std::byte buffer[PIPE_BUF];
+  CARBON_RETURN_IF_ERROR(SeekFromBeginning(0));
   for (;;) {
     auto read_result = ReadToBuffer(buffer);
     if (!read_result.ok()) {
@@ -92,8 +97,9 @@ auto Internal::FileRefBase::ReadToString() -> ErrorOr<std::string, FdError> {
   return result;
 }
 
-auto Internal::FileRefBase::WriteFromString(llvm::StringRef str)
+auto Internal::FileRefBase::WriteFileFromString(llvm::StringRef str)
     -> ErrorOr<Success, FdError> {
+  CARBON_RETURN_IF_ERROR(SeekFromBeginning(0));
   auto bytes = llvm::ArrayRef<std::byte>(
       reinterpret_cast<const std::byte*>(str.data()), str.size());
   while (!bytes.empty()) {
@@ -103,6 +109,191 @@ auto Internal::FileRefBase::WriteFromString(llvm::StringRef str)
     }
     bytes = *write_result;
   }
+  CARBON_RETURN_IF_ERROR(Truncate(str.size()));
+  return Success();
+}
+
+// A macOS specific sleep routine that builds on more standard utilities. This
+// is technically a portable implementation so we always compile it but only use
+// it on macOS where the more efficient direct use of `clock_nanosleep` isn't
+// available.
+[[maybe_unused]]
+static auto SleepMacos(Duration sleep) -> void {
+  TimePoint stop = Clock::now() + sleep;
+
+  timespec sleep_ts = Internal::DurationToTimespec(sleep);
+  for (;;) {
+    timespec rem_sleep_ts = {};
+    int result = nanosleep(&sleep_ts, &rem_sleep_ts);
+    if (result == 0) {
+      return;
+    }
+
+    // Continue sleeping if we get interrupted by a resumable signal. For
+    // everything else report it.
+    if (errno != EINTR) {
+      int errnum = errno;
+      RawStringOstream error_os;
+      PrintErrorNumber(error_os, errnum);
+      CARBON_FATAL("Unexpected error while sleeping: {0}", error_os.TakeStr());
+    }
+
+    // Update to the remaining sleep time for the next attempt at sleeping.
+    sleep_ts = rem_sleep_ts;
+
+    // Also check if the clock has passed our stop time as a fallback to avoid
+    // too much clock skew.
+    if (Clock::now() > stop) {
+      return;
+    }
+  }
+}
+
+static auto Sleep(Duration sleep) -> void {
+  // For every platform but macOS we can sleep directly on an absolute time.
+#if __APPLE__
+  // On Apple platforms, dispatch to a specialized routine.
+  SleepMacos(sleep);
+#else
+
+  // We use `clock_gettime` instead of the filesystem `Clock` or some other
+  // `std::chrono` clock because we want to use the exact same clock that we'll
+  // use for sleeping below, and we'll need the time in a `timespec` for that
+  // call anyways. We do use a monotonic clock to try and avoid sleeps being
+  // interrupted by clock changes.
+  timespec ts = {};
+  int result = clock_gettime(CLOCK_MONOTONIC, &ts);
+  CARBON_CHECK(result == 0, "Error getting the time: {0}", strerror(errno));
+
+  // Now convert the timespec to a duration that we can safely do arithmetic on.
+  // Since the sleep interval is in nanoseconds it is tempting to directly do
+  // arithmetic here, but this has a subtle pitfall near the boundary between
+  // the nanosecond component and the second component.
+  //
+  // Note that our `Duration` uses `__int128` to avoid worrying about running
+  // out of precision to represent the final deadline.
+  Duration stop_time = std::chrono::seconds(ts.tv_sec);
+  stop_time += std::chrono::nanoseconds(ts.tv_nsec);
+  stop_time += sleep;
+
+  // Now convert back to timespec.
+  ts = Internal::DurationToTimespec(stop_time);
+
+  do {
+    result = clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, &ts, nullptr);
+
+    // Continue sleeping if we get interrupted by a resumable signal. Because
+    // we're using a monotonic clock and an absolute deadline time we will
+    // eventually progress past that deadline.
+  } while (result != 0 && (errno == EINTR));
+
+  if (result != 0) {
+    int errnum = errno;
+    RawStringOstream error_os;
+    PrintErrorNumber(error_os, errnum);
+    CARBON_FATAL("Unexpected error while sleeping: {0}", error_os.TakeStr());
+  }
+#endif
+}
+
+auto Internal::FileRefBase::TryLock(FileLock::Kind kind, Duration deadline,
+                                    Duration poll_interval)
+    -> ErrorOr<FileLock, FdError> {
+  CARBON_CHECK(poll_interval <= deadline);
+  if (deadline != Duration(0) && poll_interval == Duration(0)) {
+    // If the caller didn't provide a poll interval but did provide a deadline,
+    // pick a poll interval to roughly be 1/1000th of the deadline but at least
+    // 1 microsecond. We don't support polling faster than 1 microsecond given
+    // how expensive file locking is.
+    poll_interval =
+        std::max(Duration(std::chrono::microseconds(1)), deadline / 1000);
+  }
+  if (deadline != Duration(0)) {
+    CARBON_CHECK(
+        deadline >= std::chrono::microseconds(10),
+        "A deadline for a file lock shorter than 10 microseconds is not "
+        "supported, callers can implement their own polling logic.");
+    CARBON_CHECK(poll_interval >= std::chrono::microseconds(1),
+                 "Polling for a file lock faster than every microsecond is not "
+                 "supported, callers can implement their own polling logic.");
+  }
+  auto stop = Clock::now() + deadline;
+  for (;;) {
+    int result = flock(fd_, static_cast<int>(kind) | LOCK_NB);
+    if (result == 0) {
+      return FileLock(fd_);
+    }
+
+    // Return an error if this is something other than blocking for the lock to
+    // be available, or we didn't get a deadline for continuing to try and
+    // acquire the lock, or we've reached our deadline.
+    if (errno != EWOULDBLOCK || deadline == Duration(0) ||
+        Clock::now() >= stop) {
+      return FdError(errno, "File::TryLock on '{0}'", fd_);
+    }
+
+    // The caller requested attempting to wait up to a deadline to acquire the
+    // lock with a specific poll interval. Try to sleep for that poll interval
+    // before trying the lock again.
+    Sleep(poll_interval);
+  }
+}
+
+auto DirRef::AppendEntriesIf(
+    llvm::SmallVectorImpl<std::filesystem::path>& entries,
+    llvm::function_ref<auto(llvm::StringRef name)->bool> predicate)
+    -> ErrorOr<Success, FdError> {
+  CARBON_ASSIGN_OR_RETURN(Reader reader, Read());
+  for (const Entry& entry : reader) {
+    llvm::StringRef name = entry.name();
+    if (name == "." || name == "..") {
+      continue;
+    }
+    if (predicate && !predicate(name)) {
+      continue;
+    }
+    entries.push_back(name.str());
+  }
+  return Success();
+}
+
+auto DirRef::AppendEntriesIf(
+    llvm::SmallVectorImpl<std::filesystem::path>& dir_entries,
+    llvm::SmallVectorImpl<std::filesystem::path>& non_dir_entries,
+    llvm::function_ref<auto(llvm::StringRef name)->bool> predicate)
+    -> ErrorOr<Success, FdError> {
+  CARBON_ASSIGN_OR_RETURN(Reader reader, Read());
+  for (const Entry& entry : reader) {
+    llvm::StringRef name = entry.name();
+    if (name == "." || name == "..") {
+      continue;
+    }
+    if (predicate && !predicate(name)) {
+      continue;
+    }
+    std::filesystem::path name_path = name.str();
+    if (entry.is_known_dir()) {
+      dir_entries.push_back(std::move(name_path));
+      continue;
+    }
+    if (!entry.is_unknown_type()) {
+      non_dir_entries.push_back(std::move(name_path));
+      continue;
+    }
+
+    auto stat_result = Lstat(name_path);
+    if (!stat_result.ok()) {
+      return FdError(stat_result.error().unix_errnum(),
+                     "Dir::AppendEntriesIf on '{0}' failed while stat-ing "
+                     "entries to determine which are directories",
+                     dfd_);
+    }
+    if (stat_result->is_dir()) {
+      dir_entries.push_back(std::move(name_path));
+    } else {
+      non_dir_entries.push_back(std::move(name_path));
+    }
+  }
   return Success();
 }
 
@@ -213,7 +404,7 @@ auto DirRef::OpenDir(const std::filesystem::path& path,
 auto DirRef::ReadFileToString(const std::filesystem::path& path)
     -> ErrorOr<std::string, PathError> {
   CARBON_ASSIGN_OR_RETURN(ReadFile f, OpenReadOnly(path));
-  auto result = f.ReadToString();
+  auto result = f.ReadFileToString();
   if (result.ok()) {
     return *std::move(result);
   }
@@ -227,14 +418,18 @@ auto DirRef::WriteFileFromString(const std::filesystem::path& path,
                                  CreationOptions creation_options)
     -> ErrorOr<Success, PathError> {
   CARBON_ASSIGN_OR_RETURN(WriteFile f, OpenWriteOnly(path, creation_options));
-  auto write_result = f.WriteFromString(content);
+  auto write_result = f.WriteFileFromString(content);
+  // Immediately close the file as even if there was a write error we don't want
+  // to leave the file open.
+  auto close_result = std::move(f).Close();
+
+  // Now report the first error encountered or return success.
   if (!write_result.ok()) {
     return PathError(
         write_result.error().unix_errnum(),
         "Write error in Dir::WriteFileFromString on '{0}' relative to '{1}'",
         path, dfd_);
   }
-  auto close_result = std::move(f).Close();
   if (!close_result.ok()) {
     return PathError(
         close_result.error().unix_errnum(),
@@ -499,6 +694,12 @@ auto MakeTmpDir() -> ErrorOr<RemovingDir, Error> {
 
   std::filesystem::path target = BuildData::BuildTarget.str();
   tmpdir_path /= target.filename();
+  return MakeTmpDirWithPrefix(std::move(tmpdir_path));
+}
+
+auto MakeTmpDirWithPrefix(std::filesystem::path prefix)
+    -> ErrorOr<RemovingDir, Error> {
+  std::filesystem::path tmpdir_path = std::move(prefix);
   tmpdir_path += ".XXXXXX";
 
   std::string tmpdir_path_buffer = tmpdir_path.native();

common/filesystem.h

@@ -7,10 +7,12 @@
 
 #include <dirent.h>
 #include <fcntl.h>
+#include <sys/file.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <unistd.h>
 
+#include <chrono>
 #include <concepts>
 #include <filesystem>
 #include <iterator>
@@ -189,6 +191,10 @@ enum class FileType : ModeType {
   UnixMask = S_IFMT,
 };
 
+using Clock = std::chrono::file_clock;
+using Duration = std::chrono::duration<__int128, std::nano>;
+using TimePoint = std::chrono::time_point<Clock, Duration>;
+
 // Enumerates the different open access modes available.
 //
 // These are largely used to parameterize types in order to constrain which API
@@ -227,6 +233,15 @@ consteval auto Cwd() -> Dir;
 // the caller's responsibility to clean up this directory.
 auto MakeTmpDir() -> ErrorOr<RemovingDir, Error>;
 
+// Creates a temporary directory and returns a removing directory handle to it.
+//
+// The path of the temporary directory will use the provided prefix, and will
+// not add any additional directory separators. Every component but the last in
+// the prefix path must exist and be a directory, the last directory must be
+// writable.
+auto MakeTmpDirWithPrefix(std::filesystem::path prefix)
+    -> ErrorOr<RemovingDir, Error>;
+
 // Class modeling a file (or directory) status information structure.
 //
 // This provides a largely-portable model that callers can use, as well as a few
@@ -250,6 +265,18 @@ class FileStatus {
   // the `ModeType` documentation for how to interpret the result.
   auto permissions() const -> ModeType { return stat_buf_.st_mode & 0777; }
 
+  auto mtime() const -> TimePoint {
+    // Linux, FreeBSD, and OpenBSD all use `st_mtim`, but macOS uses a different
+    // spelling.
+#if __APPLE__
+    timespec ts = stat_buf_.st_mtimespec;
+#else
+    timespec ts = stat_buf_.st_mtim;
+#endif
+    TimePoint t(std::chrono::seconds(ts.tv_sec));
+    return t + std::chrono::nanoseconds(ts.tv_nsec);
+  }
+
   // Non-portable APIs only available on Unix-like systems. See the
   // documentation of the Unix `stat` structure fields they expose for their
   // meaning.
@@ -265,6 +292,43 @@ class FileStatus {
   struct stat stat_buf_ = {};
 };
 
+// Models a held lock on a file or directory.
+//
+// Can model either a shared or exclusive lock with respect to the file, but the
+// held lock is unique.
+//
+// Must be released prior to the underlying file or directory being closed as it
+// contains a non-owning reference to that underlying file.
+class FileLock {
+ public:
+  enum class Kind {
+    Shared = LOCK_SH,
+    Exclusive = LOCK_EX,
+  };
+  using enum Kind;
+
+  FileLock() = default;
+  FileLock(FileLock&& arg) noexcept : fd_(std::exchange(arg.fd_, -1)) {}
+  auto operator=(FileLock&& arg) noexcept -> FileLock& {
+    Destroy();
+    fd_ = std::exchange(arg.fd_, -1);
+    return *this;
+  }
+  ~FileLock() { Destroy(); }
+
+  // Returns true if the lock is currently held.
+  auto is_locked() const -> bool { return fd_ != -1; }
+
+ private:
+  friend Internal::FileRefBase;
+
+  explicit FileLock(int fd) : fd_(fd) {}
+
+  auto Destroy() -> void;
+
+  int fd_ = -1;
+};
+
 // The base class defining the core `File` API.
 //
 // While not used directly, this is the base class used to implement all of the
@@ -283,11 +347,22 @@ class Internal::FileRefBase {
   // handle in that case. This is to support rebinding operations.
   FileRefBase() = default;
 
+  // Returns true if this refers to a valid open file, and false otherwise.
+  auto is_valid() const -> bool { return fd_ != -1; }
+
   // Reads the file status.
   //
   // Analogous to the Unix-like `fstat` call.
   auto Stat() -> ErrorOr<FileStatus, FdError>;
 
+  // Updates the access and modification times for the open file.
+  //
+  // If no explicit `time_point` is provided, sets both times to the current
+  // time. If an explicit `time_point` is provided, the times are updated to
+  // that time.
+  auto UpdateTimes(std::optional<TimePoint> time_point = std::nullopt)
+      -> ErrorOr<Success, FdError>;
+
   // Methods to seek the current file position, with various semantics for the
   // offset.
   auto Seek(int64_t delta) -> ErrorOr<int64_t, FdError>;
@@ -295,6 +370,13 @@ class Internal::FileRefBase {
       -> ErrorOr<int64_t, FdError>;
   auto SeekFromEnd(int64_t delta_from_end) -> ErrorOr<int64_t, FdError>;
 
+  // Truncates or extends the size of the file to the provided size.
+  //
+  // If the new size is smaller, all bytes beyond this size will be unavailable.
+  // If the new size is larger, the file will be zero-filled to the new size.
+  // The position of reads and writes does not change.
+  auto Truncate(int64_t new_size) -> ErrorOr<Success, FdError>;
+
   // Reads as much data as is available and fits into the provided buffer.
   //
   // On success, this returns a new slice from the start to the end of the
@@ -330,21 +412,50 @@ class Internal::FileRefBase {
   // which remains owned by the owning `File` object.
   auto WriteStream() -> llvm::raw_fd_ostream;
 
-  // Reads the file until EOF into the returned string.
+  // Reads the file from its start until EOF into the returned string.
   //
   // This method will retry any recoverable errors and work to completely read
-  // the file contents up to first encountering EOF.
+  // the file contents from its beginning up to first encountering EOF. This
+  // will seek the file to ensure it starts at the beginning regardless of
+  // previous read or write operations.
   //
   // Any non-recoverable errors are returned to the caller.
-  auto ReadToString() -> ErrorOr<std::string, FdError>;
+  auto ReadFileToString() -> ErrorOr<std::string, FdError>;
 
-  // Writes a string into the file starting from the current position.
+  // Writes the provided string to the file from the start and truncating to the
+  // written size.
   //
   // This method will retry any recoverable errors and work to completely write
-  // the provided content into the file.
+  // the provided content into the file from its beginning, and truncate the
+  // file's size to the provided string size. Essentially, this function
+  // replaces the file contents with the string's contents.
   //
   // Any non-recoverable errors are returned to the caller.
-  auto WriteFromString(llvm::StringRef str) -> ErrorOr<Success, FdError>;
+  auto WriteFileFromString(llvm::StringRef str) -> ErrorOr<Success, FdError>;
+
+  // Attempt to acquire an advisory shared lock on this directory.
+  //
+  // This is always done as a non-blocking operation, as blocking on advisory
+  // locks without a deadline can easily result in build systems essentially
+  // "fork-bombing" a machine. However, a `deadline` duration can be provided
+  // and this routine will block and attempt to acquire the requested lock for a
+  // bounded amount of time approximately based on that duration. Further, a
+  // `poll_interval` can be provided to control how quickly the lock will be
+  // polled during that duration. There is no scaling of the poll intervals at
+  // this layer, if a back-off heuristic is desired the caller should manage the
+  // polling themselves. The goal is to allow simple cases of spurious failures
+  // to be easily handled without unbounded blocking calls. Typically, callers
+  // should use a duration that is a reasonable upper bound on the latency to
+  // begin the locked operation and a poll interval that is a reasonably low
+  // median latency to begin the operation as 1-2 polls is expected to be
+  // common. These should not be set anywhere near the cost of acquiring a file
+  // lock, and in general file locks should only be used for expensive
+  // operations where it is worth significant delays to avoid duplicate work.
+  //
+  // If the lock cannot be acquired, the most recent lock-attempt error is
+  // returned.
+  auto TryLock(FileLock::Kind kind, Duration deadline = {},
+               Duration poll_interval = {}) -> ErrorOr<FileLock, FdError>;
 
  protected:
   explicit FileRefBase(int fd) : fd_(fd) {}
@@ -417,6 +528,8 @@ class FileRef : public Internal::FileRefBase {
   // Read and Write methods that delegate to the `FileRefBase` implementations,
   // but require the relevant access. See the methods on `FileRefBase` for full
   // documentation.
+  auto Truncate(int64_t new_size) -> ErrorOr<Success, FdError>
+    requires Writeable;
   auto ReadToBuffer(llvm::MutableArrayRef<std::byte> buffer)
       -> ErrorOr<llvm::MutableArrayRef<std::byte>, FdError>
     requires Readable;
@@ -425,9 +538,9 @@ class FileRef : public Internal::FileRefBase {
     requires Writeable;
   auto WriteStream() -> llvm::raw_fd_ostream
     requires Writeable;
-  auto ReadToString() -> ErrorOr<std::string, FdError>
+  auto ReadFileToString() -> ErrorOr<std::string, FdError>
     requires Readable;
-  auto WriteFromString(llvm::StringRef str) -> ErrorOr<Success, FdError>
+  auto WriteFileFromString(llvm::StringRef str) -> ErrorOr<Success, FdError>
     requires Writeable;
 
  protected:
@@ -540,18 +653,60 @@ class DirRef {
   class Iterator;
   class Reader;
 
+  constexpr DirRef() = default;
+
+  // Returns true if this refers to a valid open directory, and false otherwise.
+  auto is_valid() const -> bool { return dfd_ != -1; }
+
   // Begin reading the entries in a directory.
   //
   // This returns a `Reader` object that can be iterated to walk over all the
   // entries in this directory. Note that the returned `Reader` owns a newly
   // allocated handle to this directory, and provides the full `DirRef` API. If
   // it isn't necessary to keep both open, the `Dir` class offers a
-  // move-qualified overload that optimizes this case.
+  // move-qualified method `TakeAndRead` that optimizes this case.
   //
   // Note that it is unspecified whether added and removed files during the
   // lifetime of the reader will be included when iterating, but otherwise
   // concurrent mutations are well defined.
-  auto Read() & -> ErrorOr<Reader, FdError>;
+  auto Read() -> ErrorOr<Reader, FdError>;
+
+  // Reads all of the entries in this directory into a vector.
+  //
+  // A helper function wrapping `Read` and walking the resulting reader's
+  // entries to produce a container.
+  //
+  // For details on errors, see the documentation of `Read` and `Reader`.
+  auto ReadEntries()
+      -> ErrorOr<llvm::SmallVector<std::filesystem::path>, FdError>;
+
+  // Reads the directory and appends entries to a container if they pass a
+  // predicate. The predicate can be null, which is treated as if it always
+  // returns true.
+  //
+  // For details on errors, see the documentation of `Read` and `Reader`.
+  auto AppendEntriesIf(
+      llvm::SmallVectorImpl<std::filesystem::path>& entries,
+      llvm::function_ref<auto(llvm::StringRef name)->bool> predicate = {})
+      -> ErrorOr<Success, FdError>;
+
+  // Reads the directory and appends entries to one of two containers if they
+  // pass a predicate.  The predicate can be null, which is treated as if it
+  // always returns true.
+  //
+  // Which container an entry is appended to depends on its kind -- directories
+  // go to the first and non-directories go to the second. This turns out to be
+  // a very common split with subdirectories often needing separate handling
+  // from other entries.
+  //
+  // For details on errors, see the documentation of `Read` and `Reader`. This
+  // may also `Stat` entries if necessary to determine whether they are
+  // directories.
+  auto AppendEntriesIf(
+      llvm::SmallVectorImpl<std::filesystem::path>& dir_entries,
+      llvm::SmallVectorImpl<std::filesystem::path>& non_dir_entries,
+      llvm::function_ref<auto(llvm::StringRef name)->bool> predicate = {})
+      -> ErrorOr<Success, FdError>;
 
   // Checks that the provided path can be accessed.
   auto Access(const std::filesystem::path& path,
@@ -575,6 +730,15 @@ class DirRef {
   auto Lstat(const std::filesystem::path& path)
       -> ErrorOr<FileStatus, PathError>;
 
+  // Updates the access and modification times for the provided path.
+  //
+  // If no explicit `time_point` is provided, sets both times to the current
+  // time. If an explicit `time_point` is provided, the times are updated to
+  // that time.
+  auto UpdateTimes(const std::filesystem::path& path,
+                   std::optional<TimePoint> time_point = std::nullopt)
+      -> ErrorOr<Success, PathError>;
+
   // Reads the target string of the symlink at the provided path.
   //
   // This does not follow the symlink, and does not require the symlink target
@@ -682,7 +846,16 @@ class DirRef {
                            CreationOptions creation_options = CreateAlways)
       -> ErrorOr<Success, PathError>;
 
-  // Moves a file from one directory to another directory.
+  // Renames an entry from one directory to another directory, replacing any
+  // existing entry with the target path.
+  //
+  // Note that this is *not* a general purpose move! It must be possible for
+  // this operation to be performed as a metadata-only change, and so without
+  // moving any actual data. This means it will not work across devices, mounts,
+  // or filesystems. However, these restrictions make this an *atomic* rename.
+  //
+  // The most common usage is to rename an entry within a single directory, by
+  // passing `*this` as `target_dir`.
   auto Rename(const std::filesystem::path& path, DirRef target_dir,
               const std::filesystem::path& target_path)
       -> ErrorOr<Success, PathError>;
@@ -753,7 +926,6 @@ class DirRef {
   auto Rmtree(const std::filesystem::path& path) -> ErrorOr<Success, PathError>;
 
  protected:
-  constexpr DirRef() = default;
   constexpr explicit DirRef(int dfd) : dfd_(dfd) {}
 
   // Slow-path fallback when unable to read the symlink target into a small
@@ -1022,8 +1194,10 @@ class ErrnoErrorBase : public ErrorBase<ErrorT> {
   auto no_entity() const -> bool { return errnum_ == ENOENT; }
   auto not_dir() const -> bool { return errnum_ == ENOTDIR; }
   auto access_denied() const -> bool { return errnum_ == EACCES; }
+  auto would_block() const -> bool { return errnum_ == EWOULDBLOCK; }
 
-  // Specific to `Rmdir` operations, two different error values can be used.
+  // Specific to `Rmdir` and `Rename` operations that remove a directory name,
+  // two different error values can be used.
   auto not_empty() const -> bool {
     return errnum_ == ENOTEMPTY || errnum_ == EEXIST;
   }
@@ -1073,6 +1247,7 @@ class FdError : public Internal::ErrnoErrorBase<FdError> {
   auto Print(llvm::raw_ostream& out) const -> void;
 
  private:
+  friend FileLock;
   friend Internal::FileRefBase;
   friend ReadFile;
   friend WriteFile;
@@ -1136,8 +1311,37 @@ class PathError : public Internal::ErrnoErrorBase<PathError> {
 
 // Implementation details only below.
 
+namespace Internal {
+
+inline auto DurationToTimespec(Duration d) -> timespec {
+  timespec ts = {};
+  ts.tv_sec = std::chrono::duration_cast<std::chrono::seconds>(d).count();
+  d -= std::chrono::seconds(ts.tv_sec);
+  ts.tv_nsec = std::chrono::duration_cast<std::chrono::nanoseconds>(d).count();
+  return ts;
+}
+
+}  // namespace Internal
+
 consteval auto Cwd() -> Dir { return Dir(AT_FDCWD); }
 
+inline auto FileLock::Destroy() -> void {
+  if (fd_ == -1) {
+    // Nothing to unlock.
+    return;
+  }
+
+  // We always try to unlock in a non-blocking way as there should never be a
+  // reason to block here.
+  int result = flock(fd_, LOCK_UN | LOCK_NB);
+
+  // The only realistic error is `EBADF` that would represent a programming
+  // error the type system should prevent. We conservatively check-fail if an
+  // error occurs here.
+  CARBON_CHECK(result == 0, "{0}",
+               FdError(errno, "Unexpected error while _unlocking_ '{0}'", fd_));
+}
+
 inline auto Internal::FileRefBase::Stat() -> ErrorOr<FileStatus, FdError> {
   FileStatus status;
   if (fstat(fd_, &status.stat_buf_) == 0) {
@@ -1146,6 +1350,24 @@ inline auto Internal::FileRefBase::Stat() -> ErrorOr<FileStatus, FdError> {
   return FdError(errno, "File::Stat on '{0}'", fd_);
 }
 
+inline auto Internal::FileRefBase::UpdateTimes(
+    std::optional<TimePoint> time_point) -> ErrorOr<Success, FdError> {
+  if (!time_point) {
+    if (futimens(fd_, nullptr) == -1) {
+      return FdError(errno, "File::UpdateTimes to now on '{0}'", fd_);
+    }
+    return Success();
+  }
+
+  timespec times[2];
+  times[0] = Internal::DurationToTimespec(time_point->time_since_epoch());
+  times[1] = times[0];
+  if (futimens(fd_, times) == -1) {
+    return FdError(errno, "File::UpdateTimes to a specific time on '{0}'", fd_);
+  }
+  return Success();
+}
+
 inline auto Internal::FileRefBase::Seek(int64_t delta)
     -> ErrorOr<int64_t, FdError> {
   int64_t byte_offset = lseek(fd_, delta, SEEK_CUR);
@@ -1173,6 +1395,15 @@ inline auto Internal::FileRefBase::SeekFromEnd(int64_t delta_from_end)
   return byte_offset;
 }
 
+inline auto Internal::FileRefBase::Truncate(int64_t new_size)
+    -> ErrorOr<Success, FdError> {
+  int64_t result = ftruncate(fd_, new_size);
+  if (result == -1) {
+    return FdError(errno, "File::Truncate on '{0}'", fd_);
+  }
+  return Success();
+}
+
 inline auto Internal::FileRefBase::ReadToBuffer(
     llvm::MutableArrayRef<std::byte> buffer)
     -> ErrorOr<llvm::MutableArrayRef<std::byte>, FdError> {
@@ -1237,6 +1468,13 @@ inline auto Internal::FileRefBase::WriteableDestroy() -> void {
       "calling `Close` and handling any errors to avoid data loss.");
 }
 
+template <OpenAccess A>
+auto FileRef<A>::Truncate(int64_t new_size) -> ErrorOr<Success, FdError>
+  requires Writeable
+{
+  return FileRefBase::Truncate(new_size);
+}
+
 template <OpenAccess A>
 auto FileRef<A>::ReadToBuffer(llvm::MutableArrayRef<std::byte> buffer)
     -> ErrorOr<llvm::MutableArrayRef<std::byte>, FdError>
@@ -1246,10 +1484,10 @@ auto FileRef<A>::ReadToBuffer(llvm::MutableArrayRef<std::byte> buffer)
 }
 
 template <OpenAccess A>
-auto FileRef<A>::ReadToString() -> ErrorOr<std::string, FdError>
+auto FileRef<A>::ReadFileToString() -> ErrorOr<std::string, FdError>
   requires Readable
 {
-  return FileRefBase::ReadToString();
+  return FileRefBase::ReadFileToString();
 }
 
 template <OpenAccess A>
@@ -1268,11 +1506,11 @@ auto FileRef<A>::WriteStream() -> llvm::raw_fd_ostream
 }
 
 template <OpenAccess A>
-auto FileRef<A>::WriteFromString(llvm::StringRef str)
+auto FileRef<A>::WriteFileFromString(llvm::StringRef str)
     -> ErrorOr<Success, FdError>
   requires Writeable
 {
-  return FileRefBase::WriteFromString(str);
+  return FileRefBase::WriteFileFromString(str);
 }
 
 template <OpenAccess A>
@@ -1284,7 +1522,7 @@ auto File<A>::Destroy() -> void {
   }
 }
 
-inline auto DirRef::Read() & -> ErrorOr<Reader, FdError> {
+inline auto DirRef::Read() -> ErrorOr<Reader, FdError> {
   int dup_dfd = dup(dfd_);
   if (dup_dfd == -1) {
     // There are very few plausible errors here, but we can return one so it
@@ -1296,6 +1534,13 @@ inline auto DirRef::Read() & -> ErrorOr<Reader, FdError> {
   return Dir(dup_dfd).TakeAndRead();
 }
 
+inline auto DirRef::ReadEntries()
+    -> ErrorOr<llvm::SmallVector<std::filesystem::path>, FdError> {
+  llvm::SmallVector<std::filesystem::path> entries;
+  CARBON_RETURN_IF_ERROR(AppendEntriesIf(entries));
+  return entries;
+}
+
 inline auto DirRef::Access(const std::filesystem::path& path,
                            AccessCheckFlags check) -> ErrorOr<bool, PathError> {
   if (faccessat(dfd_, path.c_str(), static_cast<int>(check), /*flags=*/0) ==
@@ -1332,6 +1577,29 @@ inline auto DirRef::Lstat(const std::filesystem::path& path)
   return PathError(errno, "Dir::Lstat on '{0}' relative to '{1}'", path, dfd_);
 }
 
+inline auto DirRef::UpdateTimes(const std::filesystem::path& path,
+                                std::optional<TimePoint> time_point)
+    -> ErrorOr<Success, PathError> {
+  if (!time_point) {
+    if (utimensat(dfd_, path.c_str(), nullptr, /*flags*/ 0) == -1) {
+      return PathError(errno,
+                       "Dir::UpdateTimes to now on '{0}' relative to '{1}'",
+                       path, dfd_);
+    }
+    return Success();
+  }
+
+  timespec times[2];
+  times[0] = Internal::DurationToTimespec(time_point->time_since_epoch());
+  times[1] = times[0];
+  if (utimensat(dfd_, path.c_str(), times, /*flags*/ 0) == -1) {
+    return PathError(
+        errno, "Dir::UpdateTimes to a specific time on '{0}' relative to '{1}'",
+        path, dfd_);
+  }
+  return Success();
+}
+
 inline auto DirRef::Readlink(const std::filesystem::path& path)
     -> ErrorOr<std::string, PathError> {
   // On the fast path, we read into a small stack buffer and get the whole
@@ -1547,7 +1815,11 @@ inline auto Dir::Iterator::operator++() -> Iterator& {
   return *this;
 }
 
-inline auto Dir::Reader::begin() -> Iterator { return Iterator(dirp_); }
+inline auto Dir::Reader::begin() -> Iterator {
+  // Reset the position of the directory stream to get the actual beginning.
+  rewinddir(dirp_);
+  return Iterator(dirp_);
+}
 
 inline auto Dir::Reader::end() -> Iterator { return Iterator(); }
 

common/filesystem_test.cpp

@@ -9,6 +9,7 @@
 
 #include <concepts>
 #include <string>
+#include <thread>
 #include <utility>
 
 #include "common/error_test_helpers.h"
@@ -21,6 +22,7 @@ using ::testing::Eq;
 using ::testing::HasSubstr;
 using Testing::IsError;
 using Testing::IsSuccess;
+using ::testing::UnorderedElementsAre;
 
 class FilesystemTest : public ::testing::Test {
  public:
@@ -101,7 +103,7 @@ TEST_F(FilesystemTest, BasicWriteAndRead) {
   {
     auto f = dir_.OpenWriteOnly("test", CreationOptions::CreateNew);
     ASSERT_THAT(f, IsSuccess(_));
-    auto write_result = f->WriteFromString(content_str);
+    auto write_result = f->WriteFileFromString(content_str);
     EXPECT_THAT(write_result, IsSuccess(_));
     (*std::move(f)).Close().Check();
   }
@@ -109,7 +111,7 @@ TEST_F(FilesystemTest, BasicWriteAndRead) {
   {
     auto f = dir_.OpenReadOnly("test");
     ASSERT_THAT(f, IsSuccess(_));
-    auto read_result = f->ReadToString();
+    auto read_result = f->ReadFileToString();
     EXPECT_THAT(read_result, IsSuccess(Eq(content_str)));
   }
 
@@ -117,6 +119,74 @@ TEST_F(FilesystemTest, BasicWriteAndRead) {
   EXPECT_THAT(unlink_result, IsSuccess(_));
 }
 
+TEST_F(FilesystemTest, SeekReadAndWrite) {
+  std::string content_str = "0123456789";
+  // First write some initial content.
+  {
+    auto f = dir_.OpenWriteOnly("test", CreationOptions::CreateNew);
+    ASSERT_THAT(f, IsSuccess(_));
+    auto write_result = f->WriteFileFromString(content_str);
+    EXPECT_THAT(write_result, IsSuccess(_));
+    (*std::move(f)).Close().Check();
+  }
+
+  // Now seek and read.
+  {
+    auto f = dir_.OpenReadOnly("test");
+    ASSERT_THAT(f, IsSuccess(_));
+    auto seek_result = f->Seek(3);
+    ASSERT_THAT(seek_result, IsSuccess(Eq(3)));
+    std::array<std::byte, 4> buffer;
+    auto read_result = f->ReadToBuffer(buffer);
+    ASSERT_THAT(read_result, IsSuccess(_));
+    EXPECT_THAT(std::string(reinterpret_cast<char*>(read_result->data()),
+                            read_result->size()),
+                Eq(content_str.substr(3, read_result->size())));
+
+    // Now test that we can seek back to the beginning and read the full file.
+    auto read_file_result = f->ReadFileToString();
+    EXPECT_THAT(read_file_result, IsSuccess(Eq(content_str)));
+  }
+
+  // Now a mixture of reads, writes, an seeking.
+  {
+    auto f = dir_.OpenReadWrite("test");
+    ASSERT_THAT(f, IsSuccess(_));
+    auto seek_result = f->SeekFromEnd(-6);
+    ASSERT_THAT(seek_result, IsSuccess(Eq(content_str.size() - 6)));
+    std::string new_content_str = "abcdefg";
+    llvm::ArrayRef<std::byte> new_content_bytes(
+        reinterpret_cast<std::byte*>(new_content_str.data()),
+        new_content_str.size());
+    for (auto write_bytes = new_content_bytes.slice(0, 4);
+         !write_bytes.empty();) {
+      auto write_result = f->WriteFromBuffer(write_bytes);
+      ASSERT_THAT(write_result, IsSuccess(_));
+      write_bytes = *write_result;
+    }
+
+    std::array<std::byte, 4> buffer;
+    auto read_result = f->ReadToBuffer(buffer);
+    ASSERT_THAT(read_result, IsSuccess(_));
+    EXPECT_THAT(std::string(reinterpret_cast<char*>(read_result->data()),
+                            read_result->size()),
+                Eq(content_str.substr(8, read_result->size())));
+
+    EXPECT_THAT(*f->ReadFileToString(), "0123abcd89");
+
+    // Now write the entire file, also changing its size, after a fresh seek.
+    seek_result = f->Seek(-6);
+    ASSERT_THAT(seek_result, IsSuccess(Eq(content_str.size() - 6)));
+    auto write_file_result = f->WriteFileFromString(new_content_str);
+    EXPECT_THAT(write_file_result, IsSuccess(_));
+    EXPECT_THAT(*f->ReadFileToString(), "abcdefg");
+    (*std::move(f)).Close().Check();
+  }
+
+  auto unlink_result = dir_.Unlink("test");
+  EXPECT_THAT(unlink_result, IsSuccess(_));
+}
+
 TEST_F(FilesystemTest, CreateAndRemoveDirecotries) {
   auto d1 = Cwd().CreateDirectories(path() / "a" / "b" / "c" / "test1");
   ASSERT_THAT(d1, IsSuccess(_));
@@ -185,7 +255,7 @@ TEST_F(FilesystemTest, StatAndAccess) {
   ModeType permissions = 0450;
   auto f = dir_.OpenWriteOnly("test", CreationOptions::CreateNew, permissions);
   ASSERT_THAT(f, IsSuccess(_));
-  auto write_result = f->WriteFromString(content_str);
+  auto write_result = f->WriteFileFromString(content_str);
   EXPECT_THAT(write_result, IsSuccess(_));
 
   access_result = dir_.Access("test");
@@ -301,7 +371,7 @@ TEST_F(FilesystemTest, Chdir) {
   // Create a regular file and try to chdir to that.
   auto f = dir_.OpenWriteOnly("test2", CreationOptions::CreateNew);
   ASSERT_THAT(f, IsSuccess(_));
-  auto write_result = f->WriteFromString("test2");
+  auto write_result = f->WriteFileFromString("test2");
   EXPECT_THAT(write_result, IsSuccess(_));
   chdir_path_result = dir_.Chdir("test2");
   ASSERT_FALSE(chdir_path_result.ok());
@@ -392,5 +462,157 @@ TEST_F(FilesystemTest, Rename) {
   EXPECT_THAT(result.error().unix_errnum(), EINVAL) << result.error();
 }
 
+TEST_F(FilesystemTest, TryLock) {
+  auto file = dir_.OpenReadWrite("test_file", CreateNew);
+  ASSERT_THAT(file, IsSuccess(_));
+
+  // Acquire an exclusive lock.
+  auto lock = file->TryLock(FileLock::Exclusive);
+  ASSERT_THAT(lock, IsSuccess(_));
+  EXPECT_TRUE(lock->is_locked());
+
+  // Try to acquire a second lock from a different file object.
+  auto file2 = dir_.OpenReadOnly("test_file");
+  ASSERT_THAT(file2, IsSuccess(_));
+  auto lock2 = file2->TryLock(FileLock::Exclusive);
+  ASSERT_THAT(lock2, IsError(_));
+  EXPECT_TRUE(lock2.error().would_block());
+
+  // A shared lock should also fail.
+  auto lock3 = file2->TryLock(FileLock::Shared);
+  ASSERT_THAT(lock3, IsError(_));
+  EXPECT_TRUE(lock3.error().would_block());
+
+  // Release the first lock.
+  *lock = {};
+  EXPECT_FALSE(lock->is_locked());
+
+  // Now we can acquire an exclusive lock.
+  lock2 = file2->TryLock(FileLock::Exclusive);
+  ASSERT_THAT(lock2, IsSuccess(_));
+  EXPECT_TRUE(lock2->is_locked());
+  *lock2 = {};
+
+  // Test shared locks.
+  auto shared_lock1 = file->TryLock(FileLock::Shared);
+  ASSERT_THAT(shared_lock1, IsSuccess(_));
+  EXPECT_TRUE(shared_lock1->is_locked());
+
+  auto shared_lock2 = file2->TryLock(FileLock::Shared);
+  ASSERT_THAT(shared_lock2, IsSuccess(_));
+  EXPECT_TRUE(shared_lock2->is_locked());
+
+  // An exclusive lock should fail.
+  auto file3 = dir_.OpenReadOnly("test_file");
+  ASSERT_THAT(file3, IsSuccess(_));
+  auto exclusive_lock = file3->TryLock(FileLock::Exclusive);
+  ASSERT_THAT(exclusive_lock, IsError(_));
+  EXPECT_TRUE(exclusive_lock.error().would_block());
+
+  // Release locks and close files.
+  *shared_lock1 = {};
+  *shared_lock2 = {};
+  ASSERT_THAT((*std::move(file)).Close(), IsSuccess(_));
+  ASSERT_THAT((*std::move(file2)).Close(), IsSuccess(_));
+  ASSERT_THAT((*std::move(file3)).Close(), IsSuccess(_));
+}
+
+TEST_F(FilesystemTest, ReadAndAppendEntries) {
+  // Test with an empty directory.
+  {
+    auto entries = dir_.ReadEntries();
+    ASSERT_THAT(entries, IsSuccess(_));
+    EXPECT_TRUE(entries->empty());
+  }
+  {
+    llvm::SmallVector<std::filesystem::path> entries;
+    EXPECT_THAT(dir_.AppendEntriesIf(entries), IsSuccess(_));
+    EXPECT_TRUE(entries.empty());
+  }
+
+  // Create some files and directories.
+  ASSERT_THAT(dir_.WriteFileFromString("file1", ""), IsSuccess(_));
+  ASSERT_THAT(dir_.WriteFileFromString("file2", ""), IsSuccess(_));
+  ASSERT_THAT(dir_.WriteFileFromString(".hidden", ""), IsSuccess(_));
+  ASSERT_THAT(dir_.CreateDirectories("subdir1"), IsSuccess(_));
+  ASSERT_THAT(dir_.CreateDirectories("subdir2"), IsSuccess(_));
+
+  // Test ReadEntries.
+  {
+    auto entries = dir_.ReadEntries();
+    ASSERT_THAT(entries, IsSuccess(_));
+    EXPECT_THAT(*entries, UnorderedElementsAre(".hidden", "file1", "file2",
+                                               "subdir1", "subdir2"));
+  }
+
+  // Test AppendEntriesIf with no predicate.
+  {
+    llvm::SmallVector<std::filesystem::path> entries;
+    EXPECT_THAT(dir_.AppendEntriesIf(entries), IsSuccess(_));
+    EXPECT_THAT(entries, UnorderedElementsAre(".hidden", "file1", "file2",
+                                              "subdir1", "subdir2"));
+  }
+
+  // Test AppendEntriesIf with a predicate.
+  {
+    llvm::SmallVector<std::filesystem::path> entries;
+    auto result = dir_.AppendEntriesIf(
+        entries, [](llvm::StringRef name) { return name.starts_with("file"); });
+    EXPECT_THAT(result, IsSuccess(_));
+    EXPECT_THAT(entries, UnorderedElementsAre("file1", "file2"));
+  }
+
+  // Test AppendEntriesIf with directory splitting and a predicate.
+  {
+    llvm::SmallVector<std::filesystem::path> dir_entries;
+    llvm::SmallVector<std::filesystem::path> non_dir_entries;
+    auto result = dir_.AppendEntriesIf(
+        dir_entries, non_dir_entries,
+        [](llvm::StringRef name) { return !name.starts_with("."); });
+    EXPECT_THAT(result, IsSuccess(_));
+    EXPECT_THAT(dir_entries, UnorderedElementsAre("subdir1", "subdir2"));
+    EXPECT_THAT(non_dir_entries, UnorderedElementsAre("file1", "file2"));
+  }
+}
+
+TEST_F(FilesystemTest, MtimeAndUpdateTimes) {
+  // Test UpdateTimes on a path that doesn't exist.
+  auto update_missing = dir_.UpdateTimes("test_file");
+  ASSERT_THAT(update_missing, IsError(_));
+  EXPECT_TRUE(update_missing.error().no_entity());
+
+  // Create a file and get its initial modification time.
+  ASSERT_THAT(dir_.WriteFileFromString("test_file", "content"), IsSuccess(_));
+  auto stat = dir_.Stat("test_file");
+  ASSERT_THAT(stat, IsSuccess(_));
+  auto time1 = stat->mtime();
+
+  // Repeated stats have stable time.
+  stat = dir_.Stat("test_file");
+  ASSERT_THAT(stat, IsSuccess(_));
+  EXPECT_THAT(stat->mtime(), Eq(time1));
+
+  // Update the timestamp to a specific time in the past.
+  auto past_time = time1 - std::chrono::seconds(120);
+  ASSERT_THAT(dir_.UpdateTimes("test_file", past_time), IsSuccess(_));
+  stat = dir_.Stat("test_file");
+  ASSERT_THAT(stat, IsSuccess(_));
+  EXPECT_THAT(stat->mtime(), Eq(past_time));
+
+  // Now test updating times on an open file. Should still be at `past_time`.
+  auto file = *dir_.OpenReadWrite("test_file");
+  auto file_stat = file.Stat();
+  ASSERT_THAT(file_stat, IsSuccess(_));
+  EXPECT_THAT(file_stat->mtime(), Eq(past_time));
+
+  // Update the times through the file and verify those updates arrived.
+  ASSERT_THAT(file.UpdateTimes(time1), IsSuccess(_));
+  file_stat = file.Stat();
+  ASSERT_THAT(file_stat, IsSuccess(_));
+  EXPECT_THAT(file_stat->mtime(), Eq(time1));
+
+  ASSERT_THAT(std::move(file).Close(), IsSuccess(_));
+}
+
 }  // namespace
 }  // namespace Carbon::Filesystem

common/ostream.h

@@ -109,7 +109,8 @@ namespace llvm {
 // locally instead.
 template <typename StreamT, typename ClassT>
   requires std::derived_from<std::decay_t<StreamT>, std::ostream> &&
-           (!std::same_as<std::decay_t<ClassT>, raw_ostream>)
+           (!std::same_as<std::decay_t<ClassT>, raw_ostream>) &&
+           requires(raw_ostream& os, const ClassT& value) { os << value; }
 auto operator<<(StreamT& standard_out, const ClassT& value) -> StreamT& {
   raw_os_ostream(standard_out) << value;
   return standard_out;