diff --git a/common/filesystem.cpp b/common/filesystem.cpp index afa40d2a0a4a..44a982668cc2 100644 --- a/common/filesystem.cpp +++ b/common/filesystem.cpp @@ -5,8 +5,11 @@ #include "common/filesystem.h" #include +#include // NOLINT(modernize-deprecated-headers) #include +#include + #include "common/build_data.h" #include "llvm/Support/MathExtras.h" @@ -63,7 +66,8 @@ auto PathError::Print(llvm::raw_ostream& out) const -> void { PrintErrorNumber(out, unix_errnum()); } -auto Internal::FileRefBase::ReadToString() -> ErrorOr { +auto Internal::FileRefBase::ReadFileToString() + -> ErrorOr { std::string result; // Read a buffer at a time until we reach the end. We use the pipe buffer @@ -76,6 +80,7 @@ auto Internal::FileRefBase::ReadToString() -> ErrorOr { // be any faster, but it will be much more friendly to callers with // constrained stack sizes and use less memory overall. std::byte buffer[PIPE_BUF]; + CARBON_RETURN_IF_ERROR(SeekFromBeginning(0)); for (;;) { auto read_result = ReadToBuffer(buffer); if (!read_result.ok()) { @@ -92,8 +97,9 @@ auto Internal::FileRefBase::ReadToString() -> ErrorOr { return result; } -auto Internal::FileRefBase::WriteFromString(llvm::StringRef str) +auto Internal::FileRefBase::WriteFileFromString(llvm::StringRef str) -> ErrorOr { + CARBON_RETURN_IF_ERROR(SeekFromBeginning(0)); auto bytes = llvm::ArrayRef( reinterpret_cast(str.data()), str.size()); while (!bytes.empty()) { @@ -103,6 +109,191 @@ auto Internal::FileRefBase::WriteFromString(llvm::StringRef str) } bytes = *write_result; } + CARBON_RETURN_IF_ERROR(Truncate(str.size())); + return Success(); +} + +// A macOS specific sleep routine that builds on more standard utilities. This +// is technically a portable implementation so we always compile it but only use +// it on macOS where the more efficient direct use of `clock_nanosleep` isn't +// available. +[[maybe_unused]] +static auto SleepMacos(Duration sleep) -> void { + TimePoint stop = Clock::now() + sleep; + + timespec sleep_ts = Internal::DurationToTimespec(sleep); + for (;;) { + timespec rem_sleep_ts = {}; + int result = nanosleep(&sleep_ts, &rem_sleep_ts); + if (result == 0) { + return; + } + + // Continue sleeping if we get interrupted by a resumable signal. For + // everything else report it. + if (errno != EINTR) { + int errnum = errno; + RawStringOstream error_os; + PrintErrorNumber(error_os, errnum); + CARBON_FATAL("Unexpected error while sleeping: {0}", error_os.TakeStr()); + } + + // Update to the remaining sleep time for the next attempt at sleeping. + sleep_ts = rem_sleep_ts; + + // Also check if the clock has passed our stop time as a fallback to avoid + // too much clock skew. + if (Clock::now() > stop) { + return; + } + } +} + +static auto Sleep(Duration sleep) -> void { + // For every platform but macOS we can sleep directly on an absolute time. +#if __APPLE__ + // On Apple platforms, dispatch to a specialized routine. + SleepMacos(sleep); +#else + + // We use `clock_gettime` instead of the filesystem `Clock` or some other + // `std::chrono` clock because we want to use the exact same clock that we'll + // use for sleeping below, and we'll need the time in a `timespec` for that + // call anyways. We do use a monotonic clock to try and avoid sleeps being + // interrupted by clock changes. + timespec ts = {}; + int result = clock_gettime(CLOCK_MONOTONIC, &ts); + CARBON_CHECK(result == 0, "Error getting the time: {0}", strerror(errno)); + + // Now convert the timespec to a duration that we can safely do arithmetic on. + // Since the sleep interval is in nanoseconds it is tempting to directly do + // arithmetic here, but this has a subtle pitfall near the boundary between + // the nanosecond component and the second component. + // + // Note that our `Duration` uses `__int128` to avoid worrying about running + // out of precision to represent the final deadline. + Duration stop_time = std::chrono::seconds(ts.tv_sec); + stop_time += std::chrono::nanoseconds(ts.tv_nsec); + stop_time += sleep; + + // Now convert back to timespec. + ts = Internal::DurationToTimespec(stop_time); + + do { + result = clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, &ts, nullptr); + + // Continue sleeping if we get interrupted by a resumable signal. Because + // we're using a monotonic clock and an absolute deadline time we will + // eventually progress past that deadline. + } while (result != 0 && (errno == EINTR)); + + if (result != 0) { + int errnum = errno; + RawStringOstream error_os; + PrintErrorNumber(error_os, errnum); + CARBON_FATAL("Unexpected error while sleeping: {0}", error_os.TakeStr()); + } +#endif +} + +auto Internal::FileRefBase::TryLock(FileLock::Kind kind, Duration deadline, + Duration poll_interval) + -> ErrorOr { + CARBON_CHECK(poll_interval <= deadline); + if (deadline != Duration(0) && poll_interval == Duration(0)) { + // If the caller didn't provide a poll interval but did provide a deadline, + // pick a poll interval to roughly be 1/1000th of the deadline but at least + // 1 microsecond. We don't support polling faster than 1 microsecond given + // how expensive file locking is. + poll_interval = + std::max(Duration(std::chrono::microseconds(1)), deadline / 1000); + } + if (deadline != Duration(0)) { + CARBON_CHECK( + deadline >= std::chrono::microseconds(10), + "A deadline for a file lock shorter than 10 microseconds is not " + "supported, callers can implement their own polling logic."); + CARBON_CHECK(poll_interval >= std::chrono::microseconds(1), + "Polling for a file lock faster than every microsecond is not " + "supported, callers can implement their own polling logic."); + } + auto stop = Clock::now() + deadline; + for (;;) { + int result = flock(fd_, static_cast(kind) | LOCK_NB); + if (result == 0) { + return FileLock(fd_); + } + + // Return an error if this is something other than blocking for the lock to + // be available, or we didn't get a deadline for continuing to try and + // acquire the lock, or we've reached our deadline. + if (errno != EWOULDBLOCK || deadline == Duration(0) || + Clock::now() >= stop) { + return FdError(errno, "File::TryLock on '{0}'", fd_); + } + + // The caller requested attempting to wait up to a deadline to acquire the + // lock with a specific poll interval. Try to sleep for that poll interval + // before trying the lock again. + Sleep(poll_interval); + } +} + +auto DirRef::AppendEntriesIf( + llvm::SmallVectorImpl& entries, + llvm::function_refbool> predicate) + -> ErrorOr { + CARBON_ASSIGN_OR_RETURN(Reader reader, Read()); + for (const Entry& entry : reader) { + llvm::StringRef name = entry.name(); + if (name == "." || name == "..") { + continue; + } + if (predicate && !predicate(name)) { + continue; + } + entries.push_back(name.str()); + } + return Success(); +} + +auto DirRef::AppendEntriesIf( + llvm::SmallVectorImpl& dir_entries, + llvm::SmallVectorImpl& non_dir_entries, + llvm::function_refbool> predicate) + -> ErrorOr { + CARBON_ASSIGN_OR_RETURN(Reader reader, Read()); + for (const Entry& entry : reader) { + llvm::StringRef name = entry.name(); + if (name == "." || name == "..") { + continue; + } + if (predicate && !predicate(name)) { + continue; + } + std::filesystem::path name_path = name.str(); + if (entry.is_known_dir()) { + dir_entries.push_back(std::move(name_path)); + continue; + } + if (!entry.is_unknown_type()) { + non_dir_entries.push_back(std::move(name_path)); + continue; + } + + auto stat_result = Lstat(name_path); + if (!stat_result.ok()) { + return FdError(stat_result.error().unix_errnum(), + "Dir::AppendEntriesIf on '{0}' failed while stat-ing " + "entries to determine which are directories", + dfd_); + } + if (stat_result->is_dir()) { + dir_entries.push_back(std::move(name_path)); + } else { + non_dir_entries.push_back(std::move(name_path)); + } + } return Success(); } @@ -213,7 +404,7 @@ auto DirRef::OpenDir(const std::filesystem::path& path, auto DirRef::ReadFileToString(const std::filesystem::path& path) -> ErrorOr { CARBON_ASSIGN_OR_RETURN(ReadFile f, OpenReadOnly(path)); - auto result = f.ReadToString(); + auto result = f.ReadFileToString(); if (result.ok()) { return *std::move(result); } @@ -227,14 +418,18 @@ auto DirRef::WriteFileFromString(const std::filesystem::path& path, CreationOptions creation_options) -> ErrorOr { CARBON_ASSIGN_OR_RETURN(WriteFile f, OpenWriteOnly(path, creation_options)); - auto write_result = f.WriteFromString(content); + auto write_result = f.WriteFileFromString(content); + // Immediately close the file as even if there was a write error we don't want + // to leave the file open. + auto close_result = std::move(f).Close(); + + // Now report the first error encountered or return success. if (!write_result.ok()) { return PathError( write_result.error().unix_errnum(), "Write error in Dir::WriteFileFromString on '{0}' relative to '{1}'", path, dfd_); } - auto close_result = std::move(f).Close(); if (!close_result.ok()) { return PathError( close_result.error().unix_errnum(), @@ -499,6 +694,12 @@ auto MakeTmpDir() -> ErrorOr { std::filesystem::path target = BuildData::BuildTarget.str(); tmpdir_path /= target.filename(); + return MakeTmpDirWithPrefix(std::move(tmpdir_path)); +} + +auto MakeTmpDirWithPrefix(std::filesystem::path prefix) + -> ErrorOr { + std::filesystem::path tmpdir_path = std::move(prefix); tmpdir_path += ".XXXXXX"; std::string tmpdir_path_buffer = tmpdir_path.native(); diff --git a/common/filesystem.h b/common/filesystem.h index 70e58548fd8b..f7f972032289 100644 --- a/common/filesystem.h +++ b/common/filesystem.h @@ -7,10 +7,12 @@ #include #include +#include #include #include #include +#include #include #include #include @@ -189,6 +191,10 @@ enum class FileType : ModeType { UnixMask = S_IFMT, }; +using Clock = std::chrono::file_clock; +using Duration = std::chrono::duration<__int128, std::nano>; +using TimePoint = std::chrono::time_point; + // Enumerates the different open access modes available. // // These are largely used to parameterize types in order to constrain which API @@ -227,6 +233,15 @@ consteval auto Cwd() -> Dir; // the caller's responsibility to clean up this directory. auto MakeTmpDir() -> ErrorOr; +// Creates a temporary directory and returns a removing directory handle to it. +// +// The path of the temporary directory will use the provided prefix, and will +// not add any additional directory separators. Every component but the last in +// the prefix path must exist and be a directory, the last directory must be +// writable. +auto MakeTmpDirWithPrefix(std::filesystem::path prefix) + -> ErrorOr; + // Class modeling a file (or directory) status information structure. // // This provides a largely-portable model that callers can use, as well as a few @@ -250,6 +265,18 @@ class FileStatus { // the `ModeType` documentation for how to interpret the result. auto permissions() const -> ModeType { return stat_buf_.st_mode & 0777; } + auto mtime() const -> TimePoint { + // Linux, FreeBSD, and OpenBSD all use `st_mtim`, but macOS uses a different + // spelling. +#if __APPLE__ + timespec ts = stat_buf_.st_mtimespec; +#else + timespec ts = stat_buf_.st_mtim; +#endif + TimePoint t(std::chrono::seconds(ts.tv_sec)); + return t + std::chrono::nanoseconds(ts.tv_nsec); + } + // Non-portable APIs only available on Unix-like systems. See the // documentation of the Unix `stat` structure fields they expose for their // meaning. @@ -265,6 +292,43 @@ class FileStatus { struct stat stat_buf_ = {}; }; +// Models a held lock on a file or directory. +// +// Can model either a shared or exclusive lock with respect to the file, but the +// held lock is unique. +// +// Must be released prior to the underlying file or directory being closed as it +// contains a non-owning reference to that underlying file. +class FileLock { + public: + enum class Kind { + Shared = LOCK_SH, + Exclusive = LOCK_EX, + }; + using enum Kind; + + FileLock() = default; + FileLock(FileLock&& arg) noexcept : fd_(std::exchange(arg.fd_, -1)) {} + auto operator=(FileLock&& arg) noexcept -> FileLock& { + Destroy(); + fd_ = std::exchange(arg.fd_, -1); + return *this; + } + ~FileLock() { Destroy(); } + + // Returns true if the lock is currently held. + auto is_locked() const -> bool { return fd_ != -1; } + + private: + friend Internal::FileRefBase; + + explicit FileLock(int fd) : fd_(fd) {} + + auto Destroy() -> void; + + int fd_ = -1; +}; + // The base class defining the core `File` API. // // While not used directly, this is the base class used to implement all of the @@ -283,11 +347,22 @@ class Internal::FileRefBase { // handle in that case. This is to support rebinding operations. FileRefBase() = default; + // Returns true if this refers to a valid open file, and false otherwise. + auto is_valid() const -> bool { return fd_ != -1; } + // Reads the file status. // // Analogous to the Unix-like `fstat` call. auto Stat() -> ErrorOr; + // Updates the access and modification times for the open file. + // + // If no explicit `time_point` is provided, sets both times to the current + // time. If an explicit `time_point` is provided, the times are updated to + // that time. + auto UpdateTimes(std::optional time_point = std::nullopt) + -> ErrorOr; + // Methods to seek the current file position, with various semantics for the // offset. auto Seek(int64_t delta) -> ErrorOr; @@ -295,6 +370,13 @@ class Internal::FileRefBase { -> ErrorOr; auto SeekFromEnd(int64_t delta_from_end) -> ErrorOr; + // Truncates or extends the size of the file to the provided size. + // + // If the new size is smaller, all bytes beyond this size will be unavailable. + // If the new size is larger, the file will be zero-filled to the new size. + // The position of reads and writes does not change. + auto Truncate(int64_t new_size) -> ErrorOr; + // Reads as much data as is available and fits into the provided buffer. // // On success, this returns a new slice from the start to the end of the @@ -330,21 +412,50 @@ class Internal::FileRefBase { // which remains owned by the owning `File` object. auto WriteStream() -> llvm::raw_fd_ostream; - // Reads the file until EOF into the returned string. + // Reads the file from its start until EOF into the returned string. // // This method will retry any recoverable errors and work to completely read - // the file contents up to first encountering EOF. + // the file contents from its beginning up to first encountering EOF. This + // will seek the file to ensure it starts at the beginning regardless of + // previous read or write operations. // // Any non-recoverable errors are returned to the caller. - auto ReadToString() -> ErrorOr; + auto ReadFileToString() -> ErrorOr; - // Writes a string into the file starting from the current position. + // Writes the provided string to the file from the start and truncating to the + // written size. // // This method will retry any recoverable errors and work to completely write - // the provided content into the file. + // the provided content into the file from its beginning, and truncate the + // file's size to the provided string size. Essentially, this function + // replaces the file contents with the string's contents. // // Any non-recoverable errors are returned to the caller. - auto WriteFromString(llvm::StringRef str) -> ErrorOr; + auto WriteFileFromString(llvm::StringRef str) -> ErrorOr; + + // Attempt to acquire an advisory shared lock on this directory. + // + // This is always done as a non-blocking operation, as blocking on advisory + // locks without a deadline can easily result in build systems essentially + // "fork-bombing" a machine. However, a `deadline` duration can be provided + // and this routine will block and attempt to acquire the requested lock for a + // bounded amount of time approximately based on that duration. Further, a + // `poll_interval` can be provided to control how quickly the lock will be + // polled during that duration. There is no scaling of the poll intervals at + // this layer, if a back-off heuristic is desired the caller should manage the + // polling themselves. The goal is to allow simple cases of spurious failures + // to be easily handled without unbounded blocking calls. Typically, callers + // should use a duration that is a reasonable upper bound on the latency to + // begin the locked operation and a poll interval that is a reasonably low + // median latency to begin the operation as 1-2 polls is expected to be + // common. These should not be set anywhere near the cost of acquiring a file + // lock, and in general file locks should only be used for expensive + // operations where it is worth significant delays to avoid duplicate work. + // + // If the lock cannot be acquired, the most recent lock-attempt error is + // returned. + auto TryLock(FileLock::Kind kind, Duration deadline = {}, + Duration poll_interval = {}) -> ErrorOr; protected: explicit FileRefBase(int fd) : fd_(fd) {} @@ -417,6 +528,8 @@ class FileRef : public Internal::FileRefBase { // Read and Write methods that delegate to the `FileRefBase` implementations, // but require the relevant access. See the methods on `FileRefBase` for full // documentation. + auto Truncate(int64_t new_size) -> ErrorOr + requires Writeable; auto ReadToBuffer(llvm::MutableArrayRef buffer) -> ErrorOr, FdError> requires Readable; @@ -425,9 +538,9 @@ class FileRef : public Internal::FileRefBase { requires Writeable; auto WriteStream() -> llvm::raw_fd_ostream requires Writeable; - auto ReadToString() -> ErrorOr + auto ReadFileToString() -> ErrorOr requires Readable; - auto WriteFromString(llvm::StringRef str) -> ErrorOr + auto WriteFileFromString(llvm::StringRef str) -> ErrorOr requires Writeable; protected: @@ -540,18 +653,60 @@ class DirRef { class Iterator; class Reader; + constexpr DirRef() = default; + + // Returns true if this refers to a valid open directory, and false otherwise. + auto is_valid() const -> bool { return dfd_ != -1; } + // Begin reading the entries in a directory. // // This returns a `Reader` object that can be iterated to walk over all the // entries in this directory. Note that the returned `Reader` owns a newly // allocated handle to this directory, and provides the full `DirRef` API. If // it isn't necessary to keep both open, the `Dir` class offers a - // move-qualified overload that optimizes this case. + // move-qualified method `TakeAndRead` that optimizes this case. // // Note that it is unspecified whether added and removed files during the // lifetime of the reader will be included when iterating, but otherwise // concurrent mutations are well defined. - auto Read() & -> ErrorOr; + auto Read() -> ErrorOr; + + // Reads all of the entries in this directory into a vector. + // + // A helper function wrapping `Read` and walking the resulting reader's + // entries to produce a container. + // + // For details on errors, see the documentation of `Read` and `Reader`. + auto ReadEntries() + -> ErrorOr, FdError>; + + // Reads the directory and appends entries to a container if they pass a + // predicate. The predicate can be null, which is treated as if it always + // returns true. + // + // For details on errors, see the documentation of `Read` and `Reader`. + auto AppendEntriesIf( + llvm::SmallVectorImpl& entries, + llvm::function_refbool> predicate = {}) + -> ErrorOr; + + // Reads the directory and appends entries to one of two containers if they + // pass a predicate. The predicate can be null, which is treated as if it + // always returns true. + // + // Which container an entry is appended to depends on its kind -- directories + // go to the first and non-directories go to the second. This turns out to be + // a very common split with subdirectories often needing separate handling + // from other entries. + // + // For details on errors, see the documentation of `Read` and `Reader`. This + // may also `Stat` entries if necessary to determine whether they are + // directories. + auto AppendEntriesIf( + llvm::SmallVectorImpl& dir_entries, + llvm::SmallVectorImpl& non_dir_entries, + llvm::function_refbool> predicate = {}) + -> ErrorOr; // Checks that the provided path can be accessed. auto Access(const std::filesystem::path& path, @@ -575,6 +730,15 @@ class DirRef { auto Lstat(const std::filesystem::path& path) -> ErrorOr; + // Updates the access and modification times for the provided path. + // + // If no explicit `time_point` is provided, sets both times to the current + // time. If an explicit `time_point` is provided, the times are updated to + // that time. + auto UpdateTimes(const std::filesystem::path& path, + std::optional time_point = std::nullopt) + -> ErrorOr; + // Reads the target string of the symlink at the provided path. // // This does not follow the symlink, and does not require the symlink target @@ -682,7 +846,16 @@ class DirRef { CreationOptions creation_options = CreateAlways) -> ErrorOr; - // Moves a file from one directory to another directory. + // Renames an entry from one directory to another directory, replacing any + // existing entry with the target path. + // + // Note that this is *not* a general purpose move! It must be possible for + // this operation to be performed as a metadata-only change, and so without + // moving any actual data. This means it will not work across devices, mounts, + // or filesystems. However, these restrictions make this an *atomic* rename. + // + // The most common usage is to rename an entry within a single directory, by + // passing `*this` as `target_dir`. auto Rename(const std::filesystem::path& path, DirRef target_dir, const std::filesystem::path& target_path) -> ErrorOr; @@ -753,7 +926,6 @@ class DirRef { auto Rmtree(const std::filesystem::path& path) -> ErrorOr; protected: - constexpr DirRef() = default; constexpr explicit DirRef(int dfd) : dfd_(dfd) {} // Slow-path fallback when unable to read the symlink target into a small @@ -1022,8 +1194,10 @@ class ErrnoErrorBase : public ErrorBase { auto no_entity() const -> bool { return errnum_ == ENOENT; } auto not_dir() const -> bool { return errnum_ == ENOTDIR; } auto access_denied() const -> bool { return errnum_ == EACCES; } + auto would_block() const -> bool { return errnum_ == EWOULDBLOCK; } - // Specific to `Rmdir` operations, two different error values can be used. + // Specific to `Rmdir` and `Rename` operations that remove a directory name, + // two different error values can be used. auto not_empty() const -> bool { return errnum_ == ENOTEMPTY || errnum_ == EEXIST; } @@ -1073,6 +1247,7 @@ class FdError : public Internal::ErrnoErrorBase { auto Print(llvm::raw_ostream& out) const -> void; private: + friend FileLock; friend Internal::FileRefBase; friend ReadFile; friend WriteFile; @@ -1136,8 +1311,37 @@ class PathError : public Internal::ErrnoErrorBase { // Implementation details only below. +namespace Internal { + +inline auto DurationToTimespec(Duration d) -> timespec { + timespec ts = {}; + ts.tv_sec = std::chrono::duration_cast(d).count(); + d -= std::chrono::seconds(ts.tv_sec); + ts.tv_nsec = std::chrono::duration_cast(d).count(); + return ts; +} + +} // namespace Internal + consteval auto Cwd() -> Dir { return Dir(AT_FDCWD); } +inline auto FileLock::Destroy() -> void { + if (fd_ == -1) { + // Nothing to unlock. + return; + } + + // We always try to unlock in a non-blocking way as there should never be a + // reason to block here. + int result = flock(fd_, LOCK_UN | LOCK_NB); + + // The only realistic error is `EBADF` that would represent a programming + // error the type system should prevent. We conservatively check-fail if an + // error occurs here. + CARBON_CHECK(result == 0, "{0}", + FdError(errno, "Unexpected error while _unlocking_ '{0}'", fd_)); +} + inline auto Internal::FileRefBase::Stat() -> ErrorOr { FileStatus status; if (fstat(fd_, &status.stat_buf_) == 0) { @@ -1146,6 +1350,24 @@ inline auto Internal::FileRefBase::Stat() -> ErrorOr { return FdError(errno, "File::Stat on '{0}'", fd_); } +inline auto Internal::FileRefBase::UpdateTimes( + std::optional time_point) -> ErrorOr { + if (!time_point) { + if (futimens(fd_, nullptr) == -1) { + return FdError(errno, "File::UpdateTimes to now on '{0}'", fd_); + } + return Success(); + } + + timespec times[2]; + times[0] = Internal::DurationToTimespec(time_point->time_since_epoch()); + times[1] = times[0]; + if (futimens(fd_, times) == -1) { + return FdError(errno, "File::UpdateTimes to a specific time on '{0}'", fd_); + } + return Success(); +} + inline auto Internal::FileRefBase::Seek(int64_t delta) -> ErrorOr { int64_t byte_offset = lseek(fd_, delta, SEEK_CUR); @@ -1173,6 +1395,15 @@ inline auto Internal::FileRefBase::SeekFromEnd(int64_t delta_from_end) return byte_offset; } +inline auto Internal::FileRefBase::Truncate(int64_t new_size) + -> ErrorOr { + int64_t result = ftruncate(fd_, new_size); + if (result == -1) { + return FdError(errno, "File::Truncate on '{0}'", fd_); + } + return Success(); +} + inline auto Internal::FileRefBase::ReadToBuffer( llvm::MutableArrayRef buffer) -> ErrorOr, FdError> { @@ -1237,6 +1468,13 @@ inline auto Internal::FileRefBase::WriteableDestroy() -> void { "calling `Close` and handling any errors to avoid data loss."); } +template +auto FileRef::Truncate(int64_t new_size) -> ErrorOr + requires Writeable +{ + return FileRefBase::Truncate(new_size); +} + template auto FileRef::ReadToBuffer(llvm::MutableArrayRef buffer) -> ErrorOr, FdError> @@ -1246,10 +1484,10 @@ auto FileRef::ReadToBuffer(llvm::MutableArrayRef buffer) } template -auto FileRef::ReadToString() -> ErrorOr +auto FileRef::ReadFileToString() -> ErrorOr requires Readable { - return FileRefBase::ReadToString(); + return FileRefBase::ReadFileToString(); } template @@ -1268,11 +1506,11 @@ auto FileRef::WriteStream() -> llvm::raw_fd_ostream } template -auto FileRef::WriteFromString(llvm::StringRef str) +auto FileRef::WriteFileFromString(llvm::StringRef str) -> ErrorOr requires Writeable { - return FileRefBase::WriteFromString(str); + return FileRefBase::WriteFileFromString(str); } template @@ -1284,7 +1522,7 @@ auto File::Destroy() -> void { } } -inline auto DirRef::Read() & -> ErrorOr { +inline auto DirRef::Read() -> ErrorOr { int dup_dfd = dup(dfd_); if (dup_dfd == -1) { // There are very few plausible errors here, but we can return one so it @@ -1296,6 +1534,13 @@ inline auto DirRef::Read() & -> ErrorOr { return Dir(dup_dfd).TakeAndRead(); } +inline auto DirRef::ReadEntries() + -> ErrorOr, FdError> { + llvm::SmallVector entries; + CARBON_RETURN_IF_ERROR(AppendEntriesIf(entries)); + return entries; +} + inline auto DirRef::Access(const std::filesystem::path& path, AccessCheckFlags check) -> ErrorOr { if (faccessat(dfd_, path.c_str(), static_cast(check), /*flags=*/0) == @@ -1332,6 +1577,29 @@ inline auto DirRef::Lstat(const std::filesystem::path& path) return PathError(errno, "Dir::Lstat on '{0}' relative to '{1}'", path, dfd_); } +inline auto DirRef::UpdateTimes(const std::filesystem::path& path, + std::optional time_point) + -> ErrorOr { + if (!time_point) { + if (utimensat(dfd_, path.c_str(), nullptr, /*flags*/ 0) == -1) { + return PathError(errno, + "Dir::UpdateTimes to now on '{0}' relative to '{1}'", + path, dfd_); + } + return Success(); + } + + timespec times[2]; + times[0] = Internal::DurationToTimespec(time_point->time_since_epoch()); + times[1] = times[0]; + if (utimensat(dfd_, path.c_str(), times, /*flags*/ 0) == -1) { + return PathError( + errno, "Dir::UpdateTimes to a specific time on '{0}' relative to '{1}'", + path, dfd_); + } + return Success(); +} + inline auto DirRef::Readlink(const std::filesystem::path& path) -> ErrorOr { // On the fast path, we read into a small stack buffer and get the whole @@ -1547,7 +1815,11 @@ inline auto Dir::Iterator::operator++() -> Iterator& { return *this; } -inline auto Dir::Reader::begin() -> Iterator { return Iterator(dirp_); } +inline auto Dir::Reader::begin() -> Iterator { + // Reset the position of the directory stream to get the actual beginning. + rewinddir(dirp_); + return Iterator(dirp_); +} inline auto Dir::Reader::end() -> Iterator { return Iterator(); } diff --git a/common/filesystem_test.cpp b/common/filesystem_test.cpp index 7f592e9b2956..31bcd0808cf5 100644 --- a/common/filesystem_test.cpp +++ b/common/filesystem_test.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include "common/error_test_helpers.h" @@ -21,6 +22,7 @@ using ::testing::Eq; using ::testing::HasSubstr; using Testing::IsError; using Testing::IsSuccess; +using ::testing::UnorderedElementsAre; class FilesystemTest : public ::testing::Test { public: @@ -101,7 +103,7 @@ TEST_F(FilesystemTest, BasicWriteAndRead) { { auto f = dir_.OpenWriteOnly("test", CreationOptions::CreateNew); ASSERT_THAT(f, IsSuccess(_)); - auto write_result = f->WriteFromString(content_str); + auto write_result = f->WriteFileFromString(content_str); EXPECT_THAT(write_result, IsSuccess(_)); (*std::move(f)).Close().Check(); } @@ -109,7 +111,7 @@ TEST_F(FilesystemTest, BasicWriteAndRead) { { auto f = dir_.OpenReadOnly("test"); ASSERT_THAT(f, IsSuccess(_)); - auto read_result = f->ReadToString(); + auto read_result = f->ReadFileToString(); EXPECT_THAT(read_result, IsSuccess(Eq(content_str))); } @@ -117,6 +119,74 @@ TEST_F(FilesystemTest, BasicWriteAndRead) { EXPECT_THAT(unlink_result, IsSuccess(_)); } +TEST_F(FilesystemTest, SeekReadAndWrite) { + std::string content_str = "0123456789"; + // First write some initial content. + { + auto f = dir_.OpenWriteOnly("test", CreationOptions::CreateNew); + ASSERT_THAT(f, IsSuccess(_)); + auto write_result = f->WriteFileFromString(content_str); + EXPECT_THAT(write_result, IsSuccess(_)); + (*std::move(f)).Close().Check(); + } + + // Now seek and read. + { + auto f = dir_.OpenReadOnly("test"); + ASSERT_THAT(f, IsSuccess(_)); + auto seek_result = f->Seek(3); + ASSERT_THAT(seek_result, IsSuccess(Eq(3))); + std::array buffer; + auto read_result = f->ReadToBuffer(buffer); + ASSERT_THAT(read_result, IsSuccess(_)); + EXPECT_THAT(std::string(reinterpret_cast(read_result->data()), + read_result->size()), + Eq(content_str.substr(3, read_result->size()))); + + // Now test that we can seek back to the beginning and read the full file. + auto read_file_result = f->ReadFileToString(); + EXPECT_THAT(read_file_result, IsSuccess(Eq(content_str))); + } + + // Now a mixture of reads, writes, an seeking. + { + auto f = dir_.OpenReadWrite("test"); + ASSERT_THAT(f, IsSuccess(_)); + auto seek_result = f->SeekFromEnd(-6); + ASSERT_THAT(seek_result, IsSuccess(Eq(content_str.size() - 6))); + std::string new_content_str = "abcdefg"; + llvm::ArrayRef new_content_bytes( + reinterpret_cast(new_content_str.data()), + new_content_str.size()); + for (auto write_bytes = new_content_bytes.slice(0, 4); + !write_bytes.empty();) { + auto write_result = f->WriteFromBuffer(write_bytes); + ASSERT_THAT(write_result, IsSuccess(_)); + write_bytes = *write_result; + } + + std::array buffer; + auto read_result = f->ReadToBuffer(buffer); + ASSERT_THAT(read_result, IsSuccess(_)); + EXPECT_THAT(std::string(reinterpret_cast(read_result->data()), + read_result->size()), + Eq(content_str.substr(8, read_result->size()))); + + EXPECT_THAT(*f->ReadFileToString(), "0123abcd89"); + + // Now write the entire file, also changing its size, after a fresh seek. + seek_result = f->Seek(-6); + ASSERT_THAT(seek_result, IsSuccess(Eq(content_str.size() - 6))); + auto write_file_result = f->WriteFileFromString(new_content_str); + EXPECT_THAT(write_file_result, IsSuccess(_)); + EXPECT_THAT(*f->ReadFileToString(), "abcdefg"); + (*std::move(f)).Close().Check(); + } + + auto unlink_result = dir_.Unlink("test"); + EXPECT_THAT(unlink_result, IsSuccess(_)); +} + TEST_F(FilesystemTest, CreateAndRemoveDirecotries) { auto d1 = Cwd().CreateDirectories(path() / "a" / "b" / "c" / "test1"); ASSERT_THAT(d1, IsSuccess(_)); @@ -185,7 +255,7 @@ TEST_F(FilesystemTest, StatAndAccess) { ModeType permissions = 0450; auto f = dir_.OpenWriteOnly("test", CreationOptions::CreateNew, permissions); ASSERT_THAT(f, IsSuccess(_)); - auto write_result = f->WriteFromString(content_str); + auto write_result = f->WriteFileFromString(content_str); EXPECT_THAT(write_result, IsSuccess(_)); access_result = dir_.Access("test"); @@ -301,7 +371,7 @@ TEST_F(FilesystemTest, Chdir) { // Create a regular file and try to chdir to that. auto f = dir_.OpenWriteOnly("test2", CreationOptions::CreateNew); ASSERT_THAT(f, IsSuccess(_)); - auto write_result = f->WriteFromString("test2"); + auto write_result = f->WriteFileFromString("test2"); EXPECT_THAT(write_result, IsSuccess(_)); chdir_path_result = dir_.Chdir("test2"); ASSERT_FALSE(chdir_path_result.ok()); @@ -392,5 +462,157 @@ TEST_F(FilesystemTest, Rename) { EXPECT_THAT(result.error().unix_errnum(), EINVAL) << result.error(); } +TEST_F(FilesystemTest, TryLock) { + auto file = dir_.OpenReadWrite("test_file", CreateNew); + ASSERT_THAT(file, IsSuccess(_)); + + // Acquire an exclusive lock. + auto lock = file->TryLock(FileLock::Exclusive); + ASSERT_THAT(lock, IsSuccess(_)); + EXPECT_TRUE(lock->is_locked()); + + // Try to acquire a second lock from a different file object. + auto file2 = dir_.OpenReadOnly("test_file"); + ASSERT_THAT(file2, IsSuccess(_)); + auto lock2 = file2->TryLock(FileLock::Exclusive); + ASSERT_THAT(lock2, IsError(_)); + EXPECT_TRUE(lock2.error().would_block()); + + // A shared lock should also fail. + auto lock3 = file2->TryLock(FileLock::Shared); + ASSERT_THAT(lock3, IsError(_)); + EXPECT_TRUE(lock3.error().would_block()); + + // Release the first lock. + *lock = {}; + EXPECT_FALSE(lock->is_locked()); + + // Now we can acquire an exclusive lock. + lock2 = file2->TryLock(FileLock::Exclusive); + ASSERT_THAT(lock2, IsSuccess(_)); + EXPECT_TRUE(lock2->is_locked()); + *lock2 = {}; + + // Test shared locks. + auto shared_lock1 = file->TryLock(FileLock::Shared); + ASSERT_THAT(shared_lock1, IsSuccess(_)); + EXPECT_TRUE(shared_lock1->is_locked()); + + auto shared_lock2 = file2->TryLock(FileLock::Shared); + ASSERT_THAT(shared_lock2, IsSuccess(_)); + EXPECT_TRUE(shared_lock2->is_locked()); + + // An exclusive lock should fail. + auto file3 = dir_.OpenReadOnly("test_file"); + ASSERT_THAT(file3, IsSuccess(_)); + auto exclusive_lock = file3->TryLock(FileLock::Exclusive); + ASSERT_THAT(exclusive_lock, IsError(_)); + EXPECT_TRUE(exclusive_lock.error().would_block()); + + // Release locks and close files. + *shared_lock1 = {}; + *shared_lock2 = {}; + ASSERT_THAT((*std::move(file)).Close(), IsSuccess(_)); + ASSERT_THAT((*std::move(file2)).Close(), IsSuccess(_)); + ASSERT_THAT((*std::move(file3)).Close(), IsSuccess(_)); +} + +TEST_F(FilesystemTest, ReadAndAppendEntries) { + // Test with an empty directory. + { + auto entries = dir_.ReadEntries(); + ASSERT_THAT(entries, IsSuccess(_)); + EXPECT_TRUE(entries->empty()); + } + { + llvm::SmallVector entries; + EXPECT_THAT(dir_.AppendEntriesIf(entries), IsSuccess(_)); + EXPECT_TRUE(entries.empty()); + } + + // Create some files and directories. + ASSERT_THAT(dir_.WriteFileFromString("file1", ""), IsSuccess(_)); + ASSERT_THAT(dir_.WriteFileFromString("file2", ""), IsSuccess(_)); + ASSERT_THAT(dir_.WriteFileFromString(".hidden", ""), IsSuccess(_)); + ASSERT_THAT(dir_.CreateDirectories("subdir1"), IsSuccess(_)); + ASSERT_THAT(dir_.CreateDirectories("subdir2"), IsSuccess(_)); + + // Test ReadEntries. + { + auto entries = dir_.ReadEntries(); + ASSERT_THAT(entries, IsSuccess(_)); + EXPECT_THAT(*entries, UnorderedElementsAre(".hidden", "file1", "file2", + "subdir1", "subdir2")); + } + + // Test AppendEntriesIf with no predicate. + { + llvm::SmallVector entries; + EXPECT_THAT(dir_.AppendEntriesIf(entries), IsSuccess(_)); + EXPECT_THAT(entries, UnorderedElementsAre(".hidden", "file1", "file2", + "subdir1", "subdir2")); + } + + // Test AppendEntriesIf with a predicate. + { + llvm::SmallVector entries; + auto result = dir_.AppendEntriesIf( + entries, [](llvm::StringRef name) { return name.starts_with("file"); }); + EXPECT_THAT(result, IsSuccess(_)); + EXPECT_THAT(entries, UnorderedElementsAre("file1", "file2")); + } + + // Test AppendEntriesIf with directory splitting and a predicate. + { + llvm::SmallVector dir_entries; + llvm::SmallVector non_dir_entries; + auto result = dir_.AppendEntriesIf( + dir_entries, non_dir_entries, + [](llvm::StringRef name) { return !name.starts_with("."); }); + EXPECT_THAT(result, IsSuccess(_)); + EXPECT_THAT(dir_entries, UnorderedElementsAre("subdir1", "subdir2")); + EXPECT_THAT(non_dir_entries, UnorderedElementsAre("file1", "file2")); + } +} + +TEST_F(FilesystemTest, MtimeAndUpdateTimes) { + // Test UpdateTimes on a path that doesn't exist. + auto update_missing = dir_.UpdateTimes("test_file"); + ASSERT_THAT(update_missing, IsError(_)); + EXPECT_TRUE(update_missing.error().no_entity()); + + // Create a file and get its initial modification time. + ASSERT_THAT(dir_.WriteFileFromString("test_file", "content"), IsSuccess(_)); + auto stat = dir_.Stat("test_file"); + ASSERT_THAT(stat, IsSuccess(_)); + auto time1 = stat->mtime(); + + // Repeated stats have stable time. + stat = dir_.Stat("test_file"); + ASSERT_THAT(stat, IsSuccess(_)); + EXPECT_THAT(stat->mtime(), Eq(time1)); + + // Update the timestamp to a specific time in the past. + auto past_time = time1 - std::chrono::seconds(120); + ASSERT_THAT(dir_.UpdateTimes("test_file", past_time), IsSuccess(_)); + stat = dir_.Stat("test_file"); + ASSERT_THAT(stat, IsSuccess(_)); + EXPECT_THAT(stat->mtime(), Eq(past_time)); + + // Now test updating times on an open file. Should still be at `past_time`. + auto file = *dir_.OpenReadWrite("test_file"); + auto file_stat = file.Stat(); + ASSERT_THAT(file_stat, IsSuccess(_)); + EXPECT_THAT(file_stat->mtime(), Eq(past_time)); + + // Update the times through the file and verify those updates arrived. + ASSERT_THAT(file.UpdateTimes(time1), IsSuccess(_)); + file_stat = file.Stat(); + ASSERT_THAT(file_stat, IsSuccess(_)); + EXPECT_THAT(file_stat->mtime(), Eq(time1)); + + ASSERT_THAT(std::move(file).Close(), IsSuccess(_)); +} + } // namespace } // namespace Carbon::Filesystem diff --git a/common/ostream.h b/common/ostream.h index cd79bb75e42f..4ee984becdb4 100644 --- a/common/ostream.h +++ b/common/ostream.h @@ -109,7 +109,8 @@ namespace llvm { // locally instead. template requires std::derived_from, std::ostream> && - (!std::same_as, raw_ostream>) + (!std::same_as, raw_ostream>) && + requires(raw_ostream& os, const ClassT& value) { os << value; } auto operator<<(StreamT& standard_out, const ClassT& value) -> StreamT& { raw_os_ostream(standard_out) << value; return standard_out;