diff --git a/common/BUILD b/common/BUILD index 44cba49a8d8d..c83c50f7afdf 100644 --- a/common/BUILD +++ b/common/BUILD @@ -214,6 +214,56 @@ cc_test( ], ) +cc_library( + name = "filesystem", + srcs = ["filesystem.cpp"], + hdrs = ["filesystem.h"], + deps = [ + ":build_data", + ":check", + ":error", + ":ostream", + ":raw_string_ostream", + ":template_string", + "@llvm-project//llvm:Support", + ], +) + +cc_test( + name = "filesystem_test", + size = "small", + srcs = ["filesystem_test.cpp"], + deps = [ + ":error_test_helpers", + ":filesystem", + "//testing/base:gtest_main", + "@googletest//:gtest", + "@llvm-project//llvm:Support", + ], +) + +cc_binary( + name = "filesystem_benchmark", + testonly = 1, + srcs = ["filesystem_benchmark.cpp"], + deps = [ + ":check", + ":filesystem", + "//testing/base:benchmark_main", + "@abseil-cpp//absl/hash", + "@abseil-cpp//absl/random", + "@google_benchmark//:benchmark", + "@llvm-project//llvm:Support", + ], +) + +sh_test( + name = "filesystem_benchmark_test", + size = "small", + srcs = [":filesystem_benchmark"], + args = ["--benchmark_min_time=1x"], +) + cc_library( name = "find", hdrs = ["find.h"], diff --git a/common/filesystem.cpp b/common/filesystem.cpp new file mode 100644 index 000000000000..ce8b4b9bfc77 --- /dev/null +++ b/common/filesystem.cpp @@ -0,0 +1,534 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "common/filesystem.h" + +#include +#include + +#include "common/build_data.h" +#include "llvm/Support/MathExtras.h" + +namespace Carbon::Filesystem { + +// Render an error number from `errno` to the provided stream using the richest +// rendering available on the platform. +static auto PrintErrorNumber(llvm::raw_ostream& out, int errnum) -> void { +#ifdef _GNU_SOURCE + // Use GNU-specific routines to compute the error name and description. + llvm::StringRef name = strerrordesc_np(errnum); + llvm::StringRef desc = strerrorname_np(errnum); + + out << llvm::formatv("{0}: {1}", name, desc); +#elif defined(__APPLE__) || defined(_POSIX_SOURCE) + char buffer[4096]; + int meta_error = strerror_r(errnum, buffer, sizeof(buffer)); + if (meta_error == 0) { + out << llvm::formatv("errno {0}: {1}", errnum, llvm::StringRef(buffer)); + } else { + out << llvm::formatv( + "error number {0}; encountered meta-error number {1} while rendering " + "an error message", + errnum, meta_error); + } +#else +#error TODO: Implement this for other platforms. +#endif +} + +auto FdError::Print(llvm::raw_ostream& out) const -> void { + // The `format_` member is a `StringLiteral` that is null terminated, so + // `.data()` is safe here. + // NOLINTNEXTLINE(bugprone-suspicious-stringview-data-usage) + out << llvm::formatv(format_.data(), fd_) << " failed: "; + PrintErrorNumber(out, unix_errnum()); +} + +auto PathError::Print(llvm::raw_ostream& out) const -> void { + // The `format_` member is a `StringLiteral` that is null terminated, so + // `.data()` is safe here. + // NOLINTNEXTLINE(bugprone-suspicious-stringview-data-usage) + out << llvm::formatv(format_.data(), path_, dir_fd_) << " failed: "; + PrintErrorNumber(out, unix_errnum()); +} + +auto Internal::FileRefBase::ReadToString() -> ErrorOr { + std::string result; + + // Read a buffer at a time until we reach the end. We use the pipe buffer + // length as our max buffer size as it is likely to be small but reasonable + // for the OS, and in the case of pipes the same chunking in which the data + // will arrive. + // + // TODO: Replace this with a smaller buffer and using `resize_and_overwrite` + // to read into the string in-place for larger strings. Unclear if that will + // be any faster, but it will be much more friendly to callers with + // constrained stack sizes and use less memory overall. + std::byte buffer[PIPE_BUF]; + for (;;) { + auto read_result = ReadToBuffer(buffer); + if (!read_result.ok()) { + return std::move(read_result).error(); + } + if (read_result->empty()) { + // EOF + break; + } + result.append(reinterpret_cast(read_result->data()), + read_result->size()); + } + + return result; +} + +auto Internal::FileRefBase::WriteFromString(llvm::StringRef str) + -> ErrorOr { + auto bytes = llvm::ArrayRef( + reinterpret_cast(str.data()), str.size()); + while (!bytes.empty()) { + auto write_result = WriteFromBuffer(bytes); + if (!write_result.ok()) { + return std::move(write_result).error(); + } + bytes = *write_result; + } + return Success(); +} + +auto DirRef::OpenDir(const std::filesystem::path& path, + CreationOptions creation_options, ModeType creation_mode, + OpenFlags open_flags) -> ErrorOr { + // If we potentially need to create a directory, we have to do that + // separately as no systems support `O_CREAT | O_DIRECTORY`, even though + // that would be (much) nicer. + + if (creation_options == CreateNew) { + // If we are required to be the one that created the directory, disable + // following the last symlink when we open that directory. The last symlink + // is the only one that matters for security here because it is only valid + // to create the last component. It is that directory component that we want + // to ensure has not been replaced with a symlink by an adversarial + // concurrent process. + open_flags |= OpenFlags::NoFollow; + } + + if (creation_options != OpenExisting) { + CARBON_CHECK(creation_options != CreateAlways, + "Invalid `creation_options` value of `CreateAlways`: there is " + "no support for truncating directories, and so they cannot be " + "created in an analogous way to files if they already exist."); + + if (mkdirat(dfd_, path.c_str(), creation_mode) != 0) { + // Unless the error is just that the path already exists, and that is + // allowed for the requested creation flags, report any error here as part + // of opening just like we would if the error originated from `openat` + // with `O_CREAT`. + if (creation_options == CreateNew || errno != EEXIST) { + return PathError(errno, + "Calling `mkdirat` on '{0}' relative to '{1}' during " + "DirRef::OpenDir", + path, dfd_); + } + } + } + + // Open this path as a directory. Note that this has to succeed, and when we + // created the directory we require the last component to not be a symlink in + // case it was _replaced_ with a symlink while running. + int result_fd = + openat(dfd_, path.c_str(), static_cast(open_flags) | O_DIRECTORY); + if (result_fd == -1) { + // No need for `EINTR` handling here as if this is a FIFO it would be an + // error with `O_DIRECTORY`. + return PathError( + errno, + "Calling `openat` on '{0}' relative to '{1}' during DirRef::OpenDir", + path, dfd_); + } + Dir result(result_fd); + + // If we were required to create the directory, we also need to verify that + // the opened file descriptor continues to have the same permissions and the + // correct owner as we couldn't do the creation atomically with the open. This + // defends against an adversarial removal of the created directory and + // creation of a new directory with the same name but either with wider + // permissions such as all-write, or with a different owner. + // + // We don't defend against replacement with a directory of the same name, same + // permissions, same owner, but different group. There is no good way to do + // this defense given the complexity of group assignment, and there appears to + // be no need. Achieving such a replacement without superuser power would + // require a parent directory with `setgid` bit, and a group that gives the + // attacker access -- but such a parent directory would make *any* creation + // vulnerable without any need for a replacement, so we can't defend against + // that here. The caller has ample tools to defend against this including + // taking care with the parent directory and restricting the group permission + // bits which we *do* verify. + if (creation_options == CreateNew) { + auto stat_result = result.Stat(); + if (!stat_result.ok()) { + // Manually propagate this error so we can attach it back to the opened + // path and relative directory. + return PathError(stat_result.error().unix_errnum(), + "DirRef::Stat after opening '{0}' relative to '{1}'", + path, dfd_); + } + + // Check that the owning UID is the current effective UID. + if (stat_result->unix_uid() != geteuid()) { + // Model this as `EPERM`, which is a bit awkward, but should be fine. + return PathError(EPERM, + "Unexpected UID change after creating '{0}' relative to " + "'{1}' during DirRef::OpenDir", + path, dfd_); + } + + // Check that the permissions are a subset of the requested ones. They may + // have been masked down by `umask`, but if there are *new* permissions, + // that would be a security issue. + if ((stat_result->permissions() & creation_mode) != + stat_result->permissions()) { + // Model this with `EPERM` and a custom message. + return PathError(EPERM, + "Unexpected permissions after creating '{0}' relative " + "to '{1}' during DirRef::OpenDir", + path, dfd_); + } + } + + return result; +} + +auto DirRef::ReadFileToString(const std::filesystem::path& path) + -> ErrorOr { + CARBON_ASSIGN_OR_RETURN(ReadFile f, OpenReadOnly(path)); + auto result = f.ReadToString(); + if (result.ok()) { + return *std::move(result); + } + return PathError(result.error().unix_errnum(), + "Dir::ReadFileToString on '{0}' relative to '{1}'", path, + dfd_); +} + +auto DirRef::WriteFileFromString(const std::filesystem::path& path, + llvm::StringRef content, + CreationOptions creation_options) + -> ErrorOr { + CARBON_ASSIGN_OR_RETURN(WriteFile f, OpenWriteOnly(path, creation_options)); + auto write_result = f.WriteFromString(content); + if (!write_result.ok()) { + return PathError( + write_result.error().unix_errnum(), + "Write error in Dir::WriteFileFromString on '{0}' relative to '{1}'", + path, dfd_); + } + auto close_result = std::move(f).Close(); + if (!close_result.ok()) { + return PathError( + close_result.error().unix_errnum(), + "Close error in Dir::WriteFileFromString on '{0}' relative to '{1}'", + path, dfd_); + } + return Success(); +} + +auto DirRef::CreateDirectories(const std::filesystem::path& path, + ModeType creation_mode) + -> ErrorOr { + // Avoid having to handle an empty path by immediately rejecting it as + // invalid. + if (path.empty()) { + return PathError(EINVAL, + "DirRef::CreateDirectories on '{0}' relative to '{1}'", + path, dfd_); + } + // Try directly opening the directory and use that if successful. This is an + // important hot path case of users essentially doing an "open-always" form of + // creating multiple steps of directories. + auto open_result = OpenDir(path, OpenExisting); + if (open_result.ok()) { + return std::move(*open_result); + } else if (!open_result.error().no_entity()) { + return std::move(open_result).error(); + } + + // Walk from the full path towards this directory (or the root) to find the + // first existing directory. This is faster than walking down as no file + // descriptors have to be allocated for any intervening directories, etc. We + // keep the path components that are missing as we pop them off for easy + // traversal back down. + std::optional work_dir; + // Paths typically consist of relatively few components + // and so we can use a bit of stack and avoid allocating and moving the paths + // in common cases. We use `8` as an arbitrary but likely good for all of the + // hottest cases. + llvm::SmallVector missing_components; + missing_components.push_back(path.filename()); + for (std::filesystem::path parent_path = path.parent_path(); + !parent_path.empty(); parent_path = parent_path.parent_path()) { + auto open_result = OpenDir(parent_path, OpenExisting); + if (open_result.ok()) { + work_dir = std::move(*open_result); + break; + } + missing_components.push_back(parent_path.filename()); + } + CARBON_CHECK(!missing_components.empty()); + + // If we haven't yet opened an intermediate directory, start by creating one + // relative to this directory. We can't do this as part of the loop below as + // `this` and the newly opened directory have different types. + if (!work_dir) { + std::filesystem::path component = missing_components.pop_back_val(); + CARBON_ASSIGN_OR_RETURN( + Dir component_dir, + OpenDir(component, CreationOptions::OpenAlways, creation_mode)); + // Move this component into our temporary directory slot. + work_dir = std::move(component_dir); + } + + // Now walk through the remaining components opening and creating each + // relative to the previous. + while (!missing_components.empty()) { + std::filesystem::path component = missing_components.pop_back_val(); + CARBON_ASSIGN_OR_RETURN( + Dir component_dir, + work_dir->OpenDir(component, CreationOptions::OpenAlways, + creation_mode)); + + // Close the current temporary directory and move the new component + // directory object into its place. + work_dir = std::move(component_dir); + } + + CARBON_CHECK(work_dir, + "Should always have created at least one directory for a " + "non-empty path!"); + return std::move(work_dir).value(); +} + +auto DirRef::Rmtree(const std::filesystem::path& path) + -> ErrorOr { + struct DirAndIterator { + DirRef::Reader dir; + ssize_t dir_entry_start; + }; + llvm::SmallVector dir_stack; + + llvm::SmallVector dir_entries; + llvm::SmallVector unknown_entries; + + dir_entries.push_back(path); + for (;;) { + // When we bottom out, we're removing the initial tree path and doing so + // relative to `this` directory. + DirRef current = dir_stack.empty() ? *this : dir_stack.back().dir; + ssize_t dir_entry_start = + dir_stack.empty() ? 0 : dir_stack.back().dir_entry_start; + + // If we've finished all the child directories of the current entry in the + // stack, pop it off and continue. + if (dir_entry_start == static_cast(dir_entries.size())) { + dir_stack.pop_back(); + continue; + } + CARBON_CHECK(dir_entry_start < static_cast(dir_entries.size())); + + // Take the last entry under the current directory and try removing it. + const std::filesystem::path& entry_path = dir_entries.back(); + auto rmdir_result = current.Rmdir(entry_path); + if (rmdir_result.ok() || rmdir_result.error().no_entity()) { + // Removed here or elsewhere already, so pop the entry. + dir_entries.pop_back(); + if (dir_entries.empty()) { + // The last entry is the input path with an empty stack, so we've + // finished at this point. + CARBON_CHECK(dir_stack.empty()); + return Success(); + } + continue; + } + // If we get any error other than not-empty, just return that. + if (!rmdir_result.error().not_empty()) { + return std::move(rmdir_result).error(); + } + + // Recurse into the subdirectory since it isn't empty, opening it, getting a + // reader, and pushing it onto our stack. + CARBON_ASSIGN_OR_RETURN(Dir subdir, current.OpenDir(entry_path)); + auto read_result = std::move(subdir).TakeAndRead(); + if (!read_result.ok()) { + return PathError( + read_result.error().unix_errnum(), + "Dir::Read on '{0}' relative to '{1}' during RmdirRecursively", + entry_path, current.dfd_); + } + dir_stack.push_back( + {*std::move(read_result), static_cast(dir_entries.size())}); + + // Now read the directory entries. It would be nice to be able to directly + // remove the files and empty directories as we find them when reading, and + // the POSIX spec appears to require that to work, but testing shows at + // least some Linux environments don't work reliably in this case and will + // fail to visit some entries entirely. As a consequence, we walk the entire + // directory and collect the entries into data structures before beginning + // to remove them. + DirRef::Reader& subdir_reader = dir_stack.back().dir; + for (const auto& entry : subdir_reader) { + llvm::StringRef name = entry.name(); + if (name == "." || name == "..") { + continue; + } + if (entry.is_known_dir()) { + dir_entries.push_back(name.str()); + } else { + // We end up here for entries known to be regular files, other kinds of + // non-directory entries, or when the entry kind isn't known. + // + // Unless we *know* the entry is a directory, we put it into the unknown + // entries. For these, we unlink them first in case they are + // non-directory entries and use the failure of that to move any + // directories that end up here to the directory entries list. + unknown_entries.push_back(name.str()); + } + } + + // We can immediately try to unlink all the unknown entries, which will + // include any regular files, and use an error on directories that were + // unknown above to switch them to the `dir_entries` list. + while (!unknown_entries.empty()) { + std::filesystem::path name = unknown_entries.pop_back_val(); + auto unlink_result = subdir_reader.Unlink(name); + if (unlink_result.ok() || unlink_result.error().no_entity()) { + continue; + } else if (!unlink_result.error().is_dir()) { + return std::move(unlink_result).error(); + } + dir_entries.push_back(std::move(name)); + } + + // We'll handle the directory entries we've queued here in the next + // iteration, removing them or recursing as needed. + } +} + +auto DirRef::ReadlinkSlow(const std::filesystem::path& path) + -> ErrorOr { + constexpr ssize_t MinBufferSize = +#ifdef PATH_MAX + PATH_MAX +#else + 1024 +#endif + ; + // Read directly into a string to avoid allocating two large buffers. + std::string large_buffer; + // Stat the symlink to get an initial guess at the size. + CARBON_ASSIGN_OR_RETURN(FileStatus status, Lstat(path)); + // We try to use the size from the `lstat` unless it is empty, in which case + // we try to use our minimum buffer size which is `PATH_MAX` or a constant + // value. We have a fallback to dynamically discover an adequate buffer size + // below that will handle any inaccuracy. + ssize_t buffer_size = status.size(); + if (buffer_size == 0) { + buffer_size = MinBufferSize; + } + large_buffer.resize(status.size()); + ssize_t result = + readlinkat(dfd_, path.c_str(), large_buffer.data(), large_buffer.size()); + if (result == -1) { + return PathError(errno, "Readlink on '{0}' relative to '{1}'", path, dfd_); + } + + // Now the really bad fallback case: if there are racing writes to the + // symlink, the guessed size may not have been large enough. As a last-ditch + // effort, begin doubling (from the next power of two >= our min buffer size) + // the length until it fits. We cap this at 10 MiB to prevent egregious file + // system contents (or some bug somewhere) from exhausting memory. + constexpr ssize_t MaxBufferSize = 10 << 20; + while (result == static_cast(large_buffer.size())) { + int64_t next_buffer_size = std::max( + MinBufferSize, llvm::NextPowerOf2(large_buffer.size())); + if (next_buffer_size > MaxBufferSize) { + return PathError(ENOMEM, "Readlink on '{0}' relative to '{1}'", path, + dfd_); + } + large_buffer.resize(next_buffer_size); + result = readlinkat(dfd_, path.c_str(), large_buffer.data(), + large_buffer.size()); + if (result == -1) { + return PathError(errno, "Readlink on '{0}' relative to '{1}'", path, + dfd_); + } + } + + // Fix-up the size of the string and return it. + large_buffer.resize(result); + return large_buffer; +} + +auto MakeTmpDir() -> ErrorOr { + std::filesystem::path tmpdir_path = "/tmp"; + // We use both `TEST_TMPDIR` and `TMPDIR`. The `TEST_TMPDIR` is set by Bazel + // and preferred to keep tests using the expected output tree rather than + // the system temporary directory. + for (const char* tmpdir_env_name : {"TEST_TMPDIR", "TMPDIR"}) { + const char* tmpdir_env_cstr = getenv(tmpdir_env_name); + if (tmpdir_env_cstr == nullptr) { + continue; + } + std::filesystem::path tmpdir_env = std::string(tmpdir_env_cstr); + if (!tmpdir_env.is_absolute()) { + continue; + } + tmpdir_path = std::move(tmpdir_env); + break; + } + + std::filesystem::path target = BuildData::BuildTarget.str(); + std::string dir_name = target.filename().native(); + dir_name += ".XXXXXX"; + + tmpdir_path /= dir_name; + + std::string tmpdir_path_buffer = tmpdir_path.native(); + char* result = mkdtemp(tmpdir_path_buffer.data()); + if (result == nullptr) { + RawStringOstream os; + os << llvm::formatv("Calling mkdtemp on '{0}' failed: ", + tmpdir_path.native()); + PrintErrorNumber(os, errno); + return Error(os.TakeStr()); + } + CARBON_CHECK(result == tmpdir_path_buffer.data(), + "`mkdtemp` used a modified path"); + tmpdir_path = std::move(tmpdir_path_buffer); + + // Because `mkdtemp` doesn't return an open directory atomically, open the + // created directory and perform safety checks similar to `OpenDir` when + // creating a new directory. + CARBON_ASSIGN_OR_RETURN( + Dir tmp, Cwd().OpenDir(tmpdir_path, OpenExisting, /*creation_mode=*/0, + OpenFlags::NoFollow)); + // Make sure we try to remove the directory from here on out. + RemovingDir result_dir(std::move(tmp), tmpdir_path); + + // It's a bit awkward to report `fstat` errors as `Error`s, but we + // don't have much choice. The stat failing here would be very weird. + CARBON_ASSIGN_OR_RETURN(FileStatus stat, result_dir.Stat()); + + // The permissions must be exactly 0700 for a temporary directory, and the UID + // should be ours. + if (stat.permissions() != 0700 && stat.unix_uid() != geteuid()) { + return Error( + llvm::formatv("Found incorrect permissions or UID on tmpdir '{0}'", + tmpdir_path.native()) + .str()); + } + + return result_dir; +} + +} // namespace Carbon::Filesystem diff --git a/common/filesystem.h b/common/filesystem.h new file mode 100644 index 000000000000..7eae873167b5 --- /dev/null +++ b/common/filesystem.h @@ -0,0 +1,1536 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef CARBON_COMMON_FILESYSTEM_H_ +#define CARBON_COMMON_FILESYSTEM_H_ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "common/check.h" +#include "common/error.h" +#include "common/ostream.h" +#include "common/raw_string_ostream.h" +#include "common/template_string.h" +#include "llvm/ADT/ScopeExit.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/FormatVariadic.h" + +// Provides a filesystem library for use in the Carbon project. +// +// This library provides an API designed to support modern Unix / Linux / POSIX +// style filesystem operations, often called "Unix-like"[1] here, efficiently +// and securely, while also carefully staying to a set of abstractions and +// operations that can be reasonably implemented even on Windows platforms. +// +// TODO: Currently, there is not a Windows implementation, but this is actively +// desired when we have testing infrastructure in place for Windows development. +// Lacking that testing infrastructure and a full Windows port, the operations +// here are manually compared with LLVM's filesystem library to ensure a +// reasonable Windows implementation is possible. +// +// The library uses C++'s `std::filesystem::path` as its abstraction for +// paths. This library provides two core APIs: open directories and files. +// +// Open directories provide relative- and absolute-path based operations to open +// other directories or files. This allows secure creation of directories even +// in the face of adversarial operations for example in a shared `/tmp` +// directory. There is a `constexpr` current working directory available as +// `Cwd()` that models normal filesystem operations with paths. +// +// Open files provide read, write, and other operations on the file. There are +// separate types for read-only, write-only, and read-write files to model the +// different APIs available. +// +// The APIs for both directories and files are primarily on `*Ref` types that +// model a non-owning reference to the directory or file. These types are the +// preferred types to use on an API boundary. Owning versions are provided that +// ensure the file or directory is closed on destruction. Files support explicit +// closing in order to observe any close-specific errors. +// +// Where APIs require flag parameters of some form, this library provides +// enumerations that model those flags. The enumeration values are in turn +// chosen to simplify passing these to specific native APIs. This means the +// enumeration *values* should not be expected to be portable across platforms. +// Customizing the values is part of the larger TODO to port the implementation +// to Windows. +// +// [1]: Note that we refer to platforms as "Unix-like" rather than POSIX as we +// want to group together all the OSes where the Unix-derived APIs are the +// primary and expected way to interact with the filesystem, regardless of +// whether a POSIX conforming API happens to exist. For example, both macOS +// and WSL (Windows Subsystem for Linux) _are_ Unix-like as those are the +// primary APIs used to access files in those environments. But Windows +// itself _isn't_ Unix-like, even considering things like the defunct NT +// POSIX subsystem or modern WSL, as those aren't the primary filesystem +// APIs for the (non-WSL) Windows platform. This also matches the rough OS +// classification used in LLVM. +namespace Carbon::Filesystem { + +// The different creation options available when opening a file or directory. +// +// Because these are by far the most common parameters and they have unambiguous +// names, the enumerators are also available directly within the namespace. +enum class CreationOptions { + // Requires an existing file or directory. + OpenExisting = 0, + + // Opens an existing file or directory, and create one otherwise. + OpenAlways = O_CREAT, + + // Opens and truncates an existing file or creates a new file. Provides + // consistent behavior of an empty file regardless of the starting state. This + // cannot be used for directories as they cannot be truncated on open. This is + // essentially a short-cut for using `OpenAlways` and passing the + // `OpenFlags::Truncate` below. + CreateAlways = O_CREAT | O_TRUNC, + + // Requires no existing file or directory and will error if one is found. Only + // succeeds when it creates a new file or directory. + CreateNew = O_CREAT | O_EXCL, +}; +using enum CreationOptions; + +// General flags to control the behavior of opening files that aren't covered by +// other more specific flags. +// +// These can be combined using the `|` operator where the semantics are +// compatible, although not all are. +enum class OpenFlags : int { + None = 0, + + // Open the file for appending rather than with the position at the start. + // + // An error to combine with `Truncate` or to use with `CreateAlways`. + Append = O_APPEND, + + // Open the file and truncate its contents to be empty. + Truncate = O_TRUNC, + + // Don't follow a symlink in the final path component being opened. + NoFollow = O_NOFOLLOW, +}; +inline auto operator|(OpenFlags lhs, OpenFlags rhs) -> OpenFlags { + return static_cast(static_cast(lhs) | static_cast(rhs)); +} +inline auto operator|=(OpenFlags& lhs, OpenFlags rhs) -> OpenFlags& { + lhs = lhs | rhs; + return lhs; +} + +// Flags controlling which permissions should be checked in an `Access` call. +// +// These permissions can also be combined with the `|` operator, so +// `AccessCheckFlags::Read | AccessCheckFlags::Write` checks for both read and +// write access. +enum class AccessCheckFlags : int { + Exists = F_OK, + Read = R_OK, + Write = W_OK, + Execute = X_OK, +}; +inline auto operator|(AccessCheckFlags lhs, AccessCheckFlags rhs) + -> AccessCheckFlags { + return static_cast(static_cast(lhs) | + static_cast(rhs)); +} +inline auto operator|=(AccessCheckFlags& lhs, AccessCheckFlags rhs) + -> AccessCheckFlags& { + lhs = lhs | rhs; + return lhs; +} + +// The underlying integer type that should be used to model the mode of a file. +// +// The mode is used in this API to represent both the permission bit mask and +// special properties of a file. For example, on Unix-like systems, it combines +// permissions with set-user-ID, set-group-ID, and sticky bits. +// +// The permission bits in the mode are represented using the Unix-style bit +// pattern that facilitates octal modeling: +// - Owner bit mask: 0700 +// - Group bit mask: 0070 +// - All bit mask: 0007 +// +// For each, read is an octal value of `1`, write `2`, and execute `4`. +// +// Windows gracefully degrades to the effective permissions modeled using +// these values. +using ModeType = mode_t; + +// Enumeration of the different file types recognized. +// +// In addition to the specific type values being arranged for ease of use with +// the POSIX APIs, the underlying type of the enum is arranged to use the common +// mode type. +enum class FileType : ModeType { + // Portable file types that need to be supported across platform + // implementations. + Directory = S_IFDIR, + RegularFile = S_IFREG, + SymbolicLink = S_IFLNK, + + // Non-portable Unix-like platform specific types. + UnixFifo = S_IFIFO, + UnixCharDevice = S_IFCHR, + UnixBlockDevice = S_IFBLK, + UnixSocket = S_IFSOCK, + + // Mask for the Unix-like types to allow easy extraction. + UnixMask = S_IFMT, +}; + +// Enumerates the different open access modes available. +// +// These are largely used to parameterize types in order to constrain which API +// subset is available, and rarely needed directly. +enum class OpenAccess { + ReadOnly = O_RDONLY, + WriteOnly = O_WRONLY, + ReadWrite = O_RDWR, +}; + +// Forward declarations of various types that appear in APIs. +class DirRef; +class Dir; +class RemovingDir; +template +class FileRef; +template +class File; +class FdError; +class PathError; +namespace Internal { +class FileRefBase; +} // namespace Internal + +// Returns a constant `Dir` object that models the open current working +// directory. +// +// Whatever the working directory of the process is will be used as the base for +// any relative path operations on this object. For example, on Unix-like +// systems, `Cwd().Stat("some/path")` is equivalent to `stat("some/path")`. +consteval auto Cwd() -> Dir; + +// Creates a temporary directory and returns a removing directory handle to it. +// +// Each directory created will be unique and newly created by the call. It is +// the caller's responsibility to clean up this directory. +auto MakeTmpDir() -> ErrorOr; + +// Class modeling a file (or directory) status information structure. +// +// This provides a largely-portable model that callers can use, as well as a few +// APIs to access non-portable implementation details when necessary. +class FileStatus { + public: + // The size of the file in bytes. + auto size() const -> int64_t { return stat_buf_.st_size; } + + auto type() const -> FileType { + return static_cast(stat_buf_.st_mode & + static_cast(FileType::UnixMask)); + } + + // Convenience predicates to test for specific values of `type()`. + auto is_dir() const -> bool { return type() == FileType::Directory; } + auto is_file() const -> bool { return type() == FileType::RegularFile; } + auto is_symlink() const -> bool { return type() == FileType::SymbolicLink; } + + // The read, write, and execute permissions for user, group, and others. See + // the `ModeType` documentation for how to interpret the result. + auto permissions() const -> ModeType { return stat_buf_.st_mode & 0777; } + + // Non-portable APIs only available on Unix-like systems. See the + // documentation of the Unix `stat` structure fields they expose for their + // meaning. + auto unix_inode() const -> uint64_t { return stat_buf_.st_ino; } + auto unix_uid() const -> uid_t { return stat_buf_.st_uid; } + + private: + friend DirRef; + friend Internal::FileRefBase; + + FileStatus() = default; + + struct stat stat_buf_ = {}; +}; + +// The base class defining the core `File` API. +// +// While not used directly, this is the base class used to implement all of the +// main `File` types: `ReadFileRef`, `WriteFileRef`, and `ReadWriteFileRef`. +// +// Objects using this type have access to an open file handle to a specific file +// and expose operations on that open file. These operations may fail directly +// with their `ErrorOr` return, but some errors may be deferred until the +// underlying owning file is closed. +// +// The type provides reference semantics to the underlying file, but is +// rebindable, movable, and copyable unlike a C++ language reference. +class Internal::FileRefBase { + public: + // This object can be default constructed, but will hold an invalid file + // handle in that case. This is to support rebinding operations. + FileRefBase() = default; + + // Reads the file status. + // + // Analogous to the Unix-like `fstat` call. + auto Stat() -> ErrorOr; + + // Methods to seek the current file position, with various semantics for the + // offset. + auto Seek(int64_t delta) -> ErrorOr; + auto SeekFromBeginning(int64_t delta_from_beginning) + -> ErrorOr; + auto SeekFromEnd(int64_t delta_from_end) -> ErrorOr; + + // Reads as much data as is available and fits into the provided buffer. + // + // On success, this returns a new slice from the start to the end of the + // successfully read bytes. These will always be located in the passed-in + // buffer, but not all of the buffer may be filled. A partial read does not + // mean that the end of the file has been reached. + // + // When a successful read with an *empty* slice is returned, that represents + // reaching EOF on the underlying file successfully and there is no more data + // to read. + // + // This method retries `EINTR` on Unix-like systems and returns + // other errors to the caller. + auto ReadToBuffer(llvm::MutableArrayRef buffer) + -> ErrorOr, FdError>; + + // Writes as much data as possible from the provided buffer. + // + // On success, this returns a new slice of the *unwritten* bytes still present + // in the buffer. An empty return represents a successful write of all bytes + // in the buffer. A non-empty return does not represent an error or the + // inability to finish writing. + // + // This method retries `EINTR` on Unix-like systems and returns + // other errors to the caller. + auto WriteFromBuffer(llvm::ArrayRef buffer) + -> ErrorOr, FdError>; + + // Reads the file until EOF into the returned string. + // + // This method will retry any recoverable errors and work to completely read + // the file contents up to first encountering EOF. + // + // Any non-recoverable errors are returned to the caller. + auto ReadToString() -> ErrorOr; + + // Writes a string into the file starting from the current position. + // + // This method will retry any recoverable errors and work to completely write + // the provided content into the file. + // + // Any non-recoverable errors are returned to the caller. + auto WriteFromString(llvm::StringRef str) -> ErrorOr; + + protected: + explicit FileRefBase(int fd) : fd_(fd) {} + + // Note: this should only be used or made part of the public API by subclasses + // that provide *ownership* of the open file. It is implemented here to + // provide a single, non-templated implementation. + auto Close() && -> ErrorOr; + + // Factored out code to destroy an open read-only file. This calls `Close` + // above but ignores any errors as there is no risk of data loss for a + // read-only file. + // + // Note: this is a private API that should not be made public, and should only + // be used by the implementation of subclass destructors. It should also only + // be called for subclasses with *ownership* of the file reference, and is + // provided here as a single non-template implementation. + auto ReadOnlyDestroy() -> void; + + // Factored out code to destroy an open writable file. This _requires_ the + // file to have already been closed with an explicit `Close` call, where it + // can report any errors. Without that, destroying a writable file can easily + // result in unnoticed data loss. + // + // Note: this is a private API that should not be made public, and should only + // be used by the implementation of subclass destructors. It should also only + // be called for subclasses with *ownership* of the file reference, and is + // provided here as a single non-template implementation. + auto WriteableDestroy() -> void; + + // State representing a potentially open file. + // + // On POSIX systems, this will be a file descriptor. For moved-from and + // default-constructed file objects this may be an invalid negative value to + // signal that state. + // + // TODO: This should be customized on non-POSIX systems. + // + // This member is made protected rather than private as the derived classes + // need direct access to it in several contexts. + // NOLINTNEXTLINE(misc-non-private-member-variables-in-classes) + int fd_ = -1; +}; + +// A non-owning reference to an open file. +// +// Instances model a reference to an open file. Generally, rather than using a +// `WriteFile&`, code should use a `WriteFileRef`. +// +// A specific instance provides the subset of the file API suitable for its +// access based on its template parameter: read, write, or both. +// +// The API for file references is factored into a base class +// `Internal::FileRefBase` to avoid duplication for each access instantiation. +// Only the methods that are constrained by access are defined here, and they +// are defined as wrappers around methods in the base where the documentation +// and implementation live. +template +class FileRef : public Internal::FileRefBase { + public: + static constexpr bool Readable = + A == OpenAccess::ReadOnly || A == OpenAccess::ReadWrite; + static constexpr bool Writeable = + A == OpenAccess::WriteOnly || A == OpenAccess::ReadWrite; + + // This object can be default constructed, but will hold an invalid file + // handle in that case. This is to support rebinding operations. + FileRef() = default; + + // Read and Write methods that delegate to the `FileRefBase` implementations, + // but require the relevant access. See the methods on `FileRefBase` for full + // documentation. + auto ReadToBuffer(llvm::MutableArrayRef buffer) + -> ErrorOr, FdError> + requires Readable; + auto WriteFromBuffer(llvm::ArrayRef buffer) + -> ErrorOr, FdError> + requires Writeable; + auto ReadToString() -> ErrorOr + requires Readable; + auto WriteFromString(llvm::StringRef str) -> ErrorOr + requires Writeable; + + protected: + friend File; + friend DirRef; + + // Other constructors from the base are also available, but remain protected. + using FileRefBase::FileRefBase; +}; + +// Convenience type defs for the three access combinations. +using ReadFileRef = FileRef; +using WriteFileRef = FileRef; +using ReadWriteFileRef = FileRef; + +// An owning handle to an open file. +// +// This extends the `FileRef` API to provide ownership of the file handle. Most +// of the API is defined by `FileRef`. +// +// The file will be closed when the object is destroyed, and must close without +// errors. If there is a chance of errors on close, and that is often where +// errors are reported, code must use the `Close` API to directly handle them or +// it must be correct to check-fail on them. +// +// This type allows intentional "slicing" to the `FileRef` base class as that is +// a correct and safe conversion to pass a non-owning reference to a file to +// another function, much like binding a reference to an owning type is +// implicit. +template +class File : public FileRef { + public: + static constexpr bool Readable = + A == OpenAccess::ReadOnly || A == OpenAccess::ReadWrite; + static constexpr bool Writeable = + A == OpenAccess::WriteOnly || A == OpenAccess::ReadWrite; + + // Default constructs an invalid file. + // + // This can be destroyed or assigned safely, but no other operations are + // correct. + File() = default; + + // File objects are move-only as they model ownership. + File(File&& arg) noexcept : FileRef(std::exchange(arg.fd_, -1)) {} + auto operator=(File&& arg) noexcept -> File& { + Destroy(); + this->fd_ = std::exchange(arg.fd_, -1); + return *this; + } + File(const File&) = delete; + auto operator=(const File&) -> File& = delete; + ~File() { Destroy(); } + + // Closes the open file and leaves the file in a moved-from state. + // + // The signature is `auto Close() && -> ErrorOr`. + // + // This type provides ownership of the file, so expose the `Close` method to + // allow checked destruction and release of the file resources. + // + // If any errors are encountered during closing, returns them. Note that the + // file should still be considered closed, and the object is moved-from even + // if errors occur. + using Internal::FileRefBase::Close; + + private: + friend DirRef; + + // Destroy the file. + // + // This dispatches to non-template code in `FileRefBase` based on whether the + // file is writable or readonly. The core logic is in the non-template + // methods. + auto Destroy() -> void; + + explicit File(int fd) : FileRef(fd) {} +}; + +// Convenience type defs for the three access combinations. +using ReadFile = File; +using WriteFile = File; +using ReadWriteFile = File; + +// A non-owning reference to an open directory. +// +// This is the main API for accessing and opening files and other directories. +// Conceptually, every open file or directory is relative to some other +// directory. The symbolic current working directory object is available via the +// `Cwd()` function. When on a Unix-like platform, this is intended to provide +// the semantics of `openat` and related functions, including the ability to +// write secure filesystem operations in the face of adversarial parallel +// filesystem operations. +// +// Relative path parameters are always relative to this directory. Absolute path +// parameters are also allowed and are treated as absolute paths. This parallels +// the behavior of `/` for path concatenation where an absolute path ignores all +// preceding components. +// +// Errors for directory operations retain the path parameter used in order to +// print helpful detail when unhandled, but otherwise work to be lazy and +// lightweight to support low-overhead expected error patterns. +// +// The names are designed to mirror the underlying Unix-like APIs that implement +// them, with extensions to add clarity. However, the set of operations is +// expected to be reasonable to implement on Windows with reasonable fidelity. +class DirRef { + public: + class Entry; + class Iterator; + class Reader; + + // Begin reading the entries in a directory. + // + // This returns a `Reader` object that can be iterated to walk over all the + // entries in this directory. Note that the returned `Reader` owns a newly + // allocated handle to this directory, and provides the full `DirRef` API. If + // it isn't necessary to keep both open, the `Dir` class offers a + // move-qualified overload that optimizes this case. + // + // Note that it is unspecified whether added and removed files during the + // lifetime of the reader will be included when iterating, but otherwise + // concurrent mutations are well defined. + auto Read() & -> ErrorOr; + + // Checks that the provided path can be accessed. + auto Access(const std::filesystem::path& path, + AccessCheckFlags check = AccessCheckFlags::Exists) + -> ErrorOr; + + // Reads the `FileStatus` for the open directory. + auto Stat() -> ErrorOr; + + // Reads the `FileStatus` for the provided path (without opening it). + // + // Like the `stat` system call on Unix-like platforms, this will follow any + // symlinks and provide the status of the underlying file or directory. + auto Stat(const std::filesystem::path& path) + -> ErrorOr; + + // Reads the `FileStatus` for the provided path (without opening it). + // + // Like the `lstat` system call on Unix-like platforms, this will *not* follow + // symlinks, and instead will return the status of the symlink itself. + auto Lstat(const std::filesystem::path& path) + -> ErrorOr; + + // Reads the target string of the symlink at the provided path. + // + // This does not follow the symlink, and does not require the symlink target + // to be valid or exist. It merely reads the textual string. + // + // Returns an error if called with a path that is not a symlink. + auto Readlink(const std::filesystem::path& path) + -> ErrorOr; + + // Opens the provided path as a read-only file. + // + // The interaction with an existing file is governed by `creation_options` and + // defaults to error unless opening an existing file. When creating a file, + // only the leaf component in the provided path can be created with this call. + // + // If creating a file, the file is created with `creation_mode` which defaults + // to a restrictive `0600`. The creation permission bits are also completely + // independent of the access provided via the opened file. For example, + // creating with write permissions doesn't impact whether write access is + // available via the returned file. And creating _without_ write permission + // bits is compatible with opening the file for writing. + // + // Additional flags can be provided to `flags` to control other aspects of + // behavior on open. + // + // This is an error if the path exists and is a directory. If the path is a + // symlink, it will follow the symlink. + auto OpenReadOnly(const std::filesystem::path& path, + CreationOptions creation_options = OpenExisting, + ModeType creation_mode = 0600, + OpenFlags flags = OpenFlags::None) + -> ErrorOr; + + // Opens the provided path as a write-only file. Otherwise, behaves as + // `OpenReadOnly`. + auto OpenWriteOnly(const std::filesystem::path& path, + CreationOptions creation_options = OpenExisting, + ModeType creation_mode = 0600, + OpenFlags flags = OpenFlags::None) + -> ErrorOr; + + // Opens the provided path as a read-and-write file. Otherwise, behaves as + // `OpenReadOnly`. + auto OpenReadWrite(const std::filesystem::path& path, + CreationOptions creation_options = OpenExisting, + ModeType creation_mode = 0600, + OpenFlags flags = OpenFlags::None) + -> ErrorOr; + + // Opens the provided path as a directory. + // + // Similar to `OpenReadOnly` and other file opening APIs, accepts + // `creation_options` to control the interaction with any existing directory. + // However, `CreateAlways` is not implementable for directories and an error + // if passed. The default permissions in the `creation_mode` are `0700` which + // is more suitable for directories. There are no extra flags that can be + // passed. + // + // As with other open routines, when creating a directory, only the leaf + // component can be created by the call to this routine. + // + // When creating a directory with `CreateNew`, this routine works to be safe + // even in the presence of adversarial, concurrent operations that attempt to + // replace the created directory with one that is controlled by the adversary. + // + // Specifically, for `CreateNew` we ensure that the last component is a + // created directory in its parent, and cannot be replaced by a symlink into + // an attacker-controlled directory. We further ensure it cannot have been + // replaced by a directory with a different owner or with wider permissions + // than the created directory. + // + // However, no validation is done on any prefix path components leading to the + // leaf component created. When securely creating directories, the initial + // creation should have a single component from an opened existing parent + // directory. Also, no validation of the owning _group_ is performed. When + // securely creating a directory, the caller should either ensure the parent + // directory does not have a malicious setgid bit set, or restrict the + // created mode to not give group access, or both. In general, the lack of + // control over the owning group motivates our choice to make the default mode + // permissions restrictive and not include any group access. + // + // To securely achieve a result similar to `OpenAlways` instead of + // `CreateNew`, callers can directly `CreateNew` and handle failures with an + // explicit `OpenExisting` that also blocks following symlinks with + // `OpenFlags::NoFollow` and performs any needed validation. + auto OpenDir(const std::filesystem::path& path, + CreationOptions creation_options = OpenExisting, + ModeType creation_mode = 0700, OpenFlags flags = OpenFlags::None) + -> ErrorOr; + + // Reads the file at the provided path to a string. + // + // This is a convenience wrapper for opening the path, reading the returned + // file to a string, and closing it. Errors from any step are returned. + auto ReadFileToString(const std::filesystem::path& path) + -> ErrorOr; + + // Writes the provided `content` to the provided path. + // + // This is a convenience wrapper for opening the path, creating it according + // to `creation_options` as necessary, writing `content` to it, and closing + // it. Errors from any step are returned. + auto WriteFileFromString(const std::filesystem::path& path, + llvm::StringRef content, + CreationOptions creation_options = CreateAlways) + -> ErrorOr; + + // Changes the current working directory to this directory. + auto Chdir() -> ErrorOr; + + // Changes the current working directory to the provided path. + // + // An error if the provided path is not a directory. Does not open the + // provided path as a directory, but it will be available as the current + // working directory via `Cwd()`. + auto Chdir(const std::filesystem::path& path) -> ErrorOr; + + // Creates a symlink at the provided path with the contents of `target`. + // + // Note that the target of a symlink is an arbitrary string and there is no + // error checking on whether it exists or is sensible. Also, the target string + // set will be up to the first null byte in `target`, regardless of its + // `size`. This will not overwrite an existing symlink at the provided path. + // + // Also note that the written symlink will be the null-terminated string + // `target.c_str()`, ignoring everything past any embedded null bytes. + auto Symlink(const std::filesystem::path& path, const std::string& target) + -> ErrorOr; + + // Creates the directories in the provided path, using the permissions in + // `creation_mode`. + // + // This will create any missing directory components in `path`. Relative paths + // will be created relative to this directory, and without re-resolving its + // path. The leaf created directory is opened and returned. + // + // The implementation allows for concurrent creation of the same directory (or + // a prefix) without error or corruption and optimizes for performance of + // creating the requested path. As a consequence, this creation is _unsafe_ in + // the face of adversarial concurrent manipulation of components of the path. + // If you need to create directories securely, first create an initial + // directory securely using `OpenDir` and `CreateNew` with restricted + // permissions that preclude any adversarial behavior, then use this API to + // create tree components within that root. + auto CreateDirectories(const std::filesystem::path& path, + ModeType creation_mode = 0700) + -> ErrorOr; + + // Unlink the last component of the path, removing that name from its parent + // directory. + // + // If this was the last link to the underlying file its contents will be + // removed when the last open file handle to it is closed. + // + // The path must not be a directory. If the path is a symbolic link, the link + // will be removed, not the target. Models the behavior of `unlinkat(2)` on + // Unix-like platforms. + auto Unlink(const std::filesystem::path& path) -> ErrorOr; + + // Remove the directory entry of the last component of the path. + // + // The path must be a directory, and that directory must be empty. Models + // `rmdirat(2)` on Unix-like platforms. + auto Rmdir(const std::filesystem::path& path) -> ErrorOr; + + // Remove the directory tree identified by the last component of the path. + // + // The provided path must name a directory. This removes all files and + // subdirectories contained within that named directory and then removes the + // directory itself once empty. + auto Rmtree(const std::filesystem::path& path) -> ErrorOr; + + protected: + constexpr DirRef() = default; + constexpr explicit DirRef(int dfd) : dfd_(dfd) {} + + // Slow-path fallback when unable to read the symlink target into a small + // stack buffer. + auto ReadlinkSlow(const std::filesystem::path& path) + -> ErrorOr; + + // Generic implementation of the various `Open*` variants using the + // `OpenAccess` enumerator. + template + auto OpenImpl(const std::filesystem::path& path, + CreationOptions creation_options, ModeType creation_mode, + OpenFlags flags) -> ErrorOr, PathError>; + + // State representing an open directory. + // + // On POSIX systems, this will be a file descriptor. For moved-from and + // default-constructed file objects this may be an invalid negative value to + // signal that state. + // + // TODO: This should be customized on non-POSIX systems. + // + // The directory's file descriptor is part of the protected API. + // NOLINTNEXTLINE(misc-non-private-member-variables-in-classes): + int dfd_ = -1; +}; + +// An owning handle to an open directory. +// +// This extends the `DirRef` API to provide ownership of the directory. Most of +// the API is defined by `DirRef`. It additionally provides optimized move-based +// variations on those APIs where relevant. +// +// The directory will be closed when the object is destroyed. Closing an open +// directory isn't an interesting error reporting path and so no direct close +// API is provided. +// +// This type allows intentional "slicing" to the `DirRef` base class as that is +// a correct and safe conversion to pass a non-owning reference to a directory +// to another function, much like binding a reference to an owning type is +// implicit. +class Dir : public DirRef { + public: + Dir() = default; + + // Dir objects are move-only as they model ownership. + Dir(Dir&& arg) noexcept : DirRef(std::exchange(arg.dfd_, -1)) {} + auto operator=(Dir&& arg) noexcept -> Dir& { + Destroy(); + dfd_ = std::exchange(arg.dfd_, -1); + return *this; + } + Dir(const Dir&) = delete; + auto operator=(const Dir&) -> Dir& = delete; + constexpr ~Dir(); + + // An optimized way to read the entries in a directory when moving from an + // owning `Dir` object. + // + // This avoids creating a duplicate file handle for the returned `Reader`. + // That `Reader` also supports the full `DirRef` API and so can often be used + // without retaining the original `Dir`. + // + // For more details about reading, see the documentation on `DirRef::Read`. + auto TakeAndRead() && -> ErrorOr; + + // Also include `DirRef`'s read API. + using DirRef::Read; + + private: + friend consteval auto Cwd() -> Dir; + friend DirRef; + friend RemovingDir; + + explicit constexpr Dir(int dfd) : DirRef(dfd) {} + + // Prevent implicit creation of a `Dir` object from a `RemovingDir` which will + // end up as a subclass below and represent harmful implicit slicing. Instead, + // require friendship and an explicit construction on an _intended_ release of + // the removing semantics. + explicit Dir(RemovingDir&& arg) noexcept; + + constexpr auto Destroy() -> void; +}; + +// An owning handle to an open directory and its absolute path that will be +// removed recursively when destroyed. +// +// This can be used to ensure removal of a directory, and also exposes the +// absolute path of the directory. +// +// As removal may encounter errors, unless the desired behavior is a +// check-failure, users should explicitly move and call `Remove` at the end of +// lifetime and handle any resultant errors. +class RemovingDir : public Dir { + public: + // Takes ownership of the open directory `d` and wraps it in a `RemovingDir` + // that will remove it on destruction using `abs_path`. Requires `abs_path` to + // be an absolute path and the desired path to remove on destruction. + // + // Note that there is no way for the implementation to validate what directory + // `abs_path` refers to, that is the responsibility of the caller. + explicit RemovingDir(Dir d, std::filesystem::path abs_path) + : Dir(std::move(d)), abs_path_(std::move(abs_path)) { + CARBON_CHECK(abs_path_.is_absolute(), "Relative path used for removal: {0}", + abs_path_); + } + + RemovingDir() = default; + RemovingDir(RemovingDir&& arg) = default; + auto operator=(RemovingDir&& rhs) -> RemovingDir& = default; + ~RemovingDir(); + + auto abs_path() const [[clang::lifetimebound]] + -> const std::filesystem::path& { + return abs_path_; + } + + // Releases the directory from being removed and returns just the underlying + // owning handle. + auto Release() && -> Dir { return std::move(*this); } + + // Removes the directory immediately and surfaces any errors encountered. + auto Remove() && -> ErrorOr; + + private: + friend Dir; + + std::filesystem::path abs_path_; +}; + +// A named entry in a directory. +// +// This provides access to the scanned data when reading the entries of the +// directory. It can only be produced by iterating over a `DirRef::Reader`. +class DirRef::Entry { + public: + // The name of the entry. + // + // This is exposed as a null-terminated C-string as that is the most common + // representation. + auto name() const -> const char* { return dent_->d_name; } + + // Test if the entry has an unknown type. In this case, all other type + // predicates will return false and the caller will have to directly `Lstat()` + // the entry to determine its type. + auto is_unknown_type() const -> bool { return dent_->d_type == DT_UNKNOWN; } + + // Predicates to test for known entry types. + // + // Note that we don't provide an enumerator here as we don't have any reliable + // way to predict the set of possible values or narrow to that set. Different + // platforms and even different versions of the same header may change the set + // of types surfaced here. + auto is_known_dir() const -> bool { return dent_->d_type == DT_DIR; } + auto is_known_regular_file() const -> bool { return dent_->d_type == DT_REG; } + auto is_known_symlink() const -> bool { return dent_->d_type == DT_LNK; } + + private: + friend Dir::Reader; + friend Dir::Iterator; + + Entry() = default; + explicit Entry(dirent* dent) : dent_(dent) {} + + dirent* dent_ = nullptr; +}; + +// An iterator into a `DirRef::Reader`, used for walking the entries in a +// directory. +// +// Most of the work of iterating a directory is done when constructing the +// `Reader`, when constructing the beginning iterator, or when incrementing the +// iterator. +class DirRef::Iterator + : public llvm::iterator_facade_base { + public: + // Default construct a general end iterator. + Iterator() = default; + + auto operator==(const Iterator& rhs) const -> bool { + CARBON_DCHECK(dirp_ == nullptr || rhs.dirp_ == nullptr || + dirp_ == rhs.dirp_); + return entry_.dent_ == rhs.entry_.dent_; + } + + auto operator*() const [[clang::lifetimebound]] -> const Entry& { + return entry_; + } + auto operator++() -> Iterator&; + + private: + friend Dir::Reader; + + // Construct a begin iterator for a specific directory stream. + explicit Iterator(DIR* dirp) : dirp_(dirp) { + // Increment immediately to populate the initial entry. + ++*this; + } + + DIR* dirp_ = nullptr; + Entry entry_; +}; + +// A reader for a directory. +// +// This class owns a handle to a directory that is set up for reading the +// entries within the directory. Because it owns a handle to the directory, it +// also implements the full `DirRef` API for convenience. +// +// Beyond the `DirRef` API, this object can be iterated as a range to visit all +// the entries in the directory. +// +// Note that it is unspecified whether entries added or removed prior to being +// visited while iterating. Iterating also cannot be re-started once begun -- +// this models an input iterable range, not even a forward iterable range. +// +// This type allows intentional "slicing" to the `DirRef` base class as that is +// a correct and safe conversion to pass a non-owning reference to a directory +// to another function, much like binding a reference to an owning type is +// implicit. +class DirRef::Reader : public DirRef { + public: + Reader() = default; + Reader(Reader&& arg) noexcept + // The directory file descriptor isn't owning, but clear it for clarity. + : DirRef(std::exchange(arg.dfd_, -1)), + dirp_(std::exchange(arg.dirp_, nullptr)) {} + Reader(const Reader&) = delete; + auto operator=(Reader&& arg) noexcept -> Reader& { + Destroy(); + // The directory file descriptor isn't owning, but clear it for clarity. + dfd_ = std::exchange(arg.dfd_, -1); + dirp_ = std::exchange(arg.dirp_, nullptr); + return *this; + } + ~Reader() { Destroy(); } + + // Compute the begin and end iterators for reading the entries of the + // directory. + auto begin() -> Iterator; + auto end() -> Iterator; + + private: + friend DirRef; + friend Dir; + + explicit Reader(DIR* dirp) : DirRef(dirfd(dirp)), dirp_(dirp) {} + auto Destroy() -> void; + + DIR* dirp_ = nullptr; +}; + +namespace Internal { +// Base class for `errno` errors. +// +// This is where we extract common APIs and logic for querying the specific +// `errno`-based error. +template +class ErrnoErrorBase : public ErrorBase { + public: + // Accessors to test for specific kinds of errors that are portably available. + auto already_exists() const -> bool { return errnum_ == EEXIST; } + auto is_dir() const -> bool { return errnum_ == EISDIR; } + auto no_entity() const -> bool { return errnum_ == ENOENT; } + auto not_dir() const -> bool { return errnum_ == ENOTDIR; } + auto access_denied() const -> bool { return errnum_ == EACCES; } + + // Specific to `Rmdir` operations, two different error values can be used. + auto not_empty() const -> bool { + return errnum_ == ENOTEMPTY || errnum_ == EEXIST; + } + + // Accessor for the `errno` based error number. This is not a portable API, + // code using it will need to be ported to use a different API on Windows. + // TODO: Add a Windows-specific API for its low-level error information. + auto unix_errnum() const -> int { return errnum_; } + + protected: + // NOLINTNEXTLINE(bugprone-crtp-constructor-accessibility): + explicit ErrnoErrorBase(int errnum) : errnum_(errnum) {} + + private: + int errnum_; +}; +} // namespace Internal + +// Error from a file-descriptor operation. +// +// This is the implementation of the file-descriptor-based error type. When +// operations on a file descriptor fail, they use this object to convey the +// error plus the descriptor in question. +// +// Specific context on the exact point or nature of the operation that failed +// can be included in the custom format string. The format string should include +// a placeholder for the file descriptor to be substituted into. The format +// string should describe the _operation_ that failed, once rendered it will +// have `failed: ` and a description of the `errno`-indicated failure appended. +// +// For example: +// +// `FdError(EPERM, "Read of file '{0}'", 42)` +// +// Will be rendered similarly to: +// +// "Read of file '42' failed: EPERM: ..." +class FdError : public Internal::ErrnoErrorBase { + public: + FdError(FdError&&) noexcept = default; + auto operator=(FdError&&) noexcept -> FdError& = default; + + // Prints this error to the provided string. + // + // Works to render the `errno` in a friendly way and includes the file + // descriptor for context. + auto Print(llvm::raw_ostream& out) const -> void; + + private: + friend Internal::FileRefBase; + friend ReadFile; + friend WriteFile; + friend ReadWriteFile; + friend DirRef; + friend Dir; + + explicit FdError(int errnum, llvm::StringLiteral format, int fd) + : ErrnoErrorBase(errnum), fd_(fd), format_(format) {} + + int fd_; + llvm::StringLiteral format_; +}; + +// Error from a path-based operation. +// +// This is the implementation of the path-based error type. When operations on a +// path fail, they use this object to convey the error plus both the path and +// relevant directory FD leading to the failure. +// +// Specific context on the exact point or nature of the operation that failed +// can be included in the custom format string. The format string should include +// placeholders for the path and the directory file descriptor to be substituted +// into. The format string should describe the _operation_ that failed, once +// rendered it will have `failed: ` and a description of the `errno`-indicated +// failure appended. +// +// For example: +// +// `PathError(EPERM, "Open of '{0}' relative to '{1}'", "filename", 42)` +// +// Will be rendered similarly to: +// +// "Open of 'filename' relative to '42' failed: EPERM: ..." +class PathError : public Internal::ErrnoErrorBase { + public: + PathError(PathError&&) noexcept = default; + auto operator=(PathError&&) noexcept -> PathError& = default; + + // Prints this error to the provided string. + // + // Works to render the `errno` in a friendly way and includes the path and + // directory file descriptor for context. + auto Print(llvm::raw_ostream& out) const -> void; + + private: + friend DirRef; + friend Dir; + + explicit PathError(int errnum, llvm::StringLiteral format, + std::filesystem::path path, int dir_fd) + : ErrnoErrorBase(errnum), + dir_fd_(dir_fd), + path_(std::move(path)), + format_(format) {} + + int dir_fd_; + std::filesystem::path path_; + llvm::StringLiteral format_; +}; + +// Implementation details only below. + +consteval auto Cwd() -> Dir { return Dir(AT_FDCWD); } + +inline auto Internal::FileRefBase::Stat() -> ErrorOr { + FileStatus status; + if (fstat(fd_, &status.stat_buf_) == 0) { + return status; + } + return FdError(errno, "File::Stat on '{0}'", fd_); +} + +inline auto Internal::FileRefBase::Seek(int64_t delta) + -> ErrorOr { + int64_t byte_offset = lseek(fd_, delta, SEEK_CUR); + if (byte_offset == -1) { + return FdError(errno, "File::Seek on '{0}'", fd_); + } + return byte_offset; +} + +inline auto Internal::FileRefBase::SeekFromBeginning( + int64_t delta_from_beginning) -> ErrorOr { + int64_t byte_offset = lseek(fd_, delta_from_beginning, SEEK_SET); + if (byte_offset == -1) { + return FdError(errno, "File::SeekTo on '{0}'", fd_); + } + return byte_offset; +} + +inline auto Internal::FileRefBase::SeekFromEnd(int64_t delta_from_end) + -> ErrorOr { + int64_t byte_offset = lseek(fd_, delta_from_end, SEEK_END); + if (byte_offset == -1) { + return FdError(errno, "File::SeekFromEnd on '{0}'", fd_); + } + return byte_offset; +} + +inline auto Internal::FileRefBase::ReadToBuffer( + llvm::MutableArrayRef buffer) + -> ErrorOr, FdError> { + for (;;) { + ssize_t read_bytes = read(fd_, buffer.data(), buffer.size()); + if (read_bytes == -1) { + if (errno == EINTR) { + continue; + } + return FdError(errno, "File::Read on '{0}'", fd_); + } + return buffer.slice(0, read_bytes); + } +} + +inline auto Internal::FileRefBase::WriteFromBuffer( + llvm::ArrayRef buffer) + -> ErrorOr, FdError> { + for (;;) { + ssize_t written_bytes = write(fd_, buffer.data(), buffer.size()); + if (written_bytes == -1) { + if (errno == EINTR) { + continue; + } + return FdError(errno, "File::Write on '{0}'", fd_); + } + return buffer.drop_front(written_bytes); + } +} + +inline auto Internal::FileRefBase::Close() && -> ErrorOr { + // Put the file in a moved-from state immediately as it is invalid to + // retry closing or use the file in any way even if the close fails. + int fd = std::exchange(fd_, -1); + + int result = close(fd); + if (result == 0) { + return Success(); + } + + return FdError(errno, "File::Close on '{0}'", fd); +} + +inline auto Internal::FileRefBase::ReadOnlyDestroy() -> void { + if (fd_ >= 0) { + auto result = std::move(*this).Close(); + // Intentionally drop errors, as there is no interesting error here. There + // is no risk of data loss, and the least bad thing we can do is to just + // leak the file descriptor. + static_cast(result); + } +} + +inline auto Internal::FileRefBase::WriteableDestroy() -> void { + CARBON_CHECK( + fd_ == -1, + "Cannot destroy an open writable file, they _must_ be destroyed by " + "calling `Close` and handling any errors to avoid data loss."); +} + +template +auto FileRef::ReadToBuffer(llvm::MutableArrayRef buffer) + -> ErrorOr, FdError> + requires Readable +{ + return FileRefBase::ReadToBuffer(buffer); +} + +template +auto FileRef::ReadToString() -> ErrorOr + requires Readable +{ + return FileRefBase::ReadToString(); +} + +template +auto FileRef::WriteFromBuffer(llvm::ArrayRef buffer) + -> ErrorOr, FdError> + requires Writeable +{ + return FileRefBase::WriteFromBuffer(buffer); +} + +template +auto FileRef::WriteFromString(llvm::StringRef str) + -> ErrorOr + requires Writeable +{ + return FileRefBase::WriteFromString(str); +} + +template +auto File::Destroy() -> void { + if constexpr (Writeable) { + this->WriteableDestroy(); + } else { + this->ReadOnlyDestroy(); + } +} + +inline auto DirRef::Read() & -> ErrorOr { + int dup_dfd = dup(dfd_); + if (dup_dfd == -1) { + // There are very few plausible errors here, but we can return one so it + // doesn't hurt to do so. While `EINTR` and `EBUSY` are mentioned in some + // documentation, there is no indication that for just `dup` it is useful to + // loop and retry. + return FdError(errno, "Dir::Read on '{0}'", dfd_); + } + return Dir(dup_dfd).TakeAndRead(); +} + +inline auto DirRef::Access(const std::filesystem::path& path, + AccessCheckFlags check) -> ErrorOr { + if (faccessat(dfd_, path.c_str(), static_cast(check), /*flags=*/0) == + 0) { + return true; + } + return PathError(errno, "Dir::Access on '{0}' relative to '{1}'", path, dfd_); +} + +inline auto DirRef::Stat() -> ErrorOr { + FileStatus status; + if (fstat(dfd_, &status.stat_buf_) == 0) { + return status; + } + return FdError(errno, "Dir::Stat on '{0}': ", dfd_); +} + +inline auto DirRef::Stat(const std::filesystem::path& path) + -> ErrorOr { + FileStatus status; + if (fstatat(dfd_, path.c_str(), &status.stat_buf_, /*flags=*/0) == 0) { + return status; + } + return PathError(errno, "Dir::Stat on '{0}' relative to '{1}'", path, dfd_); +} + +inline auto DirRef::Lstat(const std::filesystem::path& path) + -> ErrorOr { + FileStatus status; + if (fstatat(dfd_, path.c_str(), &status.stat_buf_, + /*flags=*/AT_SYMLINK_NOFOLLOW) == 0) { + return status; + } + return PathError(errno, "Dir::Lstat on '{0}' relative to '{1}'", path, dfd_); +} + +inline auto DirRef::Readlink(const std::filesystem::path& path) + -> ErrorOr { + // On the fast path, we read into a small stack buffer and get the whole + // contents. + constexpr ssize_t BufferSize = 256; + char buffer[BufferSize]; + ssize_t read_bytes = readlinkat(dfd_, path.c_str(), buffer, BufferSize); + if (read_bytes == -1) { + return PathError(errno, "Dir::Readlink on '{0}' relative to '{1}'", path, + dfd_); + } + if (read_bytes < BufferSize) { + // We got the whole contents in one shot, return it. + return std::string(buffer, read_bytes); + } + + // Otherwise, fallback to an out-of-line function to handle the slow path. + return ReadlinkSlow(path); +} + +inline auto DirRef::OpenReadOnly(const std::filesystem::path& path, + CreationOptions creation_options, + ModeType creation_mode, OpenFlags flags) + -> ErrorOr { + return OpenImpl(path, creation_options, creation_mode, + flags); +} + +inline auto DirRef::OpenWriteOnly(const std::filesystem::path& path, + CreationOptions creation_options, + ModeType creation_mode, OpenFlags flags) + -> ErrorOr { + return OpenImpl(path, creation_options, creation_mode, + flags); +} + +inline auto DirRef::OpenReadWrite(const std::filesystem::path& path, + CreationOptions creation_options, + ModeType creation_mode, OpenFlags flags) + -> ErrorOr { + return OpenImpl(path, creation_options, creation_mode, + flags); +} + +inline auto DirRef::Chdir() -> ErrorOr { + if (fchdir(dfd_) == -1) { + return FdError(errno, "Dir::Chdir on '{0}'", dfd_); + } + return Success(); +} + +inline auto DirRef::Chdir(const std::filesystem::path& path) + -> ErrorOr { + if (path.is_absolute()) { + if (chdir(path.c_str()) == -1) { + return PathError(errno, "Dir::Chdir on '{0}' relative to '{1}'", path, + dfd_); + } + return Success(); + } + + CARBON_ASSIGN_OR_RETURN(Dir d, OpenDir(path)); + auto result = d.Chdir(); + if (result.ok()) { + return Success(); + } + return PathError(result.error().unix_errnum(), + "Dir::Chdir on '{0}' relative to '{1}'", path, dfd_); +} + +inline auto DirRef::Symlink(const std::filesystem::path& path, + const std::string& target) + -> ErrorOr { + if (symlinkat(target.c_str(), dfd_, path.c_str()) == -1) { + return PathError(errno, "Dir::Symlink on '{0}' relative to '{1}'", path, + dfd_); + } + return Success(); +} + +inline auto DirRef::Unlink(const std::filesystem::path& path) + -> ErrorOr { + if (unlinkat(dfd_, path.c_str(), /*flags=*/0) == -1) { + return PathError(errno, "Dir::Unlink on '{0}' relative to '{1}'", path, + dfd_); + } + return Success(); +} + +inline auto DirRef::Rmdir(const std::filesystem::path& path) + -> ErrorOr { + if (unlinkat(dfd_, path.c_str(), AT_REMOVEDIR) == -1) { + return PathError(errno, "Dir::Rmdir on '{0}' relative to '{1}'", path, + dfd_); + } + return Success(); +} + +template +inline auto DirRef::OpenImpl(const std::filesystem::path& path, + CreationOptions creation_options, + ModeType creation_mode, OpenFlags flags) + -> ErrorOr, PathError> { + for (;;) { + int fd = openat(dfd_, path.c_str(), + static_cast(A) | static_cast(creation_options) | + static_cast(flags), + creation_mode); + if (fd == -1) { + // May need to retry on `EINTR` when opening FIFOs on Linux. + if (errno == EINTR) { + continue; + } + return PathError(errno, "Dir::Open on '{0}' relative to '{1}'", path, + dfd_); + } + return File(fd); + } +} + +constexpr Dir::~Dir() { Destroy(); } + +inline auto Dir::TakeAndRead() && -> ErrorOr { + // Transition our file descriptor into a directory stream, clearing it in the + // process. + int dfd = std::exchange(dfd_, -1); + DIR* dirp = fdopendir(dfd); + if (dirp == nullptr) { + return FdError(errno, "Dir::Read on '{0}'", dfd); + } + return Dir::Reader(dirp); +} + +inline Dir::Dir(RemovingDir&& arg) noexcept : Dir(static_cast(arg)) { + arg.abs_path_.clear(); +} + +constexpr auto Dir::Destroy() -> void { + if (dfd_ != -1 && dfd_ != AT_FDCWD) { + auto result = close(dfd_); + // Closing a directory shouldn't produce errors, directly check fail on any. + // + // This is a very different case from `close` on a file producing an error. + // We don't actually write through the directory file descriptor, and for + // most platforms `closedir` (the closest thing in documentation and + // exclusively about directories), only provides a very few possible errors + // here: + // + // EBADF: This should be precluded by the types here, and so we consider + // it a programming error. + // + // EINTR: Technically, a system could fail here. We have good evidence + // that systems we practically support don't as there also is nothing + // useful to *do* in the face of this: retrying on almost all systems + // is not allowed as the file descriptor is immediately released. And + // here, there is no potentially dropped data to report. + // + // If we ever discover a platform that fails here, we should adjust this + // code to not fail in the face of that, likely by dropping the error. If we + // end up supporting a platform that actually requires well-specified + // retries, this code should handle that. Until then, we require these to + // succeed so we will learn about any issues during porting to new + // platforms. + CARBON_CHECK(result == 0, "{0}", + FdError(errno, "Dir::Destroy on '{0}'", dfd_)); + } + dfd_ = -1; +} + +inline RemovingDir::~RemovingDir() { + if (dfd_ != -1) { + auto result = std::move(*this).Remove(); + CARBON_CHECK(result.ok(), "{0}", result.error()); + } +} + +inline auto RemovingDir::Remove() && -> ErrorOr { + CARBON_CHECK(dfd_ != -1, + "Unexpected explicit remove on a `RemovingDir` with no owned " + "directory!"); + + // Close the directory base object prior to removing it. + static_cast(*this) = Dir(); + return Cwd().Rmtree(abs_path_); +} + +inline auto Dir::Iterator::operator++() -> Iterator& { + CARBON_CHECK(dirp_, "Cannot increment an end-iterator"); + + errno = 0; + entry_.dent_ = readdir(dirp_); + // There are no documented errors beyond an erroneous `dirp_` which would be + // a programming error and not due to any recoverable failure of the + // filesystem. + CARBON_CHECK(entry_.dent_ != nullptr || errno == 0, + "Using a directory iterator with a non-directory, errno '{0}'", + errno); + if (entry_.dent_ == nullptr) { + // Clear the directory pointer to ease debugging increments past the end. + dirp_ = nullptr; + } + return *this; +} + +inline auto Dir::Reader::begin() -> Iterator { return Iterator(dirp_); } + +inline auto Dir::Reader::end() -> Iterator { return Iterator(); } + +inline auto Dir::Reader::Destroy() -> void { + if (dirp_) { + int result = closedir(dirp_); + // Closing a directory shouldn't produce interesting errors, so check fail + // on them directly. + // + // See the detailed comment on `Dir::Destroy` for more context on closing of + // directories, why we check-fail, and what we should do if we discover + // platforms where an error needs to be handled here. + CARBON_CHECK(result == 0, "{0}", + FdError(errno, "Dir::Reader::Destroy on '{0}'", dfd_)); + dirp_ = nullptr; + dfd_ = -1; + } +} + +} // namespace Carbon::Filesystem + +#endif // CARBON_COMMON_FILESYSTEM_H_ diff --git a/common/filesystem_benchmark.cpp b/common/filesystem_benchmark.cpp new file mode 100644 index 000000000000..51dda1befe96 --- /dev/null +++ b/common/filesystem_benchmark.cpp @@ -0,0 +1,544 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include + +#include +#include + +#include "absl/random/random.h" +#include "common/filesystem.h" +#include "llvm/ADT/Sequence.h" +#include "llvm/ADT/StringExtras.h" + +namespace Carbon::Filesystem { +namespace { + +// Alternative implementation strategies to allow comparing performance. +// +// WHen implementing benchmarks below, we try to make them templates on this +// enum and then switch in the body between different implementations. This +// allows us to share the framework of each benchmark but select different +// implementations for different instantiations. The different instantiations +// get these enumerators in their names in the output, so we keep them short. +enum BenchmarkComparables { + Carbon, + Std, +}; + +// Filler text. +constexpr llvm::StringLiteral Text = + "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod " + "tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim " + "veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea " + "commodo consequat. Duis aute irure dolor in reprehenderit in voluptate " + "velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint " + "occaecat cupidatat non proident, sunt in culpa qui officia deserunt " + "mollit anim id est laborum."; + +// Gets the filler text repeated up to a specific length. +static auto GetText(int length) -> std::string { + std::string content; + content.reserve(length); + while (static_cast(content.size()) < length) { + content += Text.substr(0, length - content.size()); + } + CARBON_CHECK(static_cast(content.size()) == length); + return content; +} + +// We build a collection of file paths to use across different benchmarks in +// batches to avoid looking at the same file over and over again. We can even +// shuffle the file orders to further avoid hiding performance cost. If there +// are specific cases where we want to measure the cached / predicted speed, we +// can write those benchmarks against a specific file, but most often we instead +// look at the worst case scenario for wall-clock time and use cycle counters +// and instruction counters to measure aspects of the best case. The exact +// number here was chosen arbitrarily to not make running benchmarks excessively +// slow due to the large batches. +constexpr int NumFiles = 64; + +// A common set of context used in benchmarks below. A separate context object +// works better than the benchmark fixture support in practice. +// +// This is a struct as there are no invariants or contracts enforced. This is +// just a container of commonly useful data and commonly useful helper +// functions. +struct BenchContext { + RemovingDir tmpdir; + absl::BitGen rng; + std::array file_paths; + std::array missing_paths; + + BenchContext() : tmpdir(std::move(*MakeTmpDir())) { + for (int i : llvm::seq(NumFiles)) { + file_paths[i] = llvm::formatv("file_{0}", i).str(); + auto result = tmpdir.WriteFileFromString(file_paths[i], Text); + CARBON_CHECK(result.ok(), "{0}", result.error()); + missing_paths[i] = llvm::formatv("missing_{0}", i).str(); + } + ShuffleFilePaths(); + ShuffleMissingPaths(); + } + + auto ShuffleFilePaths() -> void { + std::shuffle(file_paths.begin(), file_paths.end(), rng); + } + + auto ShuffleMissingPaths() -> void { + std::shuffle(missing_paths.begin(), missing_paths.end(), rng); + } + + // Create a tree of files and directories starting from a `base` new directory + // in our tmp directory, and containing `entries` total entries with + // `entries_per_dir` in each directory. These will be a mixture of further + // subdirectories and files. + auto CreateTree(std::filesystem::path base, int entries, int entries_per_dir) + -> void { + CARBON_CHECK(entries >= 1); + CARBON_CHECK(entries_per_dir >= 1); + int num_subdirs = std::max(entries_per_dir / 2, 1); + struct DirStackEntry { + Dir dir; + int num_entries; + int subdir_count; + }; + llvm::SmallVector dir_stack; + auto d = tmpdir.OpenDir(base, CreationOptions::CreateNew); + CARBON_CHECK(d.ok(), "{0}", d.error()); + dir_stack.push_back({std::move(*d), entries, 0}); + + while (!dir_stack.empty()) { + auto& [dir, num_entries, subdir_count] = dir_stack.back(); + + // We want `num_entries` transitively in this directory, and + // `entries_per_dir` directly. Spread the remaining entries across + // `num_subdirs`. + int entries_per_subdir = ((num_entries - entries_per_dir) / num_subdirs); + CARBON_CHECK(entries_per_subdir < num_entries); + + // While we'll still put entries in a subdirectory, and we still need more + // subdirectories in this directory, create another subdirectory, push it + // on the stack, and recurse to it by continuing. + if (entries_per_subdir >= entries_per_dir && subdir_count < num_subdirs) { + auto name = llvm::formatv("dir_{0}", subdir_count).str(); + auto subdir = dir.OpenDir(name, CreationOptions::CreateNew); + CARBON_CHECK(subdir.ok(), "{0}", subdir.error()); + ++subdir_count; + + // Note we have to continue after `push_back` as this will invalidate + // the current references. + dir_stack.push_back({std::move(*subdir), entries_per_subdir, 0}); + continue; + } + + // Otherwise, we're finished with subdirectories and just need to create + // direct files. + int num_files = entries_per_dir - subdir_count; + CARBON_CHECK(num_files >= 0); + for (int i = 0; i < num_files; ++i) { + auto name = llvm::formatv("file_{0}", i).str(); + auto f = dir.OpenWriteOnly(name, CreationOptions::CreateNew); + CARBON_CHECK(f.ok(), "{0}", f.error()); + auto close_result = std::move(*f).Close(); + CARBON_CHECK(close_result.ok(), "{0}", close_result.error()); + } + dir_stack.pop_back(); + } + } +}; + +template +auto BM_Access(benchmark::State& state) -> void { + BenchContext context; + while (state.KeepRunningBatch(NumFiles)) { + for (int i : llvm::seq(NumFiles)) { + if constexpr (Comp == Carbon) { + auto result = context.tmpdir.Access(context.file_paths[i]); + CARBON_CHECK(result.ok(), "{0}", result.error()); + } else if constexpr (Comp == Std) { + std::error_code ec; + bool exists = std::filesystem::exists( + context.tmpdir.abs_path() / context.file_paths[i], ec); + CARBON_CHECK(!ec, "{0}", ec.message()); + CARBON_CHECK(exists); + } else { + static_assert(false, "Invalid benchmark comparable"); + } + } + } +} +BENCHMARK(BM_Access)->UseRealTime(); +BENCHMARK(BM_Access)->UseRealTime(); + +template +auto BM_AccessMissing(benchmark::State& state) -> void { + BenchContext context; + while (state.KeepRunningBatch(NumFiles)) { + for (int i : llvm::seq(NumFiles)) { + if constexpr (Comp == Carbon) { + auto result = context.tmpdir.Access(context.missing_paths[i]); + CARBON_CHECK(result.error().no_entity()); + } else if constexpr (Comp == Std) { + std::error_code ec; + auto exists = std::filesystem::exists( + context.tmpdir.abs_path() / context.missing_paths[i], ec); + CARBON_CHECK(!ec, "{0}", ec.message()); + CARBON_CHECK(!exists); + } else { + static_assert(false, "Invalid benchmark comparable"); + } + } + } +} +BENCHMARK(BM_AccessMissing)->UseRealTime(); +BENCHMARK(BM_AccessMissing)->UseRealTime(); + +template +auto BM_Stat(benchmark::State& state) -> void { + BenchContext context; + while (state.KeepRunningBatch(NumFiles)) { + for (int i : llvm::seq(NumFiles)) { + if constexpr (Comp == Carbon) { + auto status = context.tmpdir.Stat(context.file_paths[i]); + CARBON_CHECK(status.ok(), "{0}", status.error()); + benchmark::DoNotOptimize(status->permissions()); + } else if constexpr (Comp == Std) { + std::error_code ec; + auto status = std::filesystem::status( + context.tmpdir.abs_path() / context.file_paths[i], ec); + CARBON_CHECK(!ec, "{0}", ec.message()); + benchmark::DoNotOptimize(status.permissions()); + } else { + static_assert(false, "Invalid benchmark comparable"); + } + } + } +} +BENCHMARK(BM_Stat)->UseRealTime(); +BENCHMARK(BM_Stat)->UseRealTime(); + +template +auto BM_StatMissing(benchmark::State& state) -> void { + BenchContext context; + while (state.KeepRunningBatch(NumFiles)) { + for (int i : llvm::seq(NumFiles)) { + if constexpr (Comp == Carbon) { + auto status = context.tmpdir.Stat(context.missing_paths[i]); + CARBON_CHECK(status.error().no_entity()); + } else if constexpr (Comp == Std) { + std::error_code ec; + auto status = std::filesystem::status( + context.tmpdir.abs_path() / context.missing_paths[i], ec); + CARBON_CHECK(ec.value() == ENOENT, "{0}", ec.message()); + } else { + static_assert(false, "Invalid benchmark comparable"); + } + } + } +} +BENCHMARK(BM_StatMissing)->UseRealTime(); +BENCHMARK(BM_StatMissing)->UseRealTime(); + +template +auto BM_OpenMissing(benchmark::State& state) -> void { + BenchContext context; + while (state.KeepRunningBatch(NumFiles)) { + for (int i : llvm::seq(NumFiles)) { + if constexpr (Comp == Carbon) { + auto f = context.tmpdir.OpenReadOnly(context.missing_paths[i]); + CARBON_CHECK(f.error().no_entity()); + } else if constexpr (Comp == Std) { + std::ifstream f(context.tmpdir.abs_path() / context.missing_paths[i]); + CARBON_CHECK(!f.is_open()); + } else { + static_assert(false, "Invalid benchmark comparable"); + } + } + } +} +BENCHMARK(BM_OpenMissing)->UseRealTime(); +BENCHMARK(BM_OpenMissing)->UseRealTime(); + +template +auto BM_OpenClose(benchmark::State& state) -> void { + BenchContext context; + while (state.KeepRunningBatch(NumFiles)) { + for (int i : llvm::seq(NumFiles)) { + if constexpr (Comp == Carbon) { + auto f = context.tmpdir.OpenReadOnly(context.file_paths[i]); + CARBON_CHECK(f.ok(), "{0}", f.error()); + auto close_result = std::move(*f).Close(); + CARBON_CHECK(close_result.ok(), "{0}", close_result.error()); + } else if constexpr (Comp == Std) { + std::ifstream f(context.tmpdir.abs_path() / context.file_paths[i]); + CARBON_CHECK(f.is_open()); + } else { + static_assert(false, "Invalid benchmark comparable"); + } + } + } +} +BENCHMARK(BM_OpenClose)->UseRealTime(); +BENCHMARK(BM_OpenClose)->UseRealTime(); + +template +auto BM_CreateRemove(benchmark::State& state) -> void { + BenchContext context; + while (state.KeepRunningBatch(NumFiles)) { + for (int i : llvm::seq(NumFiles)) { + if constexpr (Comp == Carbon) { + // Create the file by opening it. + auto f = context.tmpdir.OpenWriteOnly(context.missing_paths[i], + CreationOptions::CreateNew); + CARBON_CHECK(f.ok(), "{0}", f.error()); + // Close it right away. + auto close_result = std::move(*f).Close(); + CARBON_CHECK(close_result.ok(), "{0}", close_result.error()); + // Remove it. + auto remove_result = context.tmpdir.Unlink(context.missing_paths[i]); + CARBON_CHECK(remove_result.ok(), "{0}", remove_result.error()); + } else if constexpr (Comp == Std) { + auto path = context.tmpdir.abs_path() / context.missing_paths[i]; + // Create the file by opening it. + std::ofstream f(path); + CARBON_CHECK(f.is_open()); + // Close it right away. + f.close(); + // Remove it. + std::error_code ec; + std::filesystem::remove(path, ec); + CARBON_CHECK(!ec, "{0}", ec.message()); + } else { + static_assert(false, "Invalid benchmark comparable"); + } + } + } +} +BENCHMARK(BM_CreateRemove)->UseRealTime(); +BENCHMARK(BM_CreateRemove)->UseRealTime(); + +template +auto BM_Read(benchmark::State& state) -> void { + BenchContext context; + int length = state.range(0); + std::string content = GetText(length); + for (int i : llvm::seq(NumFiles)) { + auto result = + context.tmpdir.WriteFileFromString(context.file_paths[i], content); + CARBON_CHECK(result.ok(), "{0}", result.error()); + } + while (state.KeepRunningBatch(NumFiles)) { + // Re-shuffle the order of the files for each batch to avoid exact cache + // hits. + state.PauseTiming(); + context.ShuffleFilePaths(); + state.ResumeTiming(); + + for (int i : llvm::seq(NumFiles)) { + if constexpr (Comp == Carbon) { + auto read_result = + context.tmpdir.ReadFileToString(context.file_paths[i]); + CARBON_CHECK(read_result.ok(), "{0}", read_result.error()); + benchmark::DoNotOptimize(*read_result); + } else if constexpr (Comp == Std) { + std::ifstream f(context.tmpdir.abs_path() / context.file_paths[i], + std::ios::binary); + CARBON_CHECK(f.is_open()); + // This may be a somewhat surprising implementation, but benchmarking + // against several other ways of reading the file with `std::ifstream` + // all have the same or worse performance. + std::string read_content((std::istreambuf_iterator(f)), + (std::istreambuf_iterator())); + benchmark::DoNotOptimize(read_content); + } else { + static_assert(false, "Invalid benchmark comparable"); + } + } + } +} +BENCHMARK(BM_Read)->Range(4, 1024LL * 1024)->UseRealTime(); +BENCHMARK(BM_Read)->Range(4, 1024LL * 1024)->UseRealTime(); + +template +auto BM_Write(benchmark::State& state) -> void { + BenchContext context; + int length = state.range(0); + std::string content = GetText(length); + while (state.KeepRunningBatch(NumFiles)) { + // Re-shuffle the order of the files for each batch to avoid exact cache + // hits. + state.PauseTiming(); + context.ShuffleFilePaths(); + state.ResumeTiming(); + + for (int i : llvm::seq(NumFiles)) { + if constexpr (Comp == Carbon) { + auto write_result = + context.tmpdir.WriteFileFromString(context.file_paths[i], content); + CARBON_CHECK(write_result.ok(), "{0}", write_result.error()); + } else if constexpr (Comp == Std) { + std::ofstream f(context.tmpdir.abs_path() / context.file_paths[i], + std::ios::binary | std::ios::trunc); + CARBON_CHECK(f.is_open()); + f.write(content.data(), content.length()); + } else { + static_assert(false, "Invalid benchmark comparable"); + } + } + } +} +BENCHMARK(BM_Write)->Range(4, 1024LL * 1024)->UseRealTime(); +BENCHMARK(BM_Write)->Range(4, 1024LL * 1024)->UseRealTime(); + +template +auto BM_Rmtree(benchmark::State& state) -> void { + BenchContext context; + int entries = state.range(0); + int depth = state.range(1); + + // Configure our batch size based on the number of entries. Creating large + // numbers of entries in the filesystem can cause problems, and is also very + // slow. We don't need that much accuracy once the trees get large. + int batch_size = entries <= 1024 ? 10 : entries <= (32 * 1024) ? 5 : 1; + + while (state.KeepRunningBatch(batch_size)) { + state.PauseTiming(); + for (int i : llvm::seq(batch_size)) { + context.CreateTree(llvm::formatv("tree_{0}", i).str(), entries, depth); + } + state.ResumeTiming(); + + for (int i : llvm::seq(batch_size)) { + std::string tree = llvm::formatv("tree_{0}", i).str(); + if constexpr (Comp == Carbon) { + auto rmdir_result = context.tmpdir.Rmtree(tree); + CARBON_CHECK(rmdir_result.ok(), "{0}", rmdir_result.error()); + } else if constexpr (Comp == Std) { + std::error_code ec; + std::filesystem::remove_all(context.tmpdir.abs_path() / tree, ec); + CARBON_CHECK(!ec, "{0}", ec.message()); + } else { + static_assert(false, "Invalid benchmark comparable"); + } + } + } +} +BENCHMARK(BM_Rmtree) + ->Ranges({{1, 256}, {1, 32}}) + ->Ranges({{2 * 1024, 256 * 1024}, {512, 1024}}) + ->Unit(benchmark::kMicrosecond) + ->UseRealTime(); +BENCHMARK(BM_Rmtree) + ->Ranges({{1, 256}, {1, 32}}) + ->Ranges({{2 * 1024, 256 * 1024}, {512, 1024}}) + ->Unit(benchmark::kMicrosecond) + ->UseRealTime(); + +template +auto BM_CreateDirectories(benchmark::State& state) -> void { + BenchContext context; + int depth = state.range(0); + int existing_depth = state.range(1); + CARBON_CHECK(existing_depth <= depth); + CARBON_CHECK(depth > 0); + + // Use a batch size of 10 to get avoid completely swamping the measurements + // with overhead from creating existing directories and cleaning up. + constexpr int BatchSize = 10; + + // Pre-build both the paths and the existing paths. Note that we use + // relatively short paths here, which if anything makes the benefits of the + // Carbon library smaller. + llvm::SmallVector paths; + llvm::SmallVector existing_paths; + for (int i : llvm::seq(BatchSize)) { + RawStringOstream path; + llvm::ListSeparator sep("/"); + for (int j = 0; j < existing_depth; ++j) { + path << sep << "exists_" << (j == 0 ? i : j); + } + existing_paths.push_back(path.TakeStr()); + path << existing_paths.back(); + for (int k = existing_depth; k < depth; ++k) { + path << sep << "dir_" << (k == 0 ? i : k); + } + paths.push_back(path.TakeStr()); + } + + while (state.KeepRunningBatch(BatchSize)) { + state.PauseTiming(); + for (int i : llvm::seq(BatchSize)) { + if (existing_depth > 0) { + auto result = context.tmpdir.CreateDirectories(existing_paths[i]); + CARBON_CHECK(result.ok(), "{0}", result.error()); + } + } + state.ResumeTiming(); + + for (int i : llvm::seq(BatchSize)) { + if constexpr (Comp == Carbon) { + auto result = context.tmpdir.CreateDirectories(paths[i]); + CARBON_CHECK(result.ok(), "Failed to create '{0}': {1}", paths[i], + result.error()); + + // Create a file in the provided directory. This adds some baseline + // overhead but matches the realistic use case and ensures that there + // isn't some laziness that makes just creating a directory have an + // unusually low cost. + auto f = result->OpenWriteOnly("test", CreationOptions::CreateNew); + CARBON_CHECK(f.ok(), "{0}", f.error()); + auto close_result = std::move(*f).Close(); + CARBON_CHECK(close_result.ok(), "{0}", close_result.error()); + } else if constexpr (Comp == Std) { + std::filesystem::path path = context.tmpdir.abs_path() / paths[i]; + std::error_code ec; + std::filesystem::create_directories(path, ec); + CARBON_CHECK(!ec, "{0}", ec.message()); + + // Create a file in the directory, similar to above. This has a (much) + // bigger effect though because the C++ APIs don't open the created + // directory, and so the creation cost of it can very much be hidden + // from the benchmark if we don't use it. This also lets us see the + // benefit of not needing to re-walk the path to create the file. + std::ofstream f(path / "test"); + CARBON_CHECK(f.is_open()); + f.close(); + } else { + static_assert(false, "Invalid benchmark comparable"); + } + } + + state.PauseTiming(); + for (int i : llvm::seq(BatchSize)) { + auto result = context.tmpdir.Rmtree( + llvm::formatv("{0}_{1}", existing_depth > 0 ? "exists" : "dir", i) + .str()); + CARBON_CHECK(result.ok(), "{0}", result.error()); + } + state.ResumeTiming(); + } +} +static auto CreateDirectoriesBenchArgs(benchmark::internal::Benchmark* b) { + // The first argument is the depth of directory to create. We mostly care + // about reasonably small depths here. It must be >= 1 for there to be + // something to benchmark. The second number is the depth of pre-existing + // directories which can vary from 0 to equal to the depth to benchmark the + // case of no new directory being needed. + for (int i = 1; i <= 8; i *= 2) { + b->Args({i, 0}); + for (int j = 1; j <= i; j *= 2) { + b->Args({i, j}); + } + } +} +BENCHMARK(BM_CreateDirectories) + ->Apply(CreateDirectoriesBenchArgs) + ->UseRealTime(); +BENCHMARK(BM_CreateDirectories) + ->Apply(CreateDirectoriesBenchArgs) + ->UseRealTime(); + +} // namespace +} // namespace Carbon::Filesystem diff --git a/common/filesystem_test.cpp b/common/filesystem_test.cpp new file mode 100644 index 000000000000..e4ca33378786 --- /dev/null +++ b/common/filesystem_test.cpp @@ -0,0 +1,331 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "common/filesystem.h" + +#include +#include + +#include +#include +#include + +#include "common/error_test_helpers.h" + +namespace Carbon::Filesystem { +namespace { + +using ::testing::_; +using ::testing::Eq; +using ::testing::HasSubstr; +using Testing::IsError; +using Testing::IsSuccess; + +class FilesystemTest : public ::testing::Test { + public: + explicit FilesystemTest() { + auto result = MakeTmpDir(); + CARBON_CHECK(result.ok(), "{0}", result.error()); + dir_ = std::move(*result); + } + + ~FilesystemTest() override { + auto result = std::move(dir_).Remove(); + CARBON_CHECK(result.ok(), "{0}", result.error()); + } + + auto path() const -> const std::filesystem::path& { return dir_.abs_path(); } + + // The test's temp directory, deleted on destruction. + RemovingDir dir_; +}; + +TEST_F(FilesystemTest, CreateOpenCloseAndUnlink) { + auto unlink_result = dir_.Unlink("test"); + ASSERT_FALSE(unlink_result.ok()); + EXPECT_TRUE(unlink_result.error().no_entity()); +#ifdef _GNU_SOURCE + EXPECT_THAT(unlink_result, IsError(HasSubstr("ENOENT"))); +#elif defined(__APPLE__) || defined(_POSIX_SOURCE) + EXPECT_THAT(unlink_result, IsError(HasSubstr("No such file"))); +#endif + + auto f = dir_.OpenWriteOnly("test", CreationOptions::CreateNew); + ASSERT_THAT(f, IsSuccess(_)); + auto result = (*std::move(f)).Close(); + EXPECT_THAT(result, IsSuccess(_)); + + f = dir_.OpenWriteOnly("test", CreationOptions::CreateNew); + ASSERT_FALSE(f.ok()); + EXPECT_TRUE(f.error().already_exists()); +#ifdef _GNU_SOURCE + EXPECT_THAT(f, IsError(HasSubstr("EEXIST"))); +#elif defined(__APPLE__) || defined(_POSIX_SOURCE) + EXPECT_THAT(f, IsError(HasSubstr("File exists"))); +#endif + + f = dir_.OpenWriteOnly("test"); + ASSERT_THAT(f, IsSuccess(_)); + result = std::move(*f).Close(); + EXPECT_THAT(result, IsSuccess(_)); + + f = dir_.OpenWriteOnly("test"); + ASSERT_THAT(f, IsSuccess(_)); + result = std::move(*f).Close(); + EXPECT_THAT(result, IsSuccess(_)); + + unlink_result = dir_.Unlink("test"); + EXPECT_THAT(unlink_result, IsSuccess(_)); + + f = dir_.OpenWriteOnly("test"); + EXPECT_FALSE(f.ok()); + EXPECT_TRUE(f.error().no_entity()); +#ifdef _GNU_SOURCE + EXPECT_THAT(f, IsError(HasSubstr("ENOENT"))); +#elif defined(__APPLE__) || defined(_POSIX_SOURCE) + EXPECT_THAT(f, IsError(HasSubstr("No such file"))); +#endif + + f = dir_.OpenWriteOnly("test", CreationOptions::OpenAlways); + ASSERT_THAT(f, IsSuccess(_)); + result = std::move(*f).Close(); + EXPECT_THAT(result, IsSuccess(_)); + + unlink_result = dir_.Unlink("test"); + EXPECT_THAT(unlink_result, IsSuccess(_)); +} + +TEST_F(FilesystemTest, BasicWriteAndRead) { + std::string content_str = "0123456789"; + { + auto f = dir_.OpenWriteOnly("test", CreationOptions::CreateNew); + ASSERT_THAT(f, IsSuccess(_)); + auto write_result = f->WriteFromString(content_str); + EXPECT_THAT(write_result, IsSuccess(_)); + (*std::move(f)).Close().Check(); + } + + { + auto f = dir_.OpenReadOnly("test"); + ASSERT_THAT(f, IsSuccess(_)); + auto read_result = f->ReadToString(); + EXPECT_THAT(read_result, IsSuccess(Eq(content_str))); + } + + auto unlink_result = dir_.Unlink("test"); + EXPECT_THAT(unlink_result, IsSuccess(_)); +} + +TEST_F(FilesystemTest, CreateAndRemoveDirecotries) { + auto d1 = Cwd().CreateDirectories(path() / "a" / "b" / "c" / "test1"); + ASSERT_THAT(d1, IsSuccess(_)); + auto d2 = Cwd().CreateDirectories(path() / "a" / "b" / "c" / "test2"); + ASSERT_THAT(d2, IsSuccess(_)); + auto d3 = Cwd().CreateDirectories(path() / "a" / "b" / "c" / "test3"); + ASSERT_THAT(d3, IsSuccess(_)); + // Get a directory object to use, this shouldn't cover much new. + auto d4 = Cwd().CreateDirectories(path()); + EXPECT_THAT(d4, IsSuccess(_)); + // Single, present, relative component. + auto d5 = d4->CreateDirectories("a"); + EXPECT_THAT(d5, IsSuccess(_)); + // Multiple, present, but relative components. + auto d6 = d5->CreateDirectories(std::filesystem::path("b") / "c"); + EXPECT_THAT(d6, IsSuccess(_)); + // Single new component. + auto d7 = d6->CreateDirectories("test4"); + ASSERT_THAT(d7, IsSuccess(_)); + // Two new relative components. + auto d8 = d6->CreateDirectories(std::filesystem::path("test5") / "d"); + EXPECT_THAT(d8, IsSuccess(_)); + // Mixed relative components. + auto d9 = d5->CreateDirectories(std::filesystem::path("b") / "test6"); + EXPECT_THAT(d9, IsSuccess(_)); + + { + auto f1 = d1->OpenWriteOnly("file1", CreateNew); + ASSERT_THAT(f1, IsSuccess(_)); + auto f2 = d2->OpenWriteOnly("file2", CreateNew); + ASSERT_THAT(f2, IsSuccess(_)); + auto f3 = d3->OpenWriteOnly("file3", CreateNew); + ASSERT_THAT(f3, IsSuccess(_)); + auto f4 = d7->OpenWriteOnly("file4", CreateNew); + ASSERT_THAT(f4, IsSuccess(_)); + (*std::move(f1)).Close().Check(); + (*std::move(f2)).Close().Check(); + (*std::move(f3)).Close().Check(); + (*std::move(f4)).Close().Check(); + } + + auto rm_result = Cwd().Rmtree(path() / "a"); + ASSERT_THAT(rm_result, IsSuccess(_)); +} + +TEST_F(FilesystemTest, StatAndAccess) { + auto access_result = dir_.Access("test"); + ASSERT_FALSE(access_result.ok()); + EXPECT_TRUE(access_result.error().no_entity()); + + // Make sure the flags and bit-or-ing them works in the boring case. + access_result = + dir_.Access("test", AccessCheckFlags::Read | AccessCheckFlags::Write | + AccessCheckFlags::Execute); + ASSERT_FALSE(access_result.ok()); + EXPECT_TRUE(access_result.error().no_entity()); + + auto stat_result = dir_.Stat("test"); + ASSERT_FALSE(access_result.ok()); + EXPECT_TRUE(access_result.error().no_entity()); + + // Create a file for testing, using very unusual and minimal permissions to + // help us test. Hopefully this isn't modified on the usual `umask` tests run + // under. + std::string content_str = "0123456789"; + ModeType permissions = 0450; + auto f = dir_.OpenWriteOnly("test", CreationOptions::CreateNew, permissions); + ASSERT_THAT(f, IsSuccess(_)); + auto write_result = f->WriteFromString(content_str); + EXPECT_THAT(write_result, IsSuccess(_)); + + access_result = dir_.Access("test"); + EXPECT_THAT(access_result, IsSuccess(_)); + access_result = dir_.Access("test", AccessCheckFlags::Read); + EXPECT_THAT(access_result, IsSuccess(_)); + + // Neither write nor execute permission should be present though. + access_result = dir_.Access("test", AccessCheckFlags::Write); + ASSERT_FALSE(access_result.ok()); + EXPECT_TRUE(access_result.error().access_denied()); + access_result = + dir_.Access("test", AccessCheckFlags::Read | AccessCheckFlags::Write | + AccessCheckFlags::Execute); + ASSERT_FALSE(access_result.ok()); + EXPECT_TRUE(access_result.error().access_denied()); + + stat_result = dir_.Stat("test"); + ASSERT_THAT(stat_result, IsSuccess(_)); + EXPECT_TRUE(stat_result->is_file()); + EXPECT_FALSE(stat_result->is_dir()); + EXPECT_FALSE(stat_result->is_symlink()); + EXPECT_THAT(stat_result->size(), Eq(content_str.size())); + EXPECT_THAT(stat_result->permissions(), Eq(permissions)); + + // Directory instead of file. + access_result = + dir_.Access(".", AccessCheckFlags::Read | AccessCheckFlags::Write | + AccessCheckFlags::Execute); + EXPECT_THAT(access_result, IsSuccess(_)); + + stat_result = dir_.Stat("."); + ASSERT_THAT(stat_result, IsSuccess(_)); + EXPECT_FALSE(stat_result->is_file()); + EXPECT_TRUE(stat_result->is_dir()); + EXPECT_FALSE(stat_result->is_symlink()); + + // Can remove file but still stat through the file. + auto unlink_result = dir_.Unlink("test"); + ASSERT_THAT(unlink_result, IsSuccess(_)); + auto file_stat_result = f->Stat(); + ASSERT_THAT(file_stat_result, IsSuccess(_)); + EXPECT_TRUE(file_stat_result->is_file()); + EXPECT_FALSE(file_stat_result->is_dir()); + EXPECT_FALSE(file_stat_result->is_symlink()); + EXPECT_THAT(file_stat_result->size(), Eq(content_str.size())); + EXPECT_THAT(file_stat_result->permissions(), Eq(permissions)); + (*std::move(f)).Close().Check(); +} + +TEST_F(FilesystemTest, Symlinks) { + auto readlink_result = dir_.Readlink("test"); + ASSERT_FALSE(readlink_result.ok()); + EXPECT_TRUE(readlink_result.error().no_entity()); + + auto lstat_result = dir_.Lstat("test"); + ASSERT_FALSE(lstat_result.ok()); + EXPECT_TRUE(lstat_result.error().no_entity()); + + auto symlink_result = dir_.Symlink("test", "abc"); + EXPECT_THAT(symlink_result, IsSuccess(_)); + + readlink_result = dir_.Readlink("test"); + EXPECT_THAT(readlink_result, IsSuccess(Eq("abc"))); + + symlink_result = dir_.Symlink("test", "def"); + ASSERT_FALSE(symlink_result.ok()); + EXPECT_TRUE(symlink_result.error().already_exists()); + + lstat_result = dir_.Lstat("test"); + ASSERT_THAT(lstat_result, IsSuccess(_)); + EXPECT_FALSE(lstat_result->is_file()); + EXPECT_FALSE(lstat_result->is_dir()); + EXPECT_TRUE(lstat_result->is_symlink()); + EXPECT_THAT(lstat_result->size(), Eq(strlen("abc"))); + + auto unlink_result = dir_.Unlink("test"); + EXPECT_THAT(unlink_result, IsSuccess(_)); + + readlink_result = dir_.Readlink("test"); + ASSERT_FALSE(readlink_result.ok()); + EXPECT_TRUE(readlink_result.error().no_entity()); + + // Try a symlink with null bytes for fun. This demonstrates that the symlink + // syscall only uses the leading C-string. + symlink_result = dir_.Symlink("test", std::string("a\0b\0c", 5)); + EXPECT_THAT(symlink_result, IsSuccess(_)); + readlink_result = dir_.Readlink("test"); + EXPECT_THAT(readlink_result, IsSuccess(Eq("a"))); +} + +TEST_F(FilesystemTest, Chdir) { + auto current_result = Cwd().OpenDir("."); + ASSERT_THAT(current_result, IsSuccess(_)); + + auto symlink_result = dir_.Symlink("test", "abc"); + EXPECT_THAT(symlink_result, IsSuccess(_)); + + auto chdir_result = dir_.Chdir(); + EXPECT_THAT(chdir_result, IsSuccess(_)); + auto readlink_result = Cwd().Readlink("test"); + EXPECT_THAT(readlink_result, IsSuccess(Eq("abc"))); + + auto chdir_path_result = dir_.Chdir("missing"); + ASSERT_FALSE(chdir_path_result.ok()); + EXPECT_TRUE(chdir_path_result.error().no_entity()); + + // Dangling symlink. + chdir_path_result = dir_.Chdir("test"); + ASSERT_FALSE(chdir_path_result.ok()); + EXPECT_TRUE(chdir_path_result.error().no_entity()); + + // Create a regular file and try to chdir to that. + auto f = dir_.OpenWriteOnly("test2", CreationOptions::CreateNew); + ASSERT_THAT(f, IsSuccess(_)); + auto write_result = f->WriteFromString("test2"); + EXPECT_THAT(write_result, IsSuccess(_)); + chdir_path_result = dir_.Chdir("test2"); + ASSERT_FALSE(chdir_path_result.ok()); + EXPECT_TRUE(chdir_path_result.error().not_dir()); + + auto d2_result = Cwd().OpenDir("test_d2", CreationOptions::CreateNew); + ASSERT_THAT(d2_result, IsSuccess(_)); + symlink_result = d2_result->Symlink("test2", "def"); + EXPECT_THAT(symlink_result, IsSuccess(_)); + + chdir_path_result = dir_.Chdir("test_d2"); + ASSERT_THAT(chdir_path_result, IsSuccess(_)); + readlink_result = Cwd().Readlink("test2"); + EXPECT_THAT(readlink_result, IsSuccess(Eq("def"))); + readlink_result = Cwd().Readlink("../test"); + EXPECT_THAT(readlink_result, IsSuccess(Eq("abc"))); + + chdir_result = current_result->Chdir(); + ASSERT_THAT(chdir_result, IsSuccess(_)); + readlink_result = Cwd().Readlink("test"); + ASSERT_FALSE(readlink_result.ok()); + EXPECT_TRUE(readlink_result.error().no_entity()); + (*std::move(f)).Close().Check(); +} + +} // namespace +} // namespace Carbon::Filesystem