diff --git a/toolchain/base/BUILD b/toolchain/base/BUILD index f2cee9e76fd2..5e6da8d635b9 100644 --- a/toolchain/base/BUILD +++ b/toolchain/base/BUILD @@ -239,6 +239,29 @@ cc_library( ], ) +cc_library( + name = "stable_stack", + hdrs = [ + "stable_stack.h", + ], + deps = [ + ":mem_usage", + "//common:check", + "@llvm-project//llvm:Support", + ], +) + +cc_test( + name = "stable_stack_test", + size = "small", + srcs = ["stable_stack_test.cpp"], + deps = [ + ":stable_stack", + "//testing/base:gtest_main", + "@googletest//:gtest", + ], +) + cc_library( name = "timings", hdrs = ["timings.h"], diff --git a/toolchain/base/stable_stack.h b/toolchain/base/stable_stack.h new file mode 100644 index 000000000000..adc0a2f8807d --- /dev/null +++ b/toolchain/base/stable_stack.h @@ -0,0 +1,402 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef CARBON_TOOLCHAIN_BASE_STABLE_STACK_H_ +#define CARBON_TOOLCHAIN_BASE_STABLE_STACK_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "llvm/ADT/ScopeExit.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Compiler.h" +#include "llvm/Support/MathExtras.h" +#include "toolchain/base/mem_usage.h" + +namespace Carbon { + +// A stack that never reallocates when growing so as to not invalidate pointers +// to objects that remain in the stack. +// +// As a stack, all insertions must be pushed onto the back of the stack, and +// doing so is an O(1) operation. Popping the back of the stack and peeking into +// the stack are also O(1) operations. +// +// The stack is implemented as an array of pointers to arrays. Each successive +// array gains a larger capacity than the last by multiplicative factor, to +// limit the number of allocations to be logarithmic in the number of +// insertions. But in all stacks, we use the same size progression for the +// chunks, which makes `operator==` simpler to implement. The stack uses a +// configurable value for the size of the first allocation. +// +// Like a std::vector, the stack does not free memory after Pop(). This means +// the stack retains capacity for holding the maximum number of values it ever +// held. +// +// TODO: The FirstCapacity being in the type allows for some simplifications +// internally where we can assume two stacks of the same type have the same +// chunk sizes. The typical use of StableStack is currently that we have one +// StableStack type for a given `T`, as they all have the same FirstCapacity. If +// we end up with many different StableStack types for the same `T` with +// different FirstCapacity, we should consider moving it to a runtime value. +template +class StableStack { + static_assert(FirstCapacity > 0); + static_assert(llvm::isPowerOf2_32(uint32_t{FirstCapacity}), + "FirstCapacity must be a power of 2"); + + public: + StableStack() = default; + ~StableStack() { DestroyAll(); } + + // Copying the stack. + StableStack(const StableStack& rhs) + requires(std::is_copy_constructible_v) + : used_chunks_(rhs.used_chunks_), + size_in_last_chunk_(rhs.size_in_last_chunk_) { + CopyConstructFrom(rhs); + } + auto operator=(const StableStack& rhs) -> StableStack& + requires(std::is_copy_constructible_v && std::is_copy_assignable_v) + { + if (this == &rhs) { + return *this; + } + + // TODO: We could reuse existing elements in `this` by calling `operator=` + // on them instead of destroying and reconstructing always. + // + // TODO: This is broken for assignment-from-subobject, where we would need + // to construct all of the new values before destroying the old. + DestroyAll(); + used_chunks_ = rhs.used_chunks_; + size_in_last_chunk_ = rhs.size_in_last_chunk_; + CopyConstructFrom(rhs); + return *this; + } + + // Moving the stack. The moved-from `rhs` stack will be left empty. + StableStack(StableStack&& rhs) noexcept + : chunks_(std::exchange(rhs.chunks_, {})), + used_chunks_(std::exchange(rhs.used_chunks_, 0)), + size_in_last_chunk_(std::exchange(rhs.size_in_last_chunk_, 0)) {} + auto operator=(StableStack&& rhs) noexcept -> StableStack& { + if (this == &rhs) { + return *this; + } + + // TODO: This is broken for assignment-from-subobject, where this would + // destroy the `rhs`. + DestroyAll(); + chunks_ = std::exchange(rhs.chunks_, {}); + used_chunks_ = std::exchange(rhs.used_chunks_, 0); + size_in_last_chunk_ = std::exchange(rhs.size_in_last_chunk_, 0); + return *this; + } + + // Returns a reference to the most recently pushed element at the back the + // stack. + auto Peek() const -> const T& { + CARBON_CHECK(!empty(), "Peek on an empty stack"); + auto [chunk, index] = GetIndexAtBack(); + return Get(chunk, index); + } + auto Peek() -> T& { + CARBON_CHECK(!empty(), "Peek on an empty stack"); + auto [chunk, index] = GetIndexAtBack(); + return Get(chunk, index); + } + + // Returns a reference to an element in the stack at distance `n` from the + // back of the stack. + auto Peek(int32_t n) const -> const T& { + CARBON_CHECK(!empty(), "Peek on an empty stack"); + CARBON_CHECK(n >= 0, "Peek index invalid"); + auto i = size() - 1 - n; + CARBON_CHECK(i >= 0, "Peek before the start of the stack"); + auto [chunk, index] = GetIndexFromFront(i); + return Get(chunk, index); + } + auto Peek(int32_t n) -> T& { + CARBON_CHECK(!empty(), "Peek on an empty stack"); + CARBON_CHECK(n >= 0, "Peek index invalid"); + auto i = size() - 1 - n; + CARBON_CHECK(i >= 0, "Peek before the start of the stack"); + auto [chunk, index] = GetIndexFromFront(i); + return Get(chunk, index); + } + + // Pushes a new element onto the back of the stack. + auto Push(const T& t) -> void { + auto* slot = PushSlot(); + std::construct_at(slot, t); + } + auto Push(T&& t) -> void { + auto* slot = PushSlot(); + std::construct_at(slot, std::move(t)); + } + + // Pushes a new element onto the back of the stack, constructed in place from + // the given arguments. + template + auto Emplace(Args&&... args) -> void { + auto* slot = PushSlot(); + std::construct_at(slot, std::forward(args)...); + } + + // Removes the most recently pushed element from the back of the stack. + auto Pop() -> void { + CARBON_CHECK(!empty(), "Pop on an empty stack"); + auto [chunk, index] = GetIndexAtBack(); + auto& ref = Get(chunk, index); + + ShrinkSize(); + std::destroy_at(&ref); + __asan_poison_memory_region(&ref, sizeof(T)); + } + + // Removes the most recently pushed element from the back of the stack and + // returns it. + auto PopVal() -> T { + CARBON_CHECK(!empty(), "Pop on an empty stack"); + auto [chunk, index] = GetIndexAtBack(); + auto& ref = Get(chunk, index); + + auto cleanup = llvm::scope_exit([&] { + ShrinkSize(); + std::destroy_at(&ref); + __asan_poison_memory_region(&ref, sizeof(T)); + }); + return std::move(ref); + } + + // Ensure there is memory allocated to hold at least `reserve` elements. + auto Reserve(int32_t reserve) -> void { + auto num_chunks = static_cast(NumChunksNeededForCapacity(reserve)); + chunks_.reserve(num_chunks); + while (chunks_.size() < num_chunks) { + AppendChunk(); + } + } + + // Whether there are any elements pushed onto the stack. + auto empty() const -> bool { return used_chunks_ == 0; } + // The number of elements pushed onto the stack. + auto size() const -> int32_t { + auto full_chunks_capacity = + used_chunks_ > 0 ? GetSumCapacity(used_chunks_ - 1) : 0; + return full_chunks_capacity + size_in_last_chunk_; + } + // The number of elements that the stack can hold without requiring further + // allocation. + auto capacity() const -> int32_t { return GetSumCapacity(chunks_.size()); } + + // Compares the contents of two stacks for equality. + friend auto operator==(const StableStack& lhs, const StableStack& rhs) -> bool + requires std::equality_comparable + { + if (lhs.used_chunks_ != rhs.used_chunks_) { + return false; + } + if (lhs.size_in_last_chunk_ != rhs.size_in_last_chunk_) { + return false; + } + for (auto chunk = 0; chunk < lhs.used_chunks_; ++chunk) { + const T* lhs_array = lhs.chunks_[chunk].array; + const T* rhs_array = rhs.chunks_[chunk].array; + int32_t cap = GetChunkCapacity(chunk); + auto size = chunk < lhs.used_chunks_ - 1 ? cap : lhs.size_in_last_chunk_; + if (!std::equal(lhs_array, lhs_array + size, rhs_array)) { + return false; + } + } + return true; + } + + auto CollectMemUsage(MemUsage& mem_usage, llvm::StringRef label) const + -> void { + mem_usage.Add(label.str(), size() * sizeof(T), capacity() * sizeof(T)); + } + + private: + // The total capacity of the first `num_chunks` chunks. + static constexpr auto GetSumCapacity(int32_t num_chunks) -> int32_t { + return ((1 << num_chunks) - 1) * FirstCapacity; + } + // The capacity of an individual chunk at index `chunk`. + static constexpr auto GetChunkCapacity(int32_t chunk) -> int32_t { + // We use a growth factor of 2 so that we can do bitshifts in + // GetSumCapacity() to compute total capacity in O(1) time. + return FirstCapacity << chunk; + } + // Performs the inverse of GetSumCapacity. + static constexpr auto NumChunksNeededForCapacity(int32_t cap) -> int32_t { + // Given this formula, we solve for `num_chunks`. + // cap = ((1 << num_chunks) - 1) * FirstCapacity; + auto ucap = static_cast(cap); + // Shifting by this simulates division by FirstCapacity. + auto division = llvm::Log2_32(uint32_t{FirstCapacity}); + return llvm::Log2_32_Ceil(((ucap + (FirstCapacity - 1)) >> division) + 1); + } + + struct Chunk { + T* array [[clang::require_explicit_initialization]]; + }; + + // Copy-constructs elements into this stack from `rhs`. + auto CopyConstructFrom(const StableStack& rhs) -> void { + if (rhs.empty()) { + return; + } + + chunks_.clear(); + chunks_.reserve(used_chunks_); + for (auto chunk = 0; chunk < used_chunks_; ++chunk) { + T* rhs_array = rhs.chunks_[chunk].array; + auto cap = GetChunkCapacity(chunk); + T* array = std::allocator().allocate(cap); + chunks_.push_back({.array = array}); + auto size = chunk < used_chunks_ - 1 ? cap : size_in_last_chunk_; + std::uninitialized_copy_n(rhs_array, size, array); + if (size < cap) { + __asan_poison_memory_region(array + size, (cap - size) * sizeof(T)); + } + } + } + + // Destroy all the elements in the stack and free the arrays in the chunks. + // Leaves the stack in an empty state. + auto DestroyAll() -> void { + // Make `this` appear empty while destroying its elements in case they try + // to use the stack mid-destruction. + auto size_in_last_chunk = std::exchange(size_in_last_chunk_, 0); + auto used_chunks = std::exchange(used_chunks_, 0); + auto chunks = std::exchange(chunks_, {}); + + for (auto chunk = 0; chunk < static_cast(chunks.size()); ++chunk) { + auto* array = chunks[chunk].array; + auto cap = GetChunkCapacity(chunk); + if (chunk < used_chunks - 1) { + std::destroy_n(array, cap); + } else if (chunk == used_chunks - 1) { + std::destroy_n(array, size_in_last_chunk); + } + std::allocator().deallocate(array, cap); + } + + CARBON_CHECK(chunks_.empty(), + "Mutation of StableStack while destroying its elements"); + } + + // Get the index of the element at the back of the stack. This is an O(1) + // operation. + auto GetIndexAtBack() const + -> std::pair { + int32_t chunk = used_chunks_ - 1; + int32_t index = size_in_last_chunk_ - 1; + return {chunk, index}; + } + + // Get the index of the element at position `i` from the front of the stack. + auto GetIndexFromFront(int32_t i) const + -> std::pair { + // Given a capacity, we can tell how many chunks would be needed. So we + // compute the number of chunks needed to hold a stack with capacity for + // exactly `i + 1` elements, so that `i` is the last element. Then the last + // element in that stack is the index we want to compute. + auto capacity_to_hold_index_i = i + 1; + auto num_chunks = NumChunksNeededForCapacity(capacity_to_hold_index_i); + auto capacity_before_chunk = GetSumCapacity(num_chunks - 1); + return {num_chunks - 1, i - capacity_before_chunk}; + } + + // Get the element in the specified `chunk` at position `index`. + auto Get(int32_t chunk, int32_t index) -> T& { + CARBON_DCHECK(chunk >= 0); + CARBON_DCHECK(chunk < used_chunks_); + CARBON_DCHECK(index >= 0); + auto cap = GetChunkCapacity(chunk); + auto size = chunk == used_chunks_ - 1 ? size_in_last_chunk_ : cap; + CARBON_DCHECK(index < size, + "Getting index {0} past end of last chunk with size {1}", + index, size); + return chunks_[chunk].array[index]; + } + auto Get(int32_t chunk, int32_t index) const -> const T& { + return const_cast(this)->Get(chunk, index); + } + + // Allocate memory as needed to get a pointer that we can construct `T` into. + // The pointer returned has been ASAN unpoisoned. + auto PushSlot() -> T* { + // TODO: It would be faster to make the writes to the returned slot not be + // dependent on the branches, as then the branches can be done + // asynchronously by the processor. To do that, we would need to always have + // the allocation ready for the *next* push. That means eagerly allocating + // the first chunk at construction time. And it means we can always grab a + // pointer here immediately. Then the rest of the branches and work in here + // does not affect the pointer, nor the writes through the pointer. For a + // moved-from stack we don't want to have to allocate in the RHS, so we can + // say we leave moved-from stacks with an empty set of chunks, and DCHECK + // against use-after-move. + + T* slot; + if (used_chunks_ > 0 && + size_in_last_chunk_ < GetChunkCapacity(used_chunks_ - 1)) { + // Add a slot in the last used chunk. + slot = chunks_[used_chunks_ - 1].array + size_in_last_chunk_; + ++size_in_last_chunk_; + } else if (used_chunks_ < static_cast(chunks_.size())) { + // Add a slot in an unused chunk that already exists. + slot = chunks_[used_chunks_].array; + ++used_chunks_; + size_in_last_chunk_ = 1; + } else { + // Add a slot in a new chunk. + auto& new_chunk = AppendChunk(); + slot = new_chunk.array; + ++used_chunks_; + size_in_last_chunk_ = 1; + } + __asan_unpoison_memory_region(slot, sizeof(T)); + return slot; + } + auto AppendChunk() -> Chunk& { + auto next_cap = GetChunkCapacity(chunks_.size()); + T* array = std::allocator().allocate(next_cap); + __asan_poison_memory_region(array, next_cap * sizeof(T)); + chunks_.push_back({.array = array}); + return chunks_.back(); + } + + // Shrinks the recorded size of the stack by one. Does not actually destroy + // anything. + auto ShrinkSize() -> void { + size_in_last_chunk_--; + if (size_in_last_chunk_ == 0) { + used_chunks_--; + if (used_chunks_ > 0) { + size_in_last_chunk_ = GetChunkCapacity(used_chunks_ - 1); + } else { + // Stack is empty now. + } + } + } + + llvm::SmallVector chunks_; + // The number of chunks that are being used in `chunks_`. + int32_t used_chunks_ = 0; + // The number of values in the last used chunk. + int32_t size_in_last_chunk_ = 0; +}; + +} // namespace Carbon + +#endif // CARBON_TOOLCHAIN_BASE_STABLE_STACK_H_ diff --git a/toolchain/base/stable_stack_test.cpp b/toolchain/base/stable_stack_test.cpp new file mode 100644 index 000000000000..efb7c9f79955 --- /dev/null +++ b/toolchain/base/stable_stack_test.cpp @@ -0,0 +1,554 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "toolchain/base/stable_stack.h" + +#include + +namespace Carbon { +namespace { + +TEST(StableStack, DestroyEmpty) { + StableStack stack; + EXPECT_EQ(stack.empty(), true); +} + +TEST(StableStack, Growth) { + StableStack stack; + EXPECT_EQ(stack.empty(), true); + EXPECT_EQ(stack.size(), 0); + + int i = 0; + stack.Push(i++); + EXPECT_EQ(stack.empty(), false); + EXPECT_EQ(stack.size(), i); + + auto capacity = stack.capacity(); + while (stack.capacity() == capacity) { + stack.Push(i++); + EXPECT_EQ(stack.empty(), false); + EXPECT_EQ(stack.size(), i); + } + + capacity = stack.capacity(); + while (stack.capacity() == capacity) { + stack.Push(i++); + EXPECT_EQ(stack.empty(), false); + EXPECT_EQ(stack.size(), i); + } +} + +TEST(StableStack, Emplace) { + enum Constructor { Default, Ints, String }; + struct S { + S() : ctor(Default) {} + S(int /*a*/, int /*b*/) : ctor(Ints) {} + explicit S(const char* /*s*/) : ctor(String) {} + + Constructor ctor; + }; + + StableStack stack; + stack.Emplace(); + EXPECT_EQ(stack.Peek().ctor, Default); + stack.Emplace(1, 2); + EXPECT_EQ(stack.Peek().ctor, Ints); + stack.Emplace("hi"); + EXPECT_EQ(stack.Peek().ctor, String); +} + +template +static auto GrowCapacityTimes(StableStack& stack, int n, F factory) + -> void { + while (n) { + auto capacity = stack.capacity(); + while (stack.capacity() == capacity) { + stack.Push(factory()); + } + --n; + } +} + +TEST(StableStack, PushPop) { + StableStack stack; + int i = 0; + GrowCapacityTimes(stack, 3, [&i] { return i++; }); + EXPECT_EQ(stack.size(), i); + + stack.Push(i++); + EXPECT_EQ(stack.size(), i); + + EXPECT_EQ(stack.Peek(), --i); + stack.Pop(); + EXPECT_EQ(stack.size(), i); + + // Drops below the last capacity jump. + EXPECT_EQ(stack.Peek(), --i); + stack.Pop(); + EXPECT_EQ(stack.size(), i); + + stack.Push(i++); + EXPECT_EQ(stack.size(), i); + + while (i) { + EXPECT_EQ(stack.Peek(), --i); + stack.Pop(); + EXPECT_EQ(stack.size(), i); + } + EXPECT_EQ(stack.empty(), true); +} + +TEST(StableStack, PushPopVal) { + StableStack stack; + int i = 0; + GrowCapacityTimes(stack, 3, [&i] { return i++; }); + EXPECT_EQ(stack.size(), i); + + stack.Push(i++); + EXPECT_EQ(stack.size(), i); + + EXPECT_EQ(stack.PopVal(), --i); + EXPECT_EQ(stack.size(), i); + + // Drops below the last capacity jump. + EXPECT_EQ(stack.PopVal(), --i); + EXPECT_EQ(stack.size(), i); + + stack.Push(i++); + EXPECT_EQ(stack.size(), i); + + while (i) { + EXPECT_EQ(stack.PopVal(), --i); + EXPECT_EQ(stack.size(), i); + } + EXPECT_EQ(stack.empty(), true); +} + +TEST(StableStack, EmptyPushPop) { + StableStack stack; + + stack.Push(1); + EXPECT_EQ(stack.size(), 1); + EXPECT_EQ(stack.empty(), false); + + EXPECT_EQ(stack.PopVal(), 1); + EXPECT_EQ(stack.size(), 0); + EXPECT_EQ(stack.empty(), true); + + stack.Push(2); + EXPECT_EQ(stack.size(), 1); + EXPECT_EQ(stack.empty(), false); + + EXPECT_EQ(stack.PopVal(), 2); + EXPECT_EQ(stack.size(), 0); + EXPECT_EQ(stack.empty(), true); +} + +TEST(StableStack, CopyConstructDestroy) { + int constructed = 0; + int destroyed = 0; + class Counter { + public: + Counter(int* constructed, int* destroyed) + : constructed_(constructed), destroyed_(destroyed) {} + Counter(const Counter& rhs) + : constructed_(rhs.constructed_), destroyed_(rhs.destroyed_) { + (*constructed_)++; + } + ~Counter() { (*destroyed_)++; } + + private: + int* constructed_; + int* destroyed_; + }; + + int size; + { + StableStack stack; + GrowCapacityTimes(stack, 3, + [&] { return Counter(&constructed, &destroyed); }); + EXPECT_EQ(constructed, stack.size()); + // Each Counter constructed in the lambda was destroyed. + EXPECT_EQ(destroyed, stack.size()); + size = stack.size(); + constructed = destroyed = 0; + + static_cast(stack).Peek(); + EXPECT_EQ(constructed, 0); + EXPECT_EQ(destroyed, 0); + stack.Peek(); + EXPECT_EQ(constructed, 0); + EXPECT_EQ(destroyed, 0); + } + EXPECT_EQ(constructed, 0); + // Each Counter held in the stack was destroyed. + EXPECT_EQ(destroyed, size); +} + +TEST(StableStack, MoveConstructDestroy) { + int constructed = 0; + int destroyed = 0; + class Counter { + public: + Counter(int* constructed, int* destroyed) + : constructed_(constructed), destroyed_(destroyed) {} + Counter(Counter&& rhs) noexcept + : constructed_(rhs.constructed_), destroyed_(rhs.destroyed_) { + (*constructed_)++; + } + ~Counter() { (*destroyed_)++; } + + private: + int* constructed_; + int* destroyed_; + }; + + int size; + { + StableStack stack; + GrowCapacityTimes(stack, 3, + [&] { return Counter(&constructed, &destroyed); }); + EXPECT_EQ(constructed, stack.size()); + // Each Counter constructed in the lambda was destroyed. + EXPECT_EQ(destroyed, stack.size()); + size = stack.size(); + constructed = destroyed = 0; + + static_cast(stack).Peek(); + EXPECT_EQ(constructed, 0); + EXPECT_EQ(destroyed, 0); + stack.Peek(); + EXPECT_EQ(constructed, 0); + EXPECT_EQ(destroyed, 0); + } + EXPECT_EQ(constructed, 0); + // Each Counter held in the stack was destroyed. + EXPECT_EQ(destroyed, size); +} + +TEST(StableStack, CopyStack) { + int constructed = 0; + int destroyed = 0; + class Counter { + public: + Counter(int value, int* constructed, int* destroyed) + : value_(value), constructed_(constructed), destroyed_(destroyed) {} + Counter(const Counter& rhs) + : value_(rhs.value_), + constructed_(rhs.constructed_), + destroyed_(rhs.destroyed_) { + (*constructed_)++; + } + ~Counter() { (*destroyed_)++; } + + auto operator==(const Counter& rhs) const -> bool { + return value_ == rhs.value_; + } + + private: + int value_; + int* constructed_; + int* destroyed_; + }; + + int size; + { + StableStack stack; + GrowCapacityTimes(stack, 3, [&, i = 0]() mutable { + return Counter(i++, &constructed, &destroyed); + }); + constructed = destroyed = 0; + size = stack.size(); + + // Copy construction. + StableStack stack2 = stack; + EXPECT_EQ(constructed, size); + EXPECT_EQ(destroyed, 0); + constructed = destroyed = 0; + + EXPECT_EQ(stack.size(), stack2.size()); + EXPECT_LE(stack2.capacity(), stack.capacity()); + for (auto i = 0; i < size; ++i) { + EXPECT_EQ(stack.Peek(i), stack2.Peek(i)); + } + EXPECT_EQ(constructed, 0); + EXPECT_EQ(destroyed, 0); + constructed = destroyed = 0; + + // Self-assignment. + stack2 = [&]() -> StableStack& { return stack2; }(); + EXPECT_EQ(constructed, 0); + EXPECT_EQ(destroyed, 0); + constructed = destroyed = 0; + + StableStack stack3; + EXPECT_EQ(constructed, 0); + EXPECT_EQ(destroyed, 0); + constructed = destroyed = 0; + + // Copy assignment to empty. + stack3 = stack2; + EXPECT_EQ(constructed, size); + EXPECT_EQ(destroyed, 0); + constructed = destroyed = 0; + + // Copy assignment to non-empty. We destroy the LHS and reconstruct from the + // RHS. + stack3 = stack2; + EXPECT_EQ(constructed, size); + EXPECT_EQ(destroyed, size); + constructed = destroyed = 0; + + // Copy assignment from empty. + StableStack stack4; + stack3 = stack4; + EXPECT_EQ(constructed, 0); + EXPECT_EQ(destroyed, size); + constructed = destroyed = 0; + } + EXPECT_EQ(constructed, 0); + // Each Counter held in each stack was destroyed. + EXPECT_EQ(destroyed, size * 2); +} + +TEST(StableStack, MoveStack) { + int constructed = 0; + int destroyed = 0; + class Counter { + public: + Counter(int value, int* constructed, int* destroyed) + : value_(value), constructed_(constructed), destroyed_(destroyed) {} + Counter(Counter&& rhs) noexcept + : value_(rhs.value_), + constructed_(rhs.constructed_), + destroyed_(rhs.destroyed_) { + (*constructed_)++; + } + ~Counter() { (*destroyed_)++; } + + auto value() const -> int { return value_; } + + private: + int value_; + int* constructed_; + int* destroyed_; + }; + + int size; + { + StableStack stack; + GrowCapacityTimes(stack, 3, [&, i = 0]() mutable { + return Counter(i++, &constructed, &destroyed); + }); + constructed = destroyed = 0; + size = stack.size(); + + // Move construction. + StableStack stack2 = std::move(stack); + EXPECT_EQ(constructed, 0); + EXPECT_EQ(destroyed, 0); + constructed = destroyed = 0; + + // Moving leaves the source empty. + EXPECT_EQ(stack.empty(), true); + for (auto i = 0; i < size; ++i) { + EXPECT_EQ(stack2.Peek(i).value(), size - 1 - i); + } + EXPECT_EQ(constructed, 0); + EXPECT_EQ(destroyed, 0); + constructed = destroyed = 0; + + // Self-assignment. + stack2 = [&]() -> StableStack&& { return std::move(stack2); }(); + EXPECT_EQ(constructed, 0); + EXPECT_EQ(destroyed, 0); + constructed = destroyed = 0; + + StableStack stack3; + EXPECT_EQ(constructed, 0); + EXPECT_EQ(destroyed, 0); + constructed = destroyed = 0; + + // Move assignment to empty. + stack3 = std::move(stack2); + EXPECT_EQ(constructed, 0); + EXPECT_EQ(destroyed, 0); + constructed = destroyed = 0; + + stack.Push(Counter(0, &constructed, &destroyed)); + constructed = destroyed = 0; + + // Move assignment to non-empty. We destroy the LHS. + stack = std::move(stack3); + // The single element in `stack` is destroyed. + EXPECT_EQ(constructed, 0); + EXPECT_EQ(destroyed, 1); + constructed = destroyed = 0; + + // Move assignment from empty. + stack = std::move(stack3); + EXPECT_EQ(constructed, 0); + EXPECT_EQ(destroyed, size); + constructed = destroyed = 0; + } + EXPECT_EQ(constructed, 0); + // All the stacks were empty by the end. + EXPECT_EQ(destroyed, 0); +} + +TEST(StableStack, Peek) { + StableStack stack; + const auto& const_stack = stack; + stack.Push(1); + + decltype(auto) r = const_stack.Peek(); + static_assert(std::same_as); + + decltype(auto) m = stack.Peek(); + static_assert(std::same_as); + + stack.Push(2); + EXPECT_EQ(const_stack.Peek(), 2); + EXPECT_EQ(stack.Peek(), 2); + stack.Peek() = 4; + EXPECT_EQ(const_stack.Peek(), 4); + EXPECT_EQ(stack.Peek(), 4); + EXPECT_EQ(stack.PopVal(), 4); + + EXPECT_EQ(const_stack.Peek(), 1); + EXPECT_EQ(stack.Peek(), 1); + stack.Peek() = 3; + EXPECT_EQ(const_stack.Peek(), 3); + EXPECT_EQ(stack.Peek(), 3); + EXPECT_EQ(stack.PopVal(), 3); +} + +TEST(StableStack, Reserve) { + StableStack stack; + auto capacity = stack.capacity(); + + stack.Reserve(capacity / 2); + EXPECT_EQ(stack.capacity(), capacity); + stack.Reserve(capacity); + EXPECT_EQ(stack.capacity(), capacity); + + stack.Reserve(capacity + 2); + EXPECT_GE(stack.capacity(), capacity + 2); + capacity = stack.capacity(); + stack.Reserve(1); + EXPECT_EQ(stack.capacity(), capacity); + stack.Reserve(2); + EXPECT_EQ(stack.capacity(), capacity); + + stack.Reserve(capacity + 1000); + EXPECT_GE(stack.capacity(), capacity + 1000); + EXPECT_EQ(stack.size(), 0); + EXPECT_EQ(stack.empty(), true); + + capacity = stack.capacity(); + stack.Reserve(1); + EXPECT_EQ(stack.capacity(), capacity); + stack.Reserve(0); + EXPECT_EQ(stack.capacity(), capacity); + + for (int i = 0; i < capacity; ++i) { + stack.Push(i); + } + EXPECT_EQ(stack.capacity(), capacity); + EXPECT_EQ(stack.size(), capacity); +} + +TEST(StableStack, ReserveWithFirstCapacity) { + { + StableStack stack; + EXPECT_EQ(stack.capacity(), 0); + stack.Push(0); + EXPECT_EQ(stack.capacity(), 1); + } + { + StableStack stack; + EXPECT_EQ(stack.capacity(), 0); + stack.Reserve(1); + EXPECT_EQ(stack.capacity(), 1); + stack.Reserve(2); + EXPECT_EQ(stack.capacity(), 1 + 1 * 2); + } + { + StableStack stack; + EXPECT_EQ(stack.capacity(), 0); + stack.Reserve(1); + EXPECT_EQ(stack.capacity(), 4); + stack.Reserve(4); + EXPECT_EQ(stack.capacity(), 4); + stack.Reserve(5); + EXPECT_EQ(stack.capacity(), 4 + 4 * 2); + } +} + +TEST(StableStack, Equal) { + StableStack stack; + StableStack stack2; + + EXPECT_EQ(stack, stack2); + stack.Reserve(1); + EXPECT_EQ(stack, stack2); + stack2.Reserve(1000); + EXPECT_EQ(stack, stack2); + + int i = 0; + GrowCapacityTimes(stack, 3, [&i] { return i++; }); + EXPECT_NE(stack, stack2); + + for (auto j = 0; j < stack.size(); ++j) { + stack2.Push(j); + } + EXPECT_EQ(stack, stack2); + + auto start_size = stack.size(); + while (stack.size() > start_size / 2) { + stack.Pop(); + EXPECT_NE(stack, stack2); + stack2.Pop(); + EXPECT_EQ(stack, stack2); + } + while (!stack.empty()) { + stack2.Pop(); + EXPECT_NE(stack, stack2); + stack.Pop(); + EXPECT_EQ(stack, stack2); + } +} + +TEST(StableStack, IncompleteType) { + struct S { + // Ensure we can form the StableStack type with an incomplete type + // parameter. + StableStack* stack; + }; + StableStack stack; + S _ = {.stack = &stack}; +} + +TEST(StableStack, EmptyDuringDestroy) { + bool test_empty = false; + struct S { + StableStack* stack; + bool* test_empty; + + ~S() { + if (*test_empty) { + EXPECT_TRUE(stack->empty()); + } + } + }; + StableStack stack; + GrowCapacityTimes( + stack, 3, [&] { return S{.stack = &stack, .test_empty = &test_empty}; }); + + // `S` tests that `stack` appears empty while each `S` is destroyed. + test_empty = true; +} + +} // namespace +} // namespace Carbon diff --git a/toolchain/check/BUILD b/toolchain/check/BUILD index c560a86c035d..469ca6c826f4 100644 --- a/toolchain/check/BUILD +++ b/toolchain/check/BUILD @@ -179,6 +179,7 @@ cc_library( "//toolchain/base:int", "//toolchain/base:kind_switch", "//toolchain/base:shared_value_stores", + "//toolchain/base:stable_stack", "//toolchain/base:value_ids", "//toolchain/base:value_store", "//toolchain/check:generic_region_stack", @@ -310,6 +311,7 @@ cc_library( "//common:map", "//common:ostream", "//common:vlog", + "//toolchain/base:stable_stack", "//toolchain/sem_ir:file", "//toolchain/sem_ir:typed_insts", "@llvm-project//llvm:Support", diff --git a/toolchain/check/context.h b/toolchain/check/context.h index cfd699858c74..aad74fd3bbcc 100644 --- a/toolchain/check/context.h +++ b/toolchain/check/context.h @@ -11,6 +11,7 @@ #include "common/ostream.h" #include "llvm/ADT/SmallVector.h" #include "toolchain/base/canonical_value_store.h" +#include "toolchain/base/stable_stack.h" #include "toolchain/base/value_store.h" #include "toolchain/check/core_identifier.h" #include "toolchain/check/cpp/context.h" @@ -307,7 +308,7 @@ class Context { SemIR::LocId loc_id; }; - auto where_stack() -> llvm::SmallVector& { + auto where_stack() -> StableStack& { return where_stack_; } @@ -320,7 +321,7 @@ class Context { SemIR::SpecificInterface specific_interface; auto operator==(const DeclaringImplDecl& rhs) const -> bool = default; }; - auto declaring_impl_decls() -> llvm::SmallVector& { + auto declaring_impl_decls() -> StableStack& { return declaring_impl_decls_; } @@ -644,7 +645,7 @@ class Context { // Tracks information about constraints in the current `where` expression // being checked so that they can be used by later constraints. - llvm::SmallVector where_stack_; + StableStack where_stack_; // Counts the number of `where` expressions in the type of the binding // currently being checked. @@ -653,7 +654,7 @@ class Context { // Track impl declarations that are underway. If we're declaring an impl for // `C as I`, an impl lookup query for `C as I` or `.Self as I` should find // that impl being declared (even though it does not yet exist). - llvm::SmallVector declaring_impl_decls_; + StableStack declaring_impl_decls_; // Tracks if we are currently identifying a facet type, which causes // LookupImplWitness instructions to not resolve to final witnesses by diff --git a/toolchain/check/eval.cpp b/toolchain/check/eval.cpp index b9ff0e404fe3..65764bce7736 100644 --- a/toolchain/check/eval.cpp +++ b/toolchain/check/eval.cpp @@ -3463,6 +3463,11 @@ auto TryEvalBlockForSpecific(Context& context, SemIR::LocId loc_id, // specific, we are resolving existing instructions in the eval block for a // generic. The global state should not apply to anything done while resolving // a specific. + // + // TODO: Instead of moving the whole stack here, track an index on Context + // that is the start of the stack that we're going to consider (in practice we + // only consider a single value right now, but we'd only consider the latest + // value at least at that tracked index). CheckedSaveAndRestore declaring(context.declaring_impl_decls(), {}); // The `eval_impl_lookup_to_identified_witness` is also a global state that diff --git a/toolchain/check/generic_region_stack.cpp b/toolchain/check/generic_region_stack.cpp index f82e9cfbb672..d627195225f2 100644 --- a/toolchain/check/generic_region_stack.cpp +++ b/toolchain/check/generic_region_stack.cpp @@ -14,7 +14,7 @@ auto GenericRegionStack::Push(PendingGeneric generic) -> void { pending_generic_ids_.push_back(generic); pending_eval_block_stack_.PushArray(); dependent_inst_stack_.PushArray(); - constants_in_generic_stack_.emplace_back(); + constants_in_generic_stack_.Emplace(); } auto GenericRegionStack::Pop() -> void { @@ -23,7 +23,7 @@ auto GenericRegionStack::Pop() -> void { pending.region); pending_eval_block_stack_.PopArray(); dependent_inst_stack_.PopArray(); - constants_in_generic_stack_.pop_back(); + constants_in_generic_stack_.Pop(); } } // namespace Carbon::Check diff --git a/toolchain/check/generic_region_stack.h b/toolchain/check/generic_region_stack.h index 4cf6d574064e..59b62594cad3 100644 --- a/toolchain/check/generic_region_stack.h +++ b/toolchain/check/generic_region_stack.h @@ -7,6 +7,7 @@ #include "common/array_stack.h" #include "common/map.h" +#include "toolchain/base/stable_stack.h" #include "toolchain/sem_ir/ids.h" namespace Carbon::Check { @@ -33,8 +34,8 @@ class GenericRegionStack { public: explicit GenericRegionStack(llvm::raw_ostream* vlog_stream) : vlog_stream_(vlog_stream) { - // Reserve a large enough stack that we typically won't need to reallocate. - constants_in_generic_stack_.reserve(4); + // Trigger the allocation of the first capacity specified in the type. + constants_in_generic_stack_.Reserve(1); } struct PendingGeneric { @@ -96,7 +97,7 @@ class GenericRegionStack { // instructions for the current generic. auto PeekConstantsInGenericMap() -> ConstantsInGenericMap& { CARBON_CHECK(!Empty()); - return constants_in_generic_stack_.back(); + return constants_in_generic_stack_.Peek(); } // Returns the total number of instructions tracked by all pending generic @@ -131,7 +132,9 @@ class GenericRegionStack { // Mapping from constant InstIds to the corresponding InstIds in the eval // blocks for each enclosing generic. We reserve this to a suitable size in // the constructor. - llvm::SmallVector constants_in_generic_stack_; + // + // We choose an initial capacity that typically won't need to allocate again. + StableStack constants_in_generic_stack_; }; } // namespace Carbon::Check diff --git a/toolchain/check/handle_impl.cpp b/toolchain/check/handle_impl.cpp index b0bb8696c9ad..b886a4a92a30 100644 --- a/toolchain/check/handle_impl.cpp +++ b/toolchain/check/handle_impl.cpp @@ -113,7 +113,7 @@ auto HandleParseNode(Context& context, Parse::ImplTypeAsId node_id) -> bool { // The specific interface in here, if any, is populated by the `where` // expression that introduces a `.Self` in the impl's constraint facet type. - context.declaring_impl_decls().push_back( + context.declaring_impl_decls().Push( {.self_id = self_type.type_id.AsConstantId(), .specific_interface = SemIR::SpecificInterface::None}); return true; @@ -151,7 +151,7 @@ auto HandleParseNode(Context& context, Parse::ImplDefaultSelfAsId node_id) // The specific interface in here, if any, is populated by the `where` // expression that introduces a `.Self` in the impl's constraint facet type. - context.declaring_impl_decls().push_back( + context.declaring_impl_decls().Push( {.self_id = context.constant_values().Get(self_inst_id), .specific_interface = SemIR::SpecificInterface::None}); return true; @@ -228,7 +228,7 @@ static auto BuildImplDecl(Context& context, Parse::AnyImplDeclId node_id, // Witnesses for this impl in the DeclaredFacetType of the decl will have // been evaluated to ImplSelfWitness. - context.declaring_impl_decls().pop_back(); + context.declaring_impl_decls().Pop(); // Convert the constraint expression to a type. This contains all constraints, // including rewrites and other constrains on the RHS of `where`. diff --git a/toolchain/check/handle_where.cpp b/toolchain/check/handle_where.cpp index 0b776d34a713..4383317502de 100644 --- a/toolchain/check/handle_where.cpp +++ b/toolchain/check/handle_where.cpp @@ -103,7 +103,7 @@ auto HandleParseNode(Context& context, Parse::WhereOperandId node_id) -> bool { auto impl_as_target = identified.EvaluateImplWitnessesToGetImplAsTarget(context, node_id); if (impl_as_target.is_valid()) { - auto& declaring = context.declaring_impl_decls().back(); + auto& declaring = context.declaring_impl_decls().Peek(); if (declaring.specific_interface != SemIR::SpecificInterface::None) { CARBON_CHECK(declaring.specific_interface == impl_as_target.target()); } @@ -126,7 +126,7 @@ auto HandleParseNode(Context& context, Parse::WhereOperandId node_id) -> bool { // Add a context stack for tracking constraints, that will be used to allow // later constraints to read from them eagerly. - context.where_stack().push_back({.loc_id = node_id}); + context.where_stack().Push({.loc_id = node_id}); // Track the occurrence of `where` inside a binding's type. ++context.binding_type_where_count(); @@ -146,7 +146,7 @@ auto HandleParseNode(Context& context, Parse::WhereOperandId node_id) -> bool { GetImplWitnessAccessWithoutSubstitution(context.sem_ir(), rewrite.lhs_id)); auto frozen_const_id = FreezePeriodSelf(context, const_id); - context.where_stack().back().rewrites.Insert(frozen_const_id, + context.where_stack().Peek().rewrites.Insert(frozen_const_id, rewrite.rhs_id); } } @@ -165,7 +165,7 @@ auto HandleParseNode(Context& context, Parse::WhereOperandId node_id) -> bool { GetInterfaceType(context, impls.interface_id, impls.specific_id) .AsConstantId(); auto type_frozen_const_id = FreezePeriodSelf(context, type_const_id); - context.where_stack().back().impls.push_back( + context.where_stack().Peek().impls.push_back( {.self_const_id = self_frozen_const_id, .facet_type_const_id = type_frozen_const_id}); } @@ -177,7 +177,7 @@ auto HandleParseNode(Context& context, Parse::WhereOperandId node_id) -> bool { impls.specific_id) .AsConstantId(); auto type_frozen_const_id = FreezePeriodSelf(context, type_const_id); - context.where_stack().back().impls.push_back( + context.where_stack().Peek().impls.push_back( {.self_const_id = self_frozen_const_id, .facet_type_const_id = type_frozen_const_id}); } @@ -190,7 +190,7 @@ auto HandleParseNode(Context& context, Parse::WhereOperandId node_id) -> bool { type_impls.specific_interface.specific_id) .AsConstantId(); auto type_frozen_const_id = FreezePeriodSelf(context, type_const_id); - context.where_stack().back().impls.push_back( + context.where_stack().Peek().impls.push_back( {.self_const_id = self_frozen_const_id, .facet_type_const_id = type_frozen_const_id}); } @@ -204,7 +204,7 @@ auto HandleParseNode(Context& context, Parse::WhereOperandId node_id) -> bool { type_impls.specific_named_constraint.specific_id) .AsConstantId(); auto type_frozen_const_id = FreezePeriodSelf(context, type_const_id); - context.where_stack().back().impls.push_back( + context.where_stack().Peek().impls.push_back( {.self_const_id = self_frozen_const_id, .facet_type_const_id = type_frozen_const_id}); } @@ -319,7 +319,7 @@ auto HandleParseNode(Context& context, Parse::RequirementEqualId node_id) // Note that the where_stack rewrites need to be frozen. Since this // expression is inside of facet type construction, it will already be // frozen. - context.where_stack().back().rewrites.Insert( + context.where_stack().Peek().rewrites.Insert( context.constant_values().Get( GetImplWitnessAccessWithoutSubstitution(context.sem_ir(), lhs_id)), rhs_id); @@ -451,7 +451,7 @@ auto HandleParseNode(Context& context, Parse::RequirementImplsId node_id) // Track the impls relationship so further constraints can use it // immediately, before they are evaluated. Impl lookup will search the top // of the stack. - context.where_stack().back().impls.push_back({ + context.where_stack().Peek().impls.push_back({ context.constant_values().Get(lhs_as_type.inst_id), context.constant_values().Get(rhs_as_type.inst_id), }); @@ -475,7 +475,7 @@ auto HandleParseNode(Context& context, Parse::RequirementImplsId node_id) auto lhs_id = SubstPeriodSelf( context, rhs_node, context.constant_values().Get(rewrite.lhs_id), context.constant_values().Get(lhs_as_type.inst_id)); - context.where_stack().back().rewrites.Insert(lhs_id, rewrite.rhs_id); + context.where_stack().Peek().rewrites.Insert(lhs_id, rewrite.rhs_id); } } } @@ -592,8 +592,8 @@ static auto ThawPeriodSelfInRequirements(Context& context, } auto HandleParseNode(Context& context, Parse::WhereExprId node_id) -> bool { - auto where_loc = context.where_stack().back().loc_id; - context.where_stack().pop_back(); + auto where_loc = context.where_stack().Peek().loc_id; + context.where_stack().Pop(); // Remove `PeriodSelf` from name lookup, undoing the `Push` done for the // `WhereOperand`. context.scope_stack().Pop(/*check_unused=*/true); @@ -603,7 +603,7 @@ auto HandleParseNode(Context& context, Parse::WhereExprId node_id) -> bool { if (!context.where_stack().empty()) { // This `where` expression is nested on the RHS of another `where`, which is // an error. - DiagnoseNestedWhere(context, node_id, context.where_stack().back().loc_id); + DiagnoseNestedWhere(context, node_id, context.where_stack().Peek().loc_id); type_id = SemIR::ErrorInst::TypeId; } requirements_id = CheckForNestedWhereInRequirementsAfterEval( diff --git a/toolchain/check/impl_lookup.cpp b/toolchain/check/impl_lookup.cpp index 387992e20817..daa48d993dba 100644 --- a/toolchain/check/impl_lookup.cpp +++ b/toolchain/check/impl_lookup.cpp @@ -455,7 +455,7 @@ static auto CollectFacetWitnessSources( // inner `where` expression is checked before the outer, so it needs to be // self-consistent and provide any `impls` constraints required by other // constraints. - const auto& impls = context.where_stack().back().impls; + const auto& impls = context.where_stack().Peek().impls; for (auto [self_const_id, facet_type_const_id] : impls) { self_const_id = GetCanonicalFacet(context, self_const_id); // TypeType (and ErrorInst) is never stored in the impls stack, so we @@ -1165,7 +1165,7 @@ auto EvalLookupSingleFinalWitness(Context& context, SemIR::LocId loc_id, // If the lookup comes from inside an impl decl and is for that impl, get a // witness for that impl even though it does not yet exist. if (!context.declaring_impl_decls().empty()) { - const auto& declaring = context.declaring_impl_decls().back(); + const auto& declaring = context.declaring_impl_decls().Peek(); if (query_self_const_id == declaring.self_id || IsPeriodSelf(context, eval_query.query_self_inst_id)) { if (query_specific_interface == declaring.specific_interface) { diff --git a/toolchain/check/member_access.cpp b/toolchain/check/member_access.cpp index 7a1d14d447d5..99da70a3831c 100644 --- a/toolchain/check/member_access.cpp +++ b/toolchain/check/member_access.cpp @@ -186,7 +186,7 @@ static auto PerformImplWitnessAccessAndSubstitute( GetOrAddInst(context, loc_id, access); if (!context.where_stack().empty()) { - if (auto result = context.where_stack().back().rewrites.Lookup( + if (auto result = context.where_stack().Peek().rewrites.Lookup( context.constant_values().Get(access_id))) { return GetOrAddInst( context, loc_id,