From 30318f87f7b9b6c2c923b4469e7b35ced3554874 Mon Sep 17 00:00:00 2001 From: Dana Jansens Date: Thu, 12 Nov 2026 16:48:09 +0000 Subject: [PATCH] Introduce StableStack to avoid UAF into Context stacks (#8055) Stacks in Context can grow when eval is run, and if they are implemented as SmallVector, the growing can cause a reallocation. Meanwhile for stacks that hold larger structs/classes, we hold onto the pointer into the stack instead of copying out. This can cause a UAF when reallocation happens. StableStack never reallocates when growing, as it is represented by a vector of pointers to arrays, where each array is a factor larger than the previous one. This prevents UAF from occurring when we run eval while holding a pointer into a stack. This solves a similar problem that our ValuesStores previously solved in #5576. In that case, we also represent them as a vector of pointers to arrays. The difference being our ValueStores are meant to grow but not shrink, and they are built around the ID concept where IDs happen to be an index into the store. --- toolchain/base/BUILD | 23 + toolchain/base/stable_stack.h | 402 ++++++++++++++++ toolchain/base/stable_stack_test.cpp | 554 +++++++++++++++++++++++ toolchain/check/BUILD | 2 + toolchain/check/context.h | 9 +- toolchain/check/eval.cpp | 5 + toolchain/check/generic_region_stack.cpp | 4 +- toolchain/check/generic_region_stack.h | 11 +- toolchain/check/handle_impl.cpp | 6 +- toolchain/check/handle_where.cpp | 26 +- toolchain/check/impl_lookup.cpp | 4 +- toolchain/check/member_access.cpp | 2 +- 12 files changed, 1019 insertions(+), 29 deletions(-) create mode 100644 toolchain/base/stable_stack.h create mode 100644 toolchain/base/stable_stack_test.cpp diff --git a/toolchain/base/BUILD b/toolchain/base/BUILD index f2cee9e76fd2..5e6da8d635b9 100644 --- a/toolchain/base/BUILD +++ b/toolchain/base/BUILD @@ -239,6 +239,29 @@ cc_library( ], ) +cc_library( + name = "stable_stack", + hdrs = [ + "stable_stack.h", + ], + deps = [ + ":mem_usage", + "//common:check", + "@llvm-project//llvm:Support", + ], +) + +cc_test( + name = "stable_stack_test", + size = "small", + srcs = ["stable_stack_test.cpp"], + deps = [ + ":stable_stack", + "//testing/base:gtest_main", + "@googletest//:gtest", + ], +) + cc_library( name = "timings", hdrs = ["timings.h"], diff --git a/toolchain/base/stable_stack.h b/toolchain/base/stable_stack.h new file mode 100644 index 000000000000..adc0a2f8807d --- /dev/null +++ b/toolchain/base/stable_stack.h @@ -0,0 +1,402 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef CARBON_TOOLCHAIN_BASE_STABLE_STACK_H_ +#define CARBON_TOOLCHAIN_BASE_STABLE_STACK_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "llvm/ADT/ScopeExit.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Compiler.h" +#include "llvm/Support/MathExtras.h" +#include "toolchain/base/mem_usage.h" + +namespace Carbon { + +// A stack that never reallocates when growing so as to not invalidate pointers +// to objects that remain in the stack. +// +// As a stack, all insertions must be pushed onto the back of the stack, and +// doing so is an O(1) operation. Popping the back of the stack and peeking into +// the stack are also O(1) operations. +// +// The stack is implemented as an array of pointers to arrays. Each successive +// array gains a larger capacity than the last by multiplicative factor, to +// limit the number of allocations to be logarithmic in the number of +// insertions. But in all stacks, we use the same size progression for the +// chunks, which makes `operator==` simpler to implement. The stack uses a +// configurable value for the size of the first allocation. +// +// Like a std::vector, the stack does not free memory after Pop(). This means +// the stack retains capacity for holding the maximum number of values it ever +// held. +// +// TODO: The FirstCapacity being in the type allows for some simplifications +// internally where we can assume two stacks of the same type have the same +// chunk sizes. The typical use of StableStack is currently that we have one +// StableStack type for a given `T`, as they all have the same FirstCapacity. If +// we end up with many different StableStack types for the same `T` with +// different FirstCapacity, we should consider moving it to a runtime value. +template +class StableStack { + static_assert(FirstCapacity > 0); + static_assert(llvm::isPowerOf2_32(uint32_t{FirstCapacity}), + "FirstCapacity must be a power of 2"); + + public: + StableStack() = default; + ~StableStack() { DestroyAll(); } + + // Copying the stack. + StableStack(const StableStack& rhs) + requires(std::is_copy_constructible_v) + : used_chunks_(rhs.used_chunks_), + size_in_last_chunk_(rhs.size_in_last_chunk_) { + CopyConstructFrom(rhs); + } + auto operator=(const StableStack& rhs) -> StableStack& + requires(std::is_copy_constructible_v && std::is_copy_assignable_v) + { + if (this == &rhs) { + return *this; + } + + // TODO: We could reuse existing elements in `this` by calling `operator=` + // on them instead of destroying and reconstructing always. + // + // TODO: This is broken for assignment-from-subobject, where we would need + // to construct all of the new values before destroying the old. + DestroyAll(); + used_chunks_ = rhs.used_chunks_; + size_in_last_chunk_ = rhs.size_in_last_chunk_; + CopyConstructFrom(rhs); + return *this; + } + + // Moving the stack. The moved-from `rhs` stack will be left empty. + StableStack(StableStack&& rhs) noexcept + : chunks_(std::exchange(rhs.chunks_, {})), + used_chunks_(std::exchange(rhs.used_chunks_, 0)), + size_in_last_chunk_(std::exchange(rhs.size_in_last_chunk_, 0)) {} + auto operator=(StableStack&& rhs) noexcept -> StableStack& { + if (this == &rhs) { + return *this; + } + + // TODO: This is broken for assignment-from-subobject, where this would + // destroy the `rhs`. + DestroyAll(); + chunks_ = std::exchange(rhs.chunks_, {}); + used_chunks_ = std::exchange(rhs.used_chunks_, 0); + size_in_last_chunk_ = std::exchange(rhs.size_in_last_chunk_, 0); + return *this; + } + + // Returns a reference to the most recently pushed element at the back the + // stack. + auto Peek() const -> const T& { + CARBON_CHECK(!empty(), "Peek on an empty stack"); + auto [chunk, index] = GetIndexAtBack(); + return Get(chunk, index); + } + auto Peek() -> T& { + CARBON_CHECK(!empty(), "Peek on an empty stack"); + auto [chunk, index] = GetIndexAtBack(); + return Get(chunk, index); + } + + // Returns a reference to an element in the stack at distance `n` from the + // back of the stack. + auto Peek(int32_t n) const -> const T& { + CARBON_CHECK(!empty(), "Peek on an empty stack"); + CARBON_CHECK(n >= 0, "Peek index invalid"); + auto i = size() - 1 - n; + CARBON_CHECK(i >= 0, "Peek before the start of the stack"); + auto [chunk, index] = GetIndexFromFront(i); + return Get(chunk, index); + } + auto Peek(int32_t n) -> T& { + CARBON_CHECK(!empty(), "Peek on an empty stack"); + CARBON_CHECK(n >= 0, "Peek index invalid"); + auto i = size() - 1 - n; + CARBON_CHECK(i >= 0, "Peek before the start of the stack"); + auto [chunk, index] = GetIndexFromFront(i); + return Get(chunk, index); + } + + // Pushes a new element onto the back of the stack. + auto Push(const T& t) -> void { + auto* slot = PushSlot(); + std::construct_at(slot, t); + } + auto Push(T&& t) -> void { + auto* slot = PushSlot(); + std::construct_at(slot, std::move(t)); + } + + // Pushes a new element onto the back of the stack, constructed in place from + // the given arguments. + template + auto Emplace(Args&&... args) -> void { + auto* slot = PushSlot(); + std::construct_at(slot, std::forward(args)...); + } + + // Removes the most recently pushed element from the back of the stack. + auto Pop() -> void { + CARBON_CHECK(!empty(), "Pop on an empty stack"); + auto [chunk, index] = GetIndexAtBack(); + auto& ref = Get(chunk, index); + + ShrinkSize(); + std::destroy_at(&ref); + __asan_poison_memory_region(&ref, sizeof(T)); + } + + // Removes the most recently pushed element from the back of the stack and + // returns it. + auto PopVal() -> T { + CARBON_CHECK(!empty(), "Pop on an empty stack"); + auto [chunk, index] = GetIndexAtBack(); + auto& ref = Get(chunk, index); + + auto cleanup = llvm::scope_exit([&] { + ShrinkSize(); + std::destroy_at(&ref); + __asan_poison_memory_region(&ref, sizeof(T)); + }); + return std::move(ref); + } + + // Ensure there is memory allocated to hold at least `reserve` elements. + auto Reserve(int32_t reserve) -> void { + auto num_chunks = static_cast(NumChunksNeededForCapacity(reserve)); + chunks_.reserve(num_chunks); + while (chunks_.size() < num_chunks) { + AppendChunk(); + } + } + + // Whether there are any elements pushed onto the stack. + auto empty() const -> bool { return used_chunks_ == 0; } + // The number of elements pushed onto the stack. + auto size() const -> int32_t { + auto full_chunks_capacity = + used_chunks_ > 0 ? GetSumCapacity(used_chunks_ - 1) : 0; + return full_chunks_capacity + size_in_last_chunk_; + } + // The number of elements that the stack can hold without requiring further + // allocation. + auto capacity() const -> int32_t { return GetSumCapacity(chunks_.size()); } + + // Compares the contents of two stacks for equality. + friend auto operator==(const StableStack& lhs, const StableStack& rhs) -> bool + requires std::equality_comparable + { + if (lhs.used_chunks_ != rhs.used_chunks_) { + return false; + } + if (lhs.size_in_last_chunk_ != rhs.size_in_last_chunk_) { + return false; + } + for (auto chunk = 0; chunk < lhs.used_chunks_; ++chunk) { + const T* lhs_array = lhs.chunks_[chunk].array; + const T* rhs_array = rhs.chunks_[chunk].array; + int32_t cap = GetChunkCapacity(chunk); + auto size = chunk < lhs.used_chunks_ - 1 ? cap : lhs.size_in_last_chunk_; + if (!std::equal(lhs_array, lhs_array + size, rhs_array)) { + return false; + } + } + return true; + } + + auto CollectMemUsage(MemUsage& mem_usage, llvm::StringRef label) const + -> void { + mem_usage.Add(label.str(), size() * sizeof(T), capacity() * sizeof(T)); + } + + private: + // The total capacity of the first `num_chunks` chunks. + static constexpr auto GetSumCapacity(int32_t num_chunks) -> int32_t { + return ((1 << num_chunks) - 1) * FirstCapacity; + } + // The capacity of an individual chunk at index `chunk`. + static constexpr auto GetChunkCapacity(int32_t chunk) -> int32_t { + // We use a growth factor of 2 so that we can do bitshifts in + // GetSumCapacity() to compute total capacity in O(1) time. + return FirstCapacity << chunk; + } + // Performs the inverse of GetSumCapacity. + static constexpr auto NumChunksNeededForCapacity(int32_t cap) -> int32_t { + // Given this formula, we solve for `num_chunks`. + // cap = ((1 << num_chunks) - 1) * FirstCapacity; + auto ucap = static_cast(cap); + // Shifting by this simulates division by FirstCapacity. + auto division = llvm::Log2_32(uint32_t{FirstCapacity}); + return llvm::Log2_32_Ceil(((ucap + (FirstCapacity - 1)) >> division) + 1); + } + + struct Chunk { + T* array [[clang::require_explicit_initialization]]; + }; + + // Copy-constructs elements into this stack from `rhs`. + auto CopyConstructFrom(const StableStack& rhs) -> void { + if (rhs.empty()) { + return; + } + + chunks_.clear(); + chunks_.reserve(used_chunks_); + for (auto chunk = 0; chunk < used_chunks_; ++chunk) { + T* rhs_array = rhs.chunks_[chunk].array; + auto cap = GetChunkCapacity(chunk); + T* array = std::allocator().allocate(cap); + chunks_.push_back({.array = array}); + auto size = chunk < used_chunks_ - 1 ? cap : size_in_last_chunk_; + std::uninitialized_copy_n(rhs_array, size, array); + if (size < cap) { + __asan_poison_memory_region(array + size, (cap - size) * sizeof(T)); + } + } + } + + // Destroy all the elements in the stack and free the arrays in the chunks. + // Leaves the stack in an empty state. + auto DestroyAll() -> void { + // Make `this` appear empty while destroying its elements in case they try + // to use the stack mid-destruction. + auto size_in_last_chunk = std::exchange(size_in_last_chunk_, 0); + auto used_chunks = std::exchange(used_chunks_, 0); + auto chunks = std::exchange(chunks_, {}); + + for (auto chunk = 0; chunk < static_cast(chunks.size()); ++chunk) { + auto* array = chunks[chunk].array; + auto cap = GetChunkCapacity(chunk); + if (chunk < used_chunks - 1) { + std::destroy_n(array, cap); + } else if (chunk == used_chunks - 1) { + std::destroy_n(array, size_in_last_chunk); + } + std::allocator().deallocate(array, cap); + } + + CARBON_CHECK(chunks_.empty(), + "Mutation of StableStack while destroying its elements"); + } + + // Get the index of the element at the back of the stack. This is an O(1) + // operation. + auto GetIndexAtBack() const + -> std::pair { + int32_t chunk = used_chunks_ - 1; + int32_t index = size_in_last_chunk_ - 1; + return {chunk, index}; + } + + // Get the index of the element at position `i` from the front of the stack. + auto GetIndexFromFront(int32_t i) const + -> std::pair { + // Given a capacity, we can tell how many chunks would be needed. So we + // compute the number of chunks needed to hold a stack with capacity for + // exactly `i + 1` elements, so that `i` is the last element. Then the last + // element in that stack is the index we want to compute. + auto capacity_to_hold_index_i = i + 1; + auto num_chunks = NumChunksNeededForCapacity(capacity_to_hold_index_i); + auto capacity_before_chunk = GetSumCapacity(num_chunks - 1); + return {num_chunks - 1, i - capacity_before_chunk}; + } + + // Get the element in the specified `chunk` at position `index`. + auto Get(int32_t chunk, int32_t index) -> T& { + CARBON_DCHECK(chunk >= 0); + CARBON_DCHECK(chunk < used_chunks_); + CARBON_DCHECK(index >= 0); + auto cap = GetChunkCapacity(chunk); + auto size = chunk == used_chunks_ - 1 ? size_in_last_chunk_ : cap; + CARBON_DCHECK(index < size, + "Getting index {0} past end of last chunk with size {1}", + index, size); + return chunks_[chunk].array[index]; + } + auto Get(int32_t chunk, int32_t index) const -> const T& { + return const_cast(this)->Get(chunk, index); + } + + // Allocate memory as needed to get a pointer that we can construct `T` into. + // The pointer returned has been ASAN unpoisoned. + auto PushSlot() -> T* { + // TODO: It would be faster to make the writes to the returned slot not be + // dependent on the branches, as then the branches can be done + // asynchronously by the processor. To do that, we would need to always have + // the allocation ready for the *next* push. That means eagerly allocating + // the first chunk at construction time. And it means we can always grab a + // pointer here immediately. Then the rest of the branches and work in here + // does not affect the pointer, nor the writes through the pointer. For a + // moved-from stack we don't want to have to allocate in the RHS, so we can + // say we leave moved-from stacks with an empty set of chunks, and DCHECK + // against use-after-move. + + T* slot; + if (used_chunks_ > 0 && + size_in_last_chunk_ < GetChunkCapacity(used_chunks_ - 1)) { + // Add a slot in the last used chunk. + slot = chunks_[used_chunks_ - 1].array + size_in_last_chunk_; + ++size_in_last_chunk_; + } else if (used_chunks_ < static_cast(chunks_.size())) { + // Add a slot in an unused chunk that already exists. + slot = chunks_[used_chunks_].array; + ++used_chunks_; + size_in_last_chunk_ = 1; + } else { + // Add a slot in a new chunk. + auto& new_chunk = AppendChunk(); + slot = new_chunk.array; + ++used_chunks_; + size_in_last_chunk_ = 1; + } + __asan_unpoison_memory_region(slot, sizeof(T)); + return slot; + } + auto AppendChunk() -> Chunk& { + auto next_cap = GetChunkCapacity(chunks_.size()); + T* array = std::allocator().allocate(next_cap); + __asan_poison_memory_region(array, next_cap * sizeof(T)); + chunks_.push_back({.array = array}); + return chunks_.back(); + } + + // Shrinks the recorded size of the stack by one. Does not actually destroy + // anything. + auto ShrinkSize() -> void { + size_in_last_chunk_--; + if (size_in_last_chunk_ == 0) { + used_chunks_--; + if (used_chunks_ > 0) { + size_in_last_chunk_ = GetChunkCapacity(used_chunks_ - 1); + } else { + // Stack is empty now. + } + } + } + + llvm::SmallVector chunks_; + // The number of chunks that are being used in `chunks_`. + int32_t used_chunks_ = 0; + // The number of values in the last used chunk. + int32_t size_in_last_chunk_ = 0; +}; + +} // namespace Carbon + +#endif // CARBON_TOOLCHAIN_BASE_STABLE_STACK_H_ diff --git a/toolchain/base/stable_stack_test.cpp b/toolchain/base/stable_stack_test.cpp new file mode 100644 index 000000000000..efb7c9f79955 --- /dev/null +++ b/toolchain/base/stable_stack_test.cpp @@ -0,0 +1,554 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "toolchain/base/stable_stack.h" + +#include + +namespace Carbon { +namespace { + +TEST(StableStack, DestroyEmpty) { + StableStack stack; + EXPECT_EQ(stack.empty(), true); +} + +TEST(StableStack, Growth) { + StableStack stack; + EXPECT_EQ(stack.empty(), true); + EXPECT_EQ(stack.size(), 0); + + int i = 0; + stack.Push(i++); + EXPECT_EQ(stack.empty(), false); + EXPECT_EQ(stack.size(), i); + + auto capacity = stack.capacity(); + while (stack.capacity() == capacity) { + stack.Push(i++); + EXPECT_EQ(stack.empty(), false); + EXPECT_EQ(stack.size(), i); + } + + capacity = stack.capacity(); + while (stack.capacity() == capacity) { + stack.Push(i++); + EXPECT_EQ(stack.empty(), false); + EXPECT_EQ(stack.size(), i); + } +} + +TEST(StableStack, Emplace) { + enum Constructor { Default, Ints, String }; + struct S { + S() : ctor(Default) {} + S(int /*a*/, int /*b*/) : ctor(Ints) {} + explicit S(const char* /*s*/) : ctor(String) {} + + Constructor ctor; + }; + + StableStack stack; + stack.Emplace(); + EXPECT_EQ(stack.Peek().ctor, Default); + stack.Emplace(1, 2); + EXPECT_EQ(stack.Peek().ctor, Ints); + stack.Emplace("hi"); + EXPECT_EQ(stack.Peek().ctor, String); +} + +template +static auto GrowCapacityTimes(StableStack& stack, int n, F factory) + -> void { + while (n) { + auto capacity = stack.capacity(); + while (stack.capacity() == capacity) { + stack.Push(factory()); + } + --n; + } +} + +TEST(StableStack, PushPop) { + StableStack stack; + int i = 0; + GrowCapacityTimes(stack, 3, [&i] { return i++; }); + EXPECT_EQ(stack.size(), i); + + stack.Push(i++); + EXPECT_EQ(stack.size(), i); + + EXPECT_EQ(stack.Peek(), --i); + stack.Pop(); + EXPECT_EQ(stack.size(), i); + + // Drops below the last capacity jump. + EXPECT_EQ(stack.Peek(), --i); + stack.Pop(); + EXPECT_EQ(stack.size(), i); + + stack.Push(i++); + EXPECT_EQ(stack.size(), i); + + while (i) { + EXPECT_EQ(stack.Peek(), --i); + stack.Pop(); + EXPECT_EQ(stack.size(), i); + } + EXPECT_EQ(stack.empty(), true); +} + +TEST(StableStack, PushPopVal) { + StableStack stack; + int i = 0; + GrowCapacityTimes(stack, 3, [&i] { return i++; }); + EXPECT_EQ(stack.size(), i); + + stack.Push(i++); + EXPECT_EQ(stack.size(), i); + + EXPECT_EQ(stack.PopVal(), --i); + EXPECT_EQ(stack.size(), i); + + // Drops below the last capacity jump. + EXPECT_EQ(stack.PopVal(), --i); + EXPECT_EQ(stack.size(), i); + + stack.Push(i++); + EXPECT_EQ(stack.size(), i); + + while (i) { + EXPECT_EQ(stack.PopVal(), --i); + EXPECT_EQ(stack.size(), i); + } + EXPECT_EQ(stack.empty(), true); +} + +TEST(StableStack, EmptyPushPop) { + StableStack stack; + + stack.Push(1); + EXPECT_EQ(stack.size(), 1); + EXPECT_EQ(stack.empty(), false); + + EXPECT_EQ(stack.PopVal(), 1); + EXPECT_EQ(stack.size(), 0); + EXPECT_EQ(stack.empty(), true); + + stack.Push(2); + EXPECT_EQ(stack.size(), 1); + EXPECT_EQ(stack.empty(), false); + + EXPECT_EQ(stack.PopVal(), 2); + EXPECT_EQ(stack.size(), 0); + EXPECT_EQ(stack.empty(), true); +} + +TEST(StableStack, CopyConstructDestroy) { + int constructed = 0; + int destroyed = 0; + class Counter { + public: + Counter(int* constructed, int* destroyed) + : constructed_(constructed), destroyed_(destroyed) {} + Counter(const Counter& rhs) + : constructed_(rhs.constructed_), destroyed_(rhs.destroyed_) { + (*constructed_)++; + } + ~Counter() { (*destroyed_)++; } + + private: + int* constructed_; + int* destroyed_; + }; + + int size; + { + StableStack stack; + GrowCapacityTimes(stack, 3, + [&] { return Counter(&constructed, &destroyed); }); + EXPECT_EQ(constructed, stack.size()); + // Each Counter constructed in the lambda was destroyed. + EXPECT_EQ(destroyed, stack.size()); + size = stack.size(); + constructed = destroyed = 0; + + static_cast(stack).Peek(); + EXPECT_EQ(constructed, 0); + EXPECT_EQ(destroyed, 0); + stack.Peek(); + EXPECT_EQ(constructed, 0); + EXPECT_EQ(destroyed, 0); + } + EXPECT_EQ(constructed, 0); + // Each Counter held in the stack was destroyed. + EXPECT_EQ(destroyed, size); +} + +TEST(StableStack, MoveConstructDestroy) { + int constructed = 0; + int destroyed = 0; + class Counter { + public: + Counter(int* constructed, int* destroyed) + : constructed_(constructed), destroyed_(destroyed) {} + Counter(Counter&& rhs) noexcept + : constructed_(rhs.constructed_), destroyed_(rhs.destroyed_) { + (*constructed_)++; + } + ~Counter() { (*destroyed_)++; } + + private: + int* constructed_; + int* destroyed_; + }; + + int size; + { + StableStack stack; + GrowCapacityTimes(stack, 3, + [&] { return Counter(&constructed, &destroyed); }); + EXPECT_EQ(constructed, stack.size()); + // Each Counter constructed in the lambda was destroyed. + EXPECT_EQ(destroyed, stack.size()); + size = stack.size(); + constructed = destroyed = 0; + + static_cast(stack).Peek(); + EXPECT_EQ(constructed, 0); + EXPECT_EQ(destroyed, 0); + stack.Peek(); + EXPECT_EQ(constructed, 0); + EXPECT_EQ(destroyed, 0); + } + EXPECT_EQ(constructed, 0); + // Each Counter held in the stack was destroyed. + EXPECT_EQ(destroyed, size); +} + +TEST(StableStack, CopyStack) { + int constructed = 0; + int destroyed = 0; + class Counter { + public: + Counter(int value, int* constructed, int* destroyed) + : value_(value), constructed_(constructed), destroyed_(destroyed) {} + Counter(const Counter& rhs) + : value_(rhs.value_), + constructed_(rhs.constructed_), + destroyed_(rhs.destroyed_) { + (*constructed_)++; + } + ~Counter() { (*destroyed_)++; } + + auto operator==(const Counter& rhs) const -> bool { + return value_ == rhs.value_; + } + + private: + int value_; + int* constructed_; + int* destroyed_; + }; + + int size; + { + StableStack stack; + GrowCapacityTimes(stack, 3, [&, i = 0]() mutable { + return Counter(i++, &constructed, &destroyed); + }); + constructed = destroyed = 0; + size = stack.size(); + + // Copy construction. + StableStack stack2 = stack; + EXPECT_EQ(constructed, size); + EXPECT_EQ(destroyed, 0); + constructed = destroyed = 0; + + EXPECT_EQ(stack.size(), stack2.size()); + EXPECT_LE(stack2.capacity(), stack.capacity()); + for (auto i = 0; i < size; ++i) { + EXPECT_EQ(stack.Peek(i), stack2.Peek(i)); + } + EXPECT_EQ(constructed, 0); + EXPECT_EQ(destroyed, 0); + constructed = destroyed = 0; + + // Self-assignment. + stack2 = [&]() -> StableStack& { return stack2; }(); + EXPECT_EQ(constructed, 0); + EXPECT_EQ(destroyed, 0); + constructed = destroyed = 0; + + StableStack stack3; + EXPECT_EQ(constructed, 0); + EXPECT_EQ(destroyed, 0); + constructed = destroyed = 0; + + // Copy assignment to empty. + stack3 = stack2; + EXPECT_EQ(constructed, size); + EXPECT_EQ(destroyed, 0); + constructed = destroyed = 0; + + // Copy assignment to non-empty. We destroy the LHS and reconstruct from the + // RHS. + stack3 = stack2; + EXPECT_EQ(constructed, size); + EXPECT_EQ(destroyed, size); + constructed = destroyed = 0; + + // Copy assignment from empty. + StableStack stack4; + stack3 = stack4; + EXPECT_EQ(constructed, 0); + EXPECT_EQ(destroyed, size); + constructed = destroyed = 0; + } + EXPECT_EQ(constructed, 0); + // Each Counter held in each stack was destroyed. + EXPECT_EQ(destroyed, size * 2); +} + +TEST(StableStack, MoveStack) { + int constructed = 0; + int destroyed = 0; + class Counter { + public: + Counter(int value, int* constructed, int* destroyed) + : value_(value), constructed_(constructed), destroyed_(destroyed) {} + Counter(Counter&& rhs) noexcept + : value_(rhs.value_), + constructed_(rhs.constructed_), + destroyed_(rhs.destroyed_) { + (*constructed_)++; + } + ~Counter() { (*destroyed_)++; } + + auto value() const -> int { return value_; } + + private: + int value_; + int* constructed_; + int* destroyed_; + }; + + int size; + { + StableStack stack; + GrowCapacityTimes(stack, 3, [&, i = 0]() mutable { + return Counter(i++, &constructed, &destroyed); + }); + constructed = destroyed = 0; + size = stack.size(); + + // Move construction. + StableStack stack2 = std::move(stack); + EXPECT_EQ(constructed, 0); + EXPECT_EQ(destroyed, 0); + constructed = destroyed = 0; + + // Moving leaves the source empty. + EXPECT_EQ(stack.empty(), true); + for (auto i = 0; i < size; ++i) { + EXPECT_EQ(stack2.Peek(i).value(), size - 1 - i); + } + EXPECT_EQ(constructed, 0); + EXPECT_EQ(destroyed, 0); + constructed = destroyed = 0; + + // Self-assignment. + stack2 = [&]() -> StableStack&& { return std::move(stack2); }(); + EXPECT_EQ(constructed, 0); + EXPECT_EQ(destroyed, 0); + constructed = destroyed = 0; + + StableStack stack3; + EXPECT_EQ(constructed, 0); + EXPECT_EQ(destroyed, 0); + constructed = destroyed = 0; + + // Move assignment to empty. + stack3 = std::move(stack2); + EXPECT_EQ(constructed, 0); + EXPECT_EQ(destroyed, 0); + constructed = destroyed = 0; + + stack.Push(Counter(0, &constructed, &destroyed)); + constructed = destroyed = 0; + + // Move assignment to non-empty. We destroy the LHS. + stack = std::move(stack3); + // The single element in `stack` is destroyed. + EXPECT_EQ(constructed, 0); + EXPECT_EQ(destroyed, 1); + constructed = destroyed = 0; + + // Move assignment from empty. + stack = std::move(stack3); + EXPECT_EQ(constructed, 0); + EXPECT_EQ(destroyed, size); + constructed = destroyed = 0; + } + EXPECT_EQ(constructed, 0); + // All the stacks were empty by the end. + EXPECT_EQ(destroyed, 0); +} + +TEST(StableStack, Peek) { + StableStack stack; + const auto& const_stack = stack; + stack.Push(1); + + decltype(auto) r = const_stack.Peek(); + static_assert(std::same_as); + + decltype(auto) m = stack.Peek(); + static_assert(std::same_as); + + stack.Push(2); + EXPECT_EQ(const_stack.Peek(), 2); + EXPECT_EQ(stack.Peek(), 2); + stack.Peek() = 4; + EXPECT_EQ(const_stack.Peek(), 4); + EXPECT_EQ(stack.Peek(), 4); + EXPECT_EQ(stack.PopVal(), 4); + + EXPECT_EQ(const_stack.Peek(), 1); + EXPECT_EQ(stack.Peek(), 1); + stack.Peek() = 3; + EXPECT_EQ(const_stack.Peek(), 3); + EXPECT_EQ(stack.Peek(), 3); + EXPECT_EQ(stack.PopVal(), 3); +} + +TEST(StableStack, Reserve) { + StableStack stack; + auto capacity = stack.capacity(); + + stack.Reserve(capacity / 2); + EXPECT_EQ(stack.capacity(), capacity); + stack.Reserve(capacity); + EXPECT_EQ(stack.capacity(), capacity); + + stack.Reserve(capacity + 2); + EXPECT_GE(stack.capacity(), capacity + 2); + capacity = stack.capacity(); + stack.Reserve(1); + EXPECT_EQ(stack.capacity(), capacity); + stack.Reserve(2); + EXPECT_EQ(stack.capacity(), capacity); + + stack.Reserve(capacity + 1000); + EXPECT_GE(stack.capacity(), capacity + 1000); + EXPECT_EQ(stack.size(), 0); + EXPECT_EQ(stack.empty(), true); + + capacity = stack.capacity(); + stack.Reserve(1); + EXPECT_EQ(stack.capacity(), capacity); + stack.Reserve(0); + EXPECT_EQ(stack.capacity(), capacity); + + for (int i = 0; i < capacity; ++i) { + stack.Push(i); + } + EXPECT_EQ(stack.capacity(), capacity); + EXPECT_EQ(stack.size(), capacity); +} + +TEST(StableStack, ReserveWithFirstCapacity) { + { + StableStack stack; + EXPECT_EQ(stack.capacity(), 0); + stack.Push(0); + EXPECT_EQ(stack.capacity(), 1); + } + { + StableStack stack; + EXPECT_EQ(stack.capacity(), 0); + stack.Reserve(1); + EXPECT_EQ(stack.capacity(), 1); + stack.Reserve(2); + EXPECT_EQ(stack.capacity(), 1 + 1 * 2); + } + { + StableStack stack; + EXPECT_EQ(stack.capacity(), 0); + stack.Reserve(1); + EXPECT_EQ(stack.capacity(), 4); + stack.Reserve(4); + EXPECT_EQ(stack.capacity(), 4); + stack.Reserve(5); + EXPECT_EQ(stack.capacity(), 4 + 4 * 2); + } +} + +TEST(StableStack, Equal) { + StableStack stack; + StableStack stack2; + + EXPECT_EQ(stack, stack2); + stack.Reserve(1); + EXPECT_EQ(stack, stack2); + stack2.Reserve(1000); + EXPECT_EQ(stack, stack2); + + int i = 0; + GrowCapacityTimes(stack, 3, [&i] { return i++; }); + EXPECT_NE(stack, stack2); + + for (auto j = 0; j < stack.size(); ++j) { + stack2.Push(j); + } + EXPECT_EQ(stack, stack2); + + auto start_size = stack.size(); + while (stack.size() > start_size / 2) { + stack.Pop(); + EXPECT_NE(stack, stack2); + stack2.Pop(); + EXPECT_EQ(stack, stack2); + } + while (!stack.empty()) { + stack2.Pop(); + EXPECT_NE(stack, stack2); + stack.Pop(); + EXPECT_EQ(stack, stack2); + } +} + +TEST(StableStack, IncompleteType) { + struct S { + // Ensure we can form the StableStack type with an incomplete type + // parameter. + StableStack* stack; + }; + StableStack stack; + S _ = {.stack = &stack}; +} + +TEST(StableStack, EmptyDuringDestroy) { + bool test_empty = false; + struct S { + StableStack* stack; + bool* test_empty; + + ~S() { + if (*test_empty) { + EXPECT_TRUE(stack->empty()); + } + } + }; + StableStack stack; + GrowCapacityTimes( + stack, 3, [&] { return S{.stack = &stack, .test_empty = &test_empty}; }); + + // `S` tests that `stack` appears empty while each `S` is destroyed. + test_empty = true; +} + +} // namespace +} // namespace Carbon diff --git a/toolchain/check/BUILD b/toolchain/check/BUILD index c560a86c035d..469ca6c826f4 100644 --- a/toolchain/check/BUILD +++ b/toolchain/check/BUILD @@ -179,6 +179,7 @@ cc_library( "//toolchain/base:int", "//toolchain/base:kind_switch", "//toolchain/base:shared_value_stores", + "//toolchain/base:stable_stack", "//toolchain/base:value_ids", "//toolchain/base:value_store", "//toolchain/check:generic_region_stack", @@ -310,6 +311,7 @@ cc_library( "//common:map", "//common:ostream", "//common:vlog", + "//toolchain/base:stable_stack", "//toolchain/sem_ir:file", "//toolchain/sem_ir:typed_insts", "@llvm-project//llvm:Support", diff --git a/toolchain/check/context.h b/toolchain/check/context.h index cfd699858c74..aad74fd3bbcc 100644 --- a/toolchain/check/context.h +++ b/toolchain/check/context.h @@ -11,6 +11,7 @@ #include "common/ostream.h" #include "llvm/ADT/SmallVector.h" #include "toolchain/base/canonical_value_store.h" +#include "toolchain/base/stable_stack.h" #include "toolchain/base/value_store.h" #include "toolchain/check/core_identifier.h" #include "toolchain/check/cpp/context.h" @@ -307,7 +308,7 @@ class Context { SemIR::LocId loc_id; }; - auto where_stack() -> llvm::SmallVector& { + auto where_stack() -> StableStack& { return where_stack_; } @@ -320,7 +321,7 @@ class Context { SemIR::SpecificInterface specific_interface; auto operator==(const DeclaringImplDecl& rhs) const -> bool = default; }; - auto declaring_impl_decls() -> llvm::SmallVector& { + auto declaring_impl_decls() -> StableStack& { return declaring_impl_decls_; } @@ -644,7 +645,7 @@ class Context { // Tracks information about constraints in the current `where` expression // being checked so that they can be used by later constraints. - llvm::SmallVector where_stack_; + StableStack where_stack_; // Counts the number of `where` expressions in the type of the binding // currently being checked. @@ -653,7 +654,7 @@ class Context { // Track impl declarations that are underway. If we're declaring an impl for // `C as I`, an impl lookup query for `C as I` or `.Self as I` should find // that impl being declared (even though it does not yet exist). - llvm::SmallVector declaring_impl_decls_; + StableStack declaring_impl_decls_; // Tracks if we are currently identifying a facet type, which causes // LookupImplWitness instructions to not resolve to final witnesses by diff --git a/toolchain/check/eval.cpp b/toolchain/check/eval.cpp index b9ff0e404fe3..65764bce7736 100644 --- a/toolchain/check/eval.cpp +++ b/toolchain/check/eval.cpp @@ -3463,6 +3463,11 @@ auto TryEvalBlockForSpecific(Context& context, SemIR::LocId loc_id, // specific, we are resolving existing instructions in the eval block for a // generic. The global state should not apply to anything done while resolving // a specific. + // + // TODO: Instead of moving the whole stack here, track an index on Context + // that is the start of the stack that we're going to consider (in practice we + // only consider a single value right now, but we'd only consider the latest + // value at least at that tracked index). CheckedSaveAndRestore declaring(context.declaring_impl_decls(), {}); // The `eval_impl_lookup_to_identified_witness` is also a global state that diff --git a/toolchain/check/generic_region_stack.cpp b/toolchain/check/generic_region_stack.cpp index f82e9cfbb672..d627195225f2 100644 --- a/toolchain/check/generic_region_stack.cpp +++ b/toolchain/check/generic_region_stack.cpp @@ -14,7 +14,7 @@ auto GenericRegionStack::Push(PendingGeneric generic) -> void { pending_generic_ids_.push_back(generic); pending_eval_block_stack_.PushArray(); dependent_inst_stack_.PushArray(); - constants_in_generic_stack_.emplace_back(); + constants_in_generic_stack_.Emplace(); } auto GenericRegionStack::Pop() -> void { @@ -23,7 +23,7 @@ auto GenericRegionStack::Pop() -> void { pending.region); pending_eval_block_stack_.PopArray(); dependent_inst_stack_.PopArray(); - constants_in_generic_stack_.pop_back(); + constants_in_generic_stack_.Pop(); } } // namespace Carbon::Check diff --git a/toolchain/check/generic_region_stack.h b/toolchain/check/generic_region_stack.h index 4cf6d574064e..59b62594cad3 100644 --- a/toolchain/check/generic_region_stack.h +++ b/toolchain/check/generic_region_stack.h @@ -7,6 +7,7 @@ #include "common/array_stack.h" #include "common/map.h" +#include "toolchain/base/stable_stack.h" #include "toolchain/sem_ir/ids.h" namespace Carbon::Check { @@ -33,8 +34,8 @@ class GenericRegionStack { public: explicit GenericRegionStack(llvm::raw_ostream* vlog_stream) : vlog_stream_(vlog_stream) { - // Reserve a large enough stack that we typically won't need to reallocate. - constants_in_generic_stack_.reserve(4); + // Trigger the allocation of the first capacity specified in the type. + constants_in_generic_stack_.Reserve(1); } struct PendingGeneric { @@ -96,7 +97,7 @@ class GenericRegionStack { // instructions for the current generic. auto PeekConstantsInGenericMap() -> ConstantsInGenericMap& { CARBON_CHECK(!Empty()); - return constants_in_generic_stack_.back(); + return constants_in_generic_stack_.Peek(); } // Returns the total number of instructions tracked by all pending generic @@ -131,7 +132,9 @@ class GenericRegionStack { // Mapping from constant InstIds to the corresponding InstIds in the eval // blocks for each enclosing generic. We reserve this to a suitable size in // the constructor. - llvm::SmallVector constants_in_generic_stack_; + // + // We choose an initial capacity that typically won't need to allocate again. + StableStack constants_in_generic_stack_; }; } // namespace Carbon::Check diff --git a/toolchain/check/handle_impl.cpp b/toolchain/check/handle_impl.cpp index b0bb8696c9ad..b886a4a92a30 100644 --- a/toolchain/check/handle_impl.cpp +++ b/toolchain/check/handle_impl.cpp @@ -113,7 +113,7 @@ auto HandleParseNode(Context& context, Parse::ImplTypeAsId node_id) -> bool { // The specific interface in here, if any, is populated by the `where` // expression that introduces a `.Self` in the impl's constraint facet type. - context.declaring_impl_decls().push_back( + context.declaring_impl_decls().Push( {.self_id = self_type.type_id.AsConstantId(), .specific_interface = SemIR::SpecificInterface::None}); return true; @@ -151,7 +151,7 @@ auto HandleParseNode(Context& context, Parse::ImplDefaultSelfAsId node_id) // The specific interface in here, if any, is populated by the `where` // expression that introduces a `.Self` in the impl's constraint facet type. - context.declaring_impl_decls().push_back( + context.declaring_impl_decls().Push( {.self_id = context.constant_values().Get(self_inst_id), .specific_interface = SemIR::SpecificInterface::None}); return true; @@ -228,7 +228,7 @@ static auto BuildImplDecl(Context& context, Parse::AnyImplDeclId node_id, // Witnesses for this impl in the DeclaredFacetType of the decl will have // been evaluated to ImplSelfWitness. - context.declaring_impl_decls().pop_back(); + context.declaring_impl_decls().Pop(); // Convert the constraint expression to a type. This contains all constraints, // including rewrites and other constrains on the RHS of `where`. diff --git a/toolchain/check/handle_where.cpp b/toolchain/check/handle_where.cpp index 0b776d34a713..4383317502de 100644 --- a/toolchain/check/handle_where.cpp +++ b/toolchain/check/handle_where.cpp @@ -103,7 +103,7 @@ auto HandleParseNode(Context& context, Parse::WhereOperandId node_id) -> bool { auto impl_as_target = identified.EvaluateImplWitnessesToGetImplAsTarget(context, node_id); if (impl_as_target.is_valid()) { - auto& declaring = context.declaring_impl_decls().back(); + auto& declaring = context.declaring_impl_decls().Peek(); if (declaring.specific_interface != SemIR::SpecificInterface::None) { CARBON_CHECK(declaring.specific_interface == impl_as_target.target()); } @@ -126,7 +126,7 @@ auto HandleParseNode(Context& context, Parse::WhereOperandId node_id) -> bool { // Add a context stack for tracking constraints, that will be used to allow // later constraints to read from them eagerly. - context.where_stack().push_back({.loc_id = node_id}); + context.where_stack().Push({.loc_id = node_id}); // Track the occurrence of `where` inside a binding's type. ++context.binding_type_where_count(); @@ -146,7 +146,7 @@ auto HandleParseNode(Context& context, Parse::WhereOperandId node_id) -> bool { GetImplWitnessAccessWithoutSubstitution(context.sem_ir(), rewrite.lhs_id)); auto frozen_const_id = FreezePeriodSelf(context, const_id); - context.where_stack().back().rewrites.Insert(frozen_const_id, + context.where_stack().Peek().rewrites.Insert(frozen_const_id, rewrite.rhs_id); } } @@ -165,7 +165,7 @@ auto HandleParseNode(Context& context, Parse::WhereOperandId node_id) -> bool { GetInterfaceType(context, impls.interface_id, impls.specific_id) .AsConstantId(); auto type_frozen_const_id = FreezePeriodSelf(context, type_const_id); - context.where_stack().back().impls.push_back( + context.where_stack().Peek().impls.push_back( {.self_const_id = self_frozen_const_id, .facet_type_const_id = type_frozen_const_id}); } @@ -177,7 +177,7 @@ auto HandleParseNode(Context& context, Parse::WhereOperandId node_id) -> bool { impls.specific_id) .AsConstantId(); auto type_frozen_const_id = FreezePeriodSelf(context, type_const_id); - context.where_stack().back().impls.push_back( + context.where_stack().Peek().impls.push_back( {.self_const_id = self_frozen_const_id, .facet_type_const_id = type_frozen_const_id}); } @@ -190,7 +190,7 @@ auto HandleParseNode(Context& context, Parse::WhereOperandId node_id) -> bool { type_impls.specific_interface.specific_id) .AsConstantId(); auto type_frozen_const_id = FreezePeriodSelf(context, type_const_id); - context.where_stack().back().impls.push_back( + context.where_stack().Peek().impls.push_back( {.self_const_id = self_frozen_const_id, .facet_type_const_id = type_frozen_const_id}); } @@ -204,7 +204,7 @@ auto HandleParseNode(Context& context, Parse::WhereOperandId node_id) -> bool { type_impls.specific_named_constraint.specific_id) .AsConstantId(); auto type_frozen_const_id = FreezePeriodSelf(context, type_const_id); - context.where_stack().back().impls.push_back( + context.where_stack().Peek().impls.push_back( {.self_const_id = self_frozen_const_id, .facet_type_const_id = type_frozen_const_id}); } @@ -319,7 +319,7 @@ auto HandleParseNode(Context& context, Parse::RequirementEqualId node_id) // Note that the where_stack rewrites need to be frozen. Since this // expression is inside of facet type construction, it will already be // frozen. - context.where_stack().back().rewrites.Insert( + context.where_stack().Peek().rewrites.Insert( context.constant_values().Get( GetImplWitnessAccessWithoutSubstitution(context.sem_ir(), lhs_id)), rhs_id); @@ -451,7 +451,7 @@ auto HandleParseNode(Context& context, Parse::RequirementImplsId node_id) // Track the impls relationship so further constraints can use it // immediately, before they are evaluated. Impl lookup will search the top // of the stack. - context.where_stack().back().impls.push_back({ + context.where_stack().Peek().impls.push_back({ context.constant_values().Get(lhs_as_type.inst_id), context.constant_values().Get(rhs_as_type.inst_id), }); @@ -475,7 +475,7 @@ auto HandleParseNode(Context& context, Parse::RequirementImplsId node_id) auto lhs_id = SubstPeriodSelf( context, rhs_node, context.constant_values().Get(rewrite.lhs_id), context.constant_values().Get(lhs_as_type.inst_id)); - context.where_stack().back().rewrites.Insert(lhs_id, rewrite.rhs_id); + context.where_stack().Peek().rewrites.Insert(lhs_id, rewrite.rhs_id); } } } @@ -592,8 +592,8 @@ static auto ThawPeriodSelfInRequirements(Context& context, } auto HandleParseNode(Context& context, Parse::WhereExprId node_id) -> bool { - auto where_loc = context.where_stack().back().loc_id; - context.where_stack().pop_back(); + auto where_loc = context.where_stack().Peek().loc_id; + context.where_stack().Pop(); // Remove `PeriodSelf` from name lookup, undoing the `Push` done for the // `WhereOperand`. context.scope_stack().Pop(/*check_unused=*/true); @@ -603,7 +603,7 @@ auto HandleParseNode(Context& context, Parse::WhereExprId node_id) -> bool { if (!context.where_stack().empty()) { // This `where` expression is nested on the RHS of another `where`, which is // an error. - DiagnoseNestedWhere(context, node_id, context.where_stack().back().loc_id); + DiagnoseNestedWhere(context, node_id, context.where_stack().Peek().loc_id); type_id = SemIR::ErrorInst::TypeId; } requirements_id = CheckForNestedWhereInRequirementsAfterEval( diff --git a/toolchain/check/impl_lookup.cpp b/toolchain/check/impl_lookup.cpp index 387992e20817..daa48d993dba 100644 --- a/toolchain/check/impl_lookup.cpp +++ b/toolchain/check/impl_lookup.cpp @@ -455,7 +455,7 @@ static auto CollectFacetWitnessSources( // inner `where` expression is checked before the outer, so it needs to be // self-consistent and provide any `impls` constraints required by other // constraints. - const auto& impls = context.where_stack().back().impls; + const auto& impls = context.where_stack().Peek().impls; for (auto [self_const_id, facet_type_const_id] : impls) { self_const_id = GetCanonicalFacet(context, self_const_id); // TypeType (and ErrorInst) is never stored in the impls stack, so we @@ -1165,7 +1165,7 @@ auto EvalLookupSingleFinalWitness(Context& context, SemIR::LocId loc_id, // If the lookup comes from inside an impl decl and is for that impl, get a // witness for that impl even though it does not yet exist. if (!context.declaring_impl_decls().empty()) { - const auto& declaring = context.declaring_impl_decls().back(); + const auto& declaring = context.declaring_impl_decls().Peek(); if (query_self_const_id == declaring.self_id || IsPeriodSelf(context, eval_query.query_self_inst_id)) { if (query_specific_interface == declaring.specific_interface) { diff --git a/toolchain/check/member_access.cpp b/toolchain/check/member_access.cpp index 7a1d14d447d5..99da70a3831c 100644 --- a/toolchain/check/member_access.cpp +++ b/toolchain/check/member_access.cpp @@ -186,7 +186,7 @@ static auto PerformImplWitnessAccessAndSubstitute( GetOrAddInst(context, loc_id, access); if (!context.where_stack().empty()) { - if (auto result = context.where_stack().back().rewrites.Lookup( + if (auto result = context.where_stack().Peek().rewrites.Lookup( context.constant_values().Get(access_id))) { return GetOrAddInst( context, loc_id,