mirror of
https://github.com/carbon-language/carbon-lang.git
synced 2026-09-29 14:15:04 +01:00
This switches `DCHECK` and `FATAL` as well. The goal is to reduce the code size impact of these assertions so that we can keep more of them enabled. Currently, the largest cost I see from `CHECK` is not the actual check or the cold code itself, but actually the failure to inline trivial functions due to the presence of the cold code. This means that our goal isn't to reduce apparent code size in the final binary but the LLVM IR cost assessed for these routines in the inliner, which closely correlates with code size but is a bit different. As discussed in #4283, experimentation shows that a single function call with a minimal number of arguments is the lowest cost model for these. This is easily achieved with a format-string API that internally uses `llvm::formatv`. This PR is essentially the `CHECK` version of #4283. However, the check macros are substantially harder to make work with both format strings and streaming because they also take a condition. Also, unexpectedly, I was very successful at devising a regular expression based automated rewrite from the streaming to the format string form with only low 10s of manual fixes. This includes compacting strings broken up across lines, etc. Given how well that went, I've prepared this PR which just directly switches to the format string API and migrate everything to use it. One nice side-effect is that the format string approach ends up greatly simplifying the implementation here as well. This is ... *shockingly* effective. Parsing speeds up by more than 3% with just this change. And checking speeds up by **8%** with this change alone: ``` BM_CompileAPIFileDenseDecls<Phase::Parse>/256 86.3µs ± 1% 82.9µs ± 1% -3.94% (p=0.000 n=17+19) BM_CompileAPIFileDenseDecls<Phase::Parse>/1024 431µs ± 1% 415µs ± 1% -3.76% (p=0.000 n=18+19) BM_CompileAPIFileDenseDecls<Phase::Parse>/4096 1.77ms ± 1% 1.71ms ± 1% -3.18% (p=0.000 n=18+19) BM_CompileAPIFileDenseDecls<Phase::Parse>/16384 7.44ms ± 1% 7.17ms ± 2% -3.56% (p=0.000 n=18+20) BM_CompileAPIFileDenseDecls<Phase::Parse>/65536 30.7ms ± 1% 29.7ms ± 1% -3.15% (p=0.000 n=18+20) BM_CompileAPIFileDenseDecls<Phase::Parse>/262144 131ms ± 1% 127ms ± 1% -2.81% (p=0.000 n=18+18) BM_CompileAPIFileDenseDecls<Phase::Check>/256 878µs ± 2% 800µs ± 1% -8.91% (p=0.000 n=19+20) BM_CompileAPIFileDenseDecls<Phase::Check>/1024 1.88ms ± 2% 1.72ms ± 1% -8.56% (p=0.000 n=19+20) BM_CompileAPIFileDenseDecls<Phase::Check>/4096 5.78ms ± 2% 5.28ms ± 1% -8.70% (p=0.000 n=20+18) BM_CompileAPIFileDenseDecls<Phase::Check>/16384 21.9ms ± 1% 20.1ms ± 1% -8.02% (p=0.000 n=18+20) BM_CompileAPIFileDenseDecls<Phase::Check>/65536 90.4ms ± 2% 83.1ms ± 1% -8.04% (p=0.000 n=19+20) BM_CompileAPIFileDenseDecls<Phase::Check>/262144 381ms ± 2% 352ms ± 1% -7.79% (p=0.000 n=19+19) ``` --------- Co-authored-by: Richard Smith <richard@metafoo.co.uk> Co-authored-by: josh11b <15258583+josh11b@users.noreply.github.com>
323 lines
12 KiB
C++
323 lines
12 KiB
C++
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
|
|
// Exceptions. See /LICENSE for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
|
|
#include "toolchain/check/subst.h"
|
|
|
|
#include "toolchain/check/eval.h"
|
|
#include "toolchain/check/generic.h"
|
|
#include "toolchain/sem_ir/copy_on_write_block.h"
|
|
#include "toolchain/sem_ir/ids.h"
|
|
#include "toolchain/sem_ir/inst.h"
|
|
|
|
namespace Carbon::Check {
|
|
|
|
namespace {
|
|
|
|
// Information about an instruction that we are substituting into.
|
|
struct WorklistItem {
|
|
// The instruction that we are substituting into.
|
|
SemIR::InstId inst_id;
|
|
// Whether the operands of this instruction have been added to the worklist.
|
|
bool is_expanded : 1;
|
|
// The index of the worklist item to process after we finish updating this
|
|
// one. For the final child of an instruction, this is the parent. For any
|
|
// other child, this is the index of the next child of the parent. For the
|
|
// root, this is -1.
|
|
int next_index : 31;
|
|
};
|
|
|
|
// A list of instructions that we're currently in the process of substituting
|
|
// into. For details of the algorithm used here, see `SubstConstant`.
|
|
class Worklist {
|
|
public:
|
|
explicit Worklist(SemIR::InstId root_id) {
|
|
worklist_.push_back(
|
|
{.inst_id = root_id, .is_expanded = false, .next_index = -1});
|
|
}
|
|
|
|
auto operator[](int index) -> WorklistItem& { return worklist_[index]; }
|
|
auto size() -> int { return worklist_.size(); }
|
|
auto back() -> WorklistItem& { return worklist_.back(); }
|
|
|
|
auto Push(SemIR::InstId inst_id) -> void {
|
|
CARBON_CHECK(inst_id.is_valid());
|
|
worklist_.push_back({.inst_id = inst_id,
|
|
.is_expanded = false,
|
|
.next_index = static_cast<int>(worklist_.size() + 1)});
|
|
CARBON_CHECK(worklist_.back().next_index > 0, "Constant too large.");
|
|
}
|
|
auto Pop() -> SemIR::InstId { return worklist_.pop_back_val().inst_id; }
|
|
|
|
private:
|
|
// Constants can get pretty large, so use a large worklist. This should be
|
|
// about 4KiB, which should be small enough to comfortably fit on the stack,
|
|
// but large enough that it's unlikely that we'll need a heap allocation.
|
|
llvm::SmallVector<WorklistItem, 512> worklist_;
|
|
};
|
|
|
|
} // namespace
|
|
|
|
// Pushes the specified operand onto the worklist.
|
|
static auto PushOperand(Context& context, Worklist& worklist,
|
|
SemIR::IdKind kind, int32_t arg) -> void {
|
|
switch (kind) {
|
|
case SemIR::IdKind::For<SemIR::InstId>:
|
|
if (SemIR::InstId inst_id(arg); inst_id.is_valid()) {
|
|
worklist.Push(inst_id);
|
|
}
|
|
break;
|
|
case SemIR::IdKind::For<SemIR::TypeId>:
|
|
if (SemIR::TypeId type_id(arg); type_id.is_valid()) {
|
|
worklist.Push(context.types().GetInstId(type_id));
|
|
}
|
|
break;
|
|
case SemIR::IdKind::For<SemIR::InstBlockId>:
|
|
for (auto inst_id : context.inst_blocks().Get(SemIR::InstBlockId(arg))) {
|
|
worklist.Push(inst_id);
|
|
}
|
|
break;
|
|
case SemIR::IdKind::For<SemIR::TypeBlockId>:
|
|
for (auto type_id : context.type_blocks().Get(SemIR::TypeBlockId(arg))) {
|
|
worklist.Push(context.types().GetInstId(type_id));
|
|
}
|
|
break;
|
|
case SemIR::IdKind::For<SemIR::SpecificId>:
|
|
if (auto specific_id = static_cast<SemIR::SpecificId>(arg);
|
|
specific_id.is_valid()) {
|
|
PushOperand(context, worklist, SemIR::IdKind::For<SemIR::InstBlockId>,
|
|
context.specifics().Get(specific_id).args_id.index);
|
|
}
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Converts the operands of this instruction into `InstId`s and pushes them onto
|
|
// the worklist.
|
|
static auto ExpandOperands(Context& context, Worklist& worklist,
|
|
SemIR::InstId inst_id) -> void {
|
|
auto inst = context.insts().Get(inst_id);
|
|
auto kinds = inst.ArgKinds();
|
|
PushOperand(context, worklist, SemIR::IdKind::For<SemIR::TypeId>,
|
|
inst.type_id().index);
|
|
PushOperand(context, worklist, kinds.first, inst.arg0());
|
|
PushOperand(context, worklist, kinds.second, inst.arg1());
|
|
}
|
|
|
|
// Pops the specified operand from the worklist and returns it.
|
|
static auto PopOperand(Context& context, Worklist& worklist, SemIR::IdKind kind,
|
|
int32_t arg) -> int32_t {
|
|
switch (kind) {
|
|
case SemIR::IdKind::For<SemIR::InstId>: {
|
|
SemIR::InstId inst_id(arg);
|
|
if (!inst_id.is_valid()) {
|
|
return arg;
|
|
}
|
|
return worklist.Pop().index;
|
|
}
|
|
case SemIR::IdKind::For<SemIR::TypeId>: {
|
|
SemIR::TypeId type_id(arg);
|
|
if (!type_id.is_valid()) {
|
|
return arg;
|
|
}
|
|
return context.GetTypeIdForTypeInst(worklist.Pop()).index;
|
|
}
|
|
case SemIR::IdKind::For<SemIR::InstBlockId>: {
|
|
SemIR::InstBlockId old_inst_block_id(arg);
|
|
auto size = context.inst_blocks().Get(old_inst_block_id).size();
|
|
SemIR::CopyOnWriteInstBlock new_inst_block(context.sem_ir(),
|
|
old_inst_block_id);
|
|
for (auto i : llvm::reverse(llvm::seq(size))) {
|
|
new_inst_block.Set(i, worklist.Pop());
|
|
}
|
|
return new_inst_block.GetCanonical().index;
|
|
}
|
|
case SemIR::IdKind::For<SemIR::TypeBlockId>: {
|
|
SemIR::TypeBlockId old_type_block_id(arg);
|
|
auto size = context.type_blocks().Get(old_type_block_id).size();
|
|
SemIR::CopyOnWriteTypeBlock new_type_block(context.sem_ir(),
|
|
old_type_block_id);
|
|
for (auto i : llvm::index_range(0, size)) {
|
|
new_type_block.Set(size - i - 1,
|
|
context.GetTypeIdForTypeInst(worklist.Pop()));
|
|
}
|
|
return new_type_block.GetCanonical().index;
|
|
}
|
|
case SemIR::IdKind::For<SemIR::SpecificId>: {
|
|
SemIR::SpecificId specific_id(arg);
|
|
if (!specific_id.is_valid()) {
|
|
return arg;
|
|
}
|
|
auto& specific = context.specifics().Get(specific_id);
|
|
auto args_id =
|
|
PopOperand(context, worklist, SemIR::IdKind::For<SemIR::InstBlockId>,
|
|
specific.args_id.index);
|
|
return MakeSpecific(context, specific.generic_id,
|
|
SemIR::InstBlockId(args_id))
|
|
.index;
|
|
}
|
|
default:
|
|
return arg;
|
|
}
|
|
}
|
|
|
|
// Pops the operands of the specified instruction off the worklist and rebuilds
|
|
// the instruction with the updated operands if it has changed.
|
|
static auto Rebuild(Context& context, Worklist& worklist, SemIR::InstId inst_id,
|
|
const SubstInstCallbacks& callbacks) -> SemIR::InstId {
|
|
auto inst = context.insts().Get(inst_id);
|
|
auto kinds = inst.ArgKinds();
|
|
|
|
// Note that we pop in reverse order because we pushed them in forwards order.
|
|
int32_t arg1 = PopOperand(context, worklist, kinds.second, inst.arg1());
|
|
int32_t arg0 = PopOperand(context, worklist, kinds.first, inst.arg0());
|
|
int32_t type_id =
|
|
PopOperand(context, worklist, SemIR::IdKind::For<SemIR::TypeId>,
|
|
inst.type_id().index);
|
|
if (type_id == inst.type_id().index && arg0 == inst.arg0() &&
|
|
arg1 == inst.arg1()) {
|
|
return inst_id;
|
|
}
|
|
|
|
// TODO: Do we need to require this type to be complete?
|
|
inst.SetType(SemIR::TypeId(type_id));
|
|
inst.SetArgs(arg0, arg1);
|
|
return callbacks.Rebuild(inst_id, inst);
|
|
}
|
|
|
|
auto SubstInst(Context& context, SemIR::InstId inst_id,
|
|
const SubstInstCallbacks& callbacks) -> SemIR::InstId {
|
|
Worklist worklist(inst_id);
|
|
|
|
// For each instruction that forms part of the constant, we will visit it
|
|
// twice:
|
|
//
|
|
// - First, we visit it with `is_expanded == false`, we add all of its
|
|
// operands onto the worklist, and process them by following this same
|
|
// process.
|
|
// - Then, once all operands are processed, we visit the instruction with
|
|
// `is_expanded == true`, pop the operands back off the worklist, and if any
|
|
// of them changed, rebuild this instruction.
|
|
//
|
|
// The second step is skipped if we can detect in the first step that the
|
|
// instruction will not need to be rebuilt.
|
|
int index = 0;
|
|
while (index != -1) {
|
|
auto& item = worklist[index];
|
|
|
|
if (item.is_expanded) {
|
|
// Rebuild this item if necessary. Note that this might pop items from the
|
|
// worklist but does not reallocate, so does not invalidate `item`.
|
|
item.inst_id = Rebuild(context, worklist, item.inst_id, callbacks);
|
|
index = item.next_index;
|
|
continue;
|
|
}
|
|
|
|
if (callbacks.Subst(item.inst_id)) {
|
|
index = item.next_index;
|
|
continue;
|
|
}
|
|
|
|
// Extract the operands of this item into the worklist. Note that this
|
|
// modifies the worklist, so it's not safe to use `item` after
|
|
// `ExpandOperands` returns.
|
|
item.is_expanded = true;
|
|
int first_operand = worklist.size();
|
|
int next_index = item.next_index;
|
|
ExpandOperands(context, worklist, item.inst_id);
|
|
|
|
// If there are any operands, go and update them before rebuilding this
|
|
// item.
|
|
if (worklist.size() > first_operand) {
|
|
worklist.back().next_index = index;
|
|
index = first_operand;
|
|
} else {
|
|
// No need to rebuild this instruction: its operands can't be changed by
|
|
// substitution because it has none.
|
|
index = next_index;
|
|
}
|
|
}
|
|
|
|
CARBON_CHECK(worklist.size() == 1,
|
|
"Unexpected data left behind in work list");
|
|
return worklist.back().inst_id;
|
|
}
|
|
|
|
namespace {
|
|
// Callbacks for performing substitution of a set of Substitutions into a
|
|
// symbolic constant.
|
|
class SubstConstantCallbacks final : public SubstInstCallbacks {
|
|
public:
|
|
SubstConstantCallbacks(Context& context, Substitutions substitutions)
|
|
: context_(context), substitutions_(substitutions) {}
|
|
|
|
// Applies the given Substitutions to an instruction, in order to replace
|
|
// BindSymbolicName instructions with the value of the binding.
|
|
auto Subst(SemIR::InstId& inst_id) const -> bool override {
|
|
if (context_.constant_values().Get(inst_id).is_template()) {
|
|
// This instruction is a template constant, so can't contain any
|
|
// bindings that need to be substituted.
|
|
return true;
|
|
}
|
|
|
|
auto bind = context_.insts().TryGetAs<SemIR::BindSymbolicName>(inst_id);
|
|
if (!bind) {
|
|
return false;
|
|
}
|
|
|
|
// This is a symbolic binding. Check if we're substituting it.
|
|
// TODO: Consider building a hash map for substitutions. We might have a
|
|
// lot of them.
|
|
for (auto [bind_index, replacement_id] : substitutions_) {
|
|
if (context_.entity_names().Get(bind->entity_name_id).bind_index ==
|
|
bind_index) {
|
|
// This is the binding we're replacing. Perform substitution.
|
|
inst_id = context_.constant_values().GetInstId(replacement_id);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// If it's not being substituted, don't look through it. Its constant
|
|
// value doesn't depend on its operand.
|
|
return true;
|
|
}
|
|
|
|
// Rebuilds an instruction by building a new constant.
|
|
auto Rebuild(SemIR::InstId /*old_inst_id*/, SemIR::Inst new_inst) const
|
|
-> SemIR::InstId override {
|
|
auto result_id = TryEvalInst(context_, SemIR::InstId::Invalid, new_inst);
|
|
CARBON_CHECK(result_id.is_constant(),
|
|
"Substitution into constant produced non-constant");
|
|
return context_.constant_values().GetInstId(result_id);
|
|
}
|
|
|
|
private:
|
|
Context& context_;
|
|
Substitutions substitutions_;
|
|
};
|
|
} // namespace
|
|
|
|
auto SubstConstant(Context& context, SemIR::ConstantId const_id,
|
|
Substitutions substitutions) -> SemIR::ConstantId {
|
|
CARBON_CHECK(const_id.is_constant(), "Substituting into non-constant");
|
|
|
|
if (substitutions.empty()) {
|
|
// Nothing to substitute.
|
|
return const_id;
|
|
}
|
|
|
|
if (!const_id.is_symbolic()) {
|
|
// A template constant can't contain a reference to a symbolic binding.
|
|
return const_id;
|
|
}
|
|
|
|
auto subst_inst_id =
|
|
SubstInst(context, context.constant_values().GetInstId(const_id),
|
|
SubstConstantCallbacks(context, substitutions));
|
|
return context.constant_values().Get(subst_inst_id);
|
|
}
|
|
|
|
} // namespace Carbon::Check
|