mirror of
https://github.com/carbon-language/carbon-lang.git
synced 2026-10-05 16:01:04 +01:00
Richard Smith
3bbc03f527
Adds an algorithm to compute where to insert brackets to repair bracketing mismatches during lexing. This takes indentation, as well as a number of other cues, into account to predict where the brackets should have gone. Detects when there is ambiguity between solutions and makes no suggestion in that case. Reduces the problem by splitting on properly bracketed top-level constructs, then uses a beam search to find good candidate solutions quickly. This includes both a fuzzer and an eval tool that can be used to determine how well the algorithm fares against a given corpus of valid Carbon code, by damaging it in various ways and seeing whether the algorithm can correctly fix it. On all the eval modes, this algorithm can correctly infer the positions for over 80% of lost brackets (and can correctly restore 95+% of brackets in some modes), with low rates of incorrect suggestions. See added documentation for full details. Assisted-by: Gemini via Antigravity, Claude via Claude Code
2657 lines
107 KiB
C++
2657 lines
107 KiB
C++
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
|
|
// Exceptions. See /LICENSE for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
|
|
#include "toolchain/lex/mismatched_brackets.h"
|
|
|
|
#include <algorithm>
|
|
#include <array>
|
|
#include <bit>
|
|
#include <cmath>
|
|
#include <iterator>
|
|
#include <limits>
|
|
#include <optional>
|
|
|
|
#include "common/check.h"
|
|
#include "common/hashing.h"
|
|
#include "common/hashtable_key_context.h"
|
|
#include "common/map.h"
|
|
#include "common/set.h"
|
|
#include "llvm/ADT/BitVector.h"
|
|
#include "llvm/ADT/BitmaskEnum.h"
|
|
#include "llvm/ADT/STLExtras.h"
|
|
#include "llvm/ADT/STLFunctionalExtras.h"
|
|
#include "llvm/ADT/Sequence.h"
|
|
#include "llvm/ADT/StringRef.h"
|
|
|
|
namespace Carbon::Lex {
|
|
|
|
LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
|
|
|
|
// Maximum number of collapsed items in a damaged region before falling back to
|
|
// naive greedy recovery. The beam search below is linear in the region size,
|
|
// so this is mostly a defense against pathological inputs.
|
|
constexpr int32_t MaxRegionItemsForSearch = 1500;
|
|
|
|
// Layered beam search width limit.
|
|
constexpr size_t MaxBeamWidth = 16;
|
|
|
|
// Maximum stack depth allowed during search before capping.
|
|
constexpr size_t MaxSearchStackDepth = 12;
|
|
|
|
// Maximum number of distinct optimal repair paths enumerated when checking
|
|
// whether the optimal repairs agree about each correction. Paths beyond this
|
|
// are not examined, so a correction they alone would dispute stays untied.
|
|
constexpr size_t MaxOptimalPaths = 100;
|
|
|
|
// The cost model. All costs are relative; the search finds the cheapest way
|
|
// to make the whole region well-bracketed, and the resulting insertions and
|
|
// replacements become the suggested corrections. When several cheapest ways
|
|
// exist, corrections they disagree about are marked tied and downgraded to
|
|
// error replacements, so it's important that the intended repair for a common
|
|
// mistake is *strictly* cheaper than the alternatives.
|
|
//
|
|
// The individual numbers here and in the rule tables below carry no meaning
|
|
// beyond how they order the repairs they price. They were not derived; they
|
|
// were hill-climbed by coordinate descent against `mismatched_brackets_eval`,
|
|
// which deletes brackets from real Carbon source and scores what recovery puts
|
|
// back, including a per-rule precision breakdown. So a cost is best changed the
|
|
// same way it was chosen: adjust it, rerun the evaluation, and keep the change
|
|
// only if the scores improve. Rebalancing is worth doing when the evaluation
|
|
// reports poor precision for a rule, and is required when adding a rule, whose
|
|
// cost only means something relative to the rules it competes with. See
|
|
// /toolchain/docs/lex/mismatched_brackets.md for how to run it.
|
|
//
|
|
// Costs of replacing a real bracket with an error token. These are the
|
|
// "give up on this bracket" fallbacks; a good targeted repair should beat
|
|
// them, and a dubious one should lose to them.
|
|
constexpr int32_t CostReplaceClosing = 30;
|
|
constexpr int32_t CostReplaceOpening = 50;
|
|
|
|
// Costs of inserting a synthetic closing bracket in front of the current
|
|
// token live in the `CloserRules` table below, which is keyed by how strongly
|
|
// the context suggests the group ends here. These few are shared with the
|
|
// region-end handling, which isn't part of that table.
|
|
// Closing anything at the end of the file or region.
|
|
constexpr int32_t CostCloseAtEnd = 12;
|
|
constexpr int32_t CostCloseParenAtEnd = 22;
|
|
constexpr int32_t CostCloseStructAtEnd = 20;
|
|
// Closing a paren/square bracket before a mid-line `.` that has whitespace
|
|
// before it: member access is normally written without spaces. Priced below the
|
|
// `Adv_SpacedPeriodInParen` rule in `AdvanceRules` below, which penalizes
|
|
// stepping over such a `.` instead, so that closing here beats closing earlier
|
|
// and leaving the spaced `.` unexplained.
|
|
constexpr int32_t CostCloseParenBeforeSpacedPeriod = 6;
|
|
// Closing a group at a wide mid-line whitespace gap, which suggests the
|
|
// closer was deleted in the gap.
|
|
constexpr int32_t CostCloseAtWideGap = 6;
|
|
|
|
// Costs of inserting a synthetic opening bracket in front of the current
|
|
// token live in the `OpenerRules` table below, and penalties for advancing
|
|
// over a token in a context where it doesn't belong in `AdvanceRules`.
|
|
|
|
// Penalty for matching a scope `}` whose indentation doesn't match its
|
|
// opener's header, when they're on different lines.
|
|
constexpr int32_t CostBraceIndentMismatchBase = 30;
|
|
constexpr int32_t CostBraceIndentMismatchPerColumn = 20;
|
|
|
|
namespace {
|
|
|
|
// A contextual cue a rule can test, beyond the bucket's context category and
|
|
// token kind. These are bit indexes into a `CueSet`; no cue's value is ever
|
|
// significant, so cues can be added, removed, and reordered freely.
|
|
enum class Cue : uint8_t {
|
|
// Properties of the current token.
|
|
//
|
|
// The current token is the first on its line.
|
|
FirstOnLine,
|
|
// There is whitespace (or a comment) directly before the current token.
|
|
LeadingSpace,
|
|
// The current token is mid-line with two or more bytes of space before it.
|
|
WideGap,
|
|
// The current token is a `{` with struct-literal cues.
|
|
StructBrace,
|
|
// The current token is `else`.
|
|
ElseKeyword,
|
|
|
|
// Properties of the token directly before the current one. All are false at
|
|
// the start of the input.
|
|
//
|
|
// The previous token is value-ending, an adjacency that is illegal before a
|
|
// leaf.
|
|
PrevValueEnding,
|
|
// As `PrevValueEnding`, but also counting `]` and `}`: the previous token
|
|
// ends a value, whether or not a leaf may follow it.
|
|
PrevValueLike,
|
|
// The previous token itself has whitespace before it.
|
|
PrevHasLeadingSpace,
|
|
// The previous token is a keyword that must be followed by `(` (`if`,
|
|
// `while`, `for`, `match`), or one that must be followed by `[` (`forall`).
|
|
PrevKeywordWantsParen,
|
|
PrevKeywordWantsSquare,
|
|
// The previous token is `as`, `->`, or `where`, so what follows it is a type
|
|
// rather than something callable. (Not `=`, which is followed by a value.)
|
|
PrevIntroducesType,
|
|
// The previous token's kind, for the kinds any rule distinguishes. At most
|
|
// one of these holds at a time.
|
|
PrevIsPeriod,
|
|
PrevIsComma,
|
|
// `PrevIsOpenBracket` covers `(` and `[` together, which no rule separates.
|
|
PrevIsOpenBracket,
|
|
PrevIsOpenCurly,
|
|
PrevIsCloseParen,
|
|
PrevIsCloseSquare,
|
|
PrevIsCloseCurly,
|
|
|
|
// Properties of the item directly before the current one. Unlike the `Prev`
|
|
// cues above, which look at the last token of that item, these look at the
|
|
// item as a whole, so they say nothing about a collapsed `(...)` block's
|
|
// contents.
|
|
//
|
|
// The previous item is a leaf.
|
|
PrevItemIsLeaf,
|
|
// The previous item is a name directly following `as` or `->`, so it is a
|
|
// type rather than something callable.
|
|
PrevItemIsTypeName,
|
|
|
|
// Properties of the item, from the analysis of the surrounding tokens.
|
|
//
|
|
// This item is a collapsed well-bracketed block rather than a single token.
|
|
CollapsedBlock,
|
|
// This collapsed block contains a scope `{`.
|
|
ContainsScopeBrace,
|
|
// The current token starts the body of a statement or declaration whose
|
|
// header is complete, and whose header didn't contain a `{`.
|
|
FollowsStatementHeader,
|
|
HeaderHasOpenCurly,
|
|
|
|
// Properties of the innermost open bracket, and of its relationship to the
|
|
// current token. All are false when no bracket is open.
|
|
//
|
|
// The innermost open bracket is a call or index `(`/`[`, rather than one
|
|
// following a keyword such as `if` or `forall`.
|
|
CallParenTop,
|
|
// The innermost open bracket is the token directly before the current one, so
|
|
// closing here would make an empty group.
|
|
AfterOpenTop,
|
|
// The current token is indented no further than the header of the statement
|
|
// or declaration containing the innermost open bracket.
|
|
DedentToHeader,
|
|
// The current token is indented no further than the line of the innermost
|
|
// open bracket itself, as opposed to that of its statement header.
|
|
DedentToOpenerLine,
|
|
// The current token is on a later line than the innermost open bracket.
|
|
NewLineFromTop,
|
|
// The current token is a closer matching an opener further out on the stack,
|
|
// so the innermost group has to close first.
|
|
Cascade,
|
|
|
|
// Properties of the repair this search path has made so far.
|
|
//
|
|
// This path inserted a closer directly before the current token.
|
|
CloserInserted,
|
|
// The closer this path inserted can be directly followed by a leaf (`]` or
|
|
// `}`), so it repairs an otherwise illegal adjacency.
|
|
CloserFixesAdjacency,
|
|
// This path synthesized an opener directly before the current token.
|
|
OpenerHere,
|
|
// This path inserted some bracket directly before the current token, so
|
|
// suspicious whitespace before it is already explained.
|
|
BracketInsertedHere,
|
|
|
|
// Must stay last: it counts the cues.
|
|
Count,
|
|
};
|
|
|
|
// A set of cues.
|
|
enum class CueSet : uint64_t {
|
|
None = 0,
|
|
LLVM_MARK_AS_BITMASK_ENUM(uint64_t{1}
|
|
<< (static_cast<uint8_t>(Cue::Count) - 1))
|
|
};
|
|
|
|
} // namespace
|
|
|
|
// The set containing exactly `cues`.
|
|
template <std::same_as<Cue>... CueT>
|
|
static constexpr auto CueSetOf(CueT... cues) -> CueSet {
|
|
return (CueSet::None | ... |
|
|
static_cast<CueSet>(uint64_t{1} << static_cast<int>(cues)));
|
|
}
|
|
|
|
// Convenience synonym for our token kinds.
|
|
using Kind = BracketTokenKind;
|
|
|
|
namespace {
|
|
|
|
// Internal representation of an item after clean subrange collapsing.
|
|
struct Item {
|
|
int32_t token_start_index;
|
|
int32_t token_end_index;
|
|
MismatchedBracketToken token;
|
|
int32_t effective_header_indent = 0;
|
|
// Every boolean property of this item and its neighbours, computed once by
|
|
// `ComputeItemCues`. This is the single record of them: the rule tables match
|
|
// against these (combined with the few cues that depend on the search state),
|
|
// and the search itself tests them through `Has`.
|
|
CueSet cues = CueSet::None;
|
|
|
|
// Whether every one of `wanted` holds for this item.
|
|
template <typename... CueT>
|
|
constexpr auto HasAll(CueT... wanted) const -> bool {
|
|
CueSet wanted_set = CueSetOf(wanted...);
|
|
return (cues & wanted_set) == wanted_set;
|
|
}
|
|
};
|
|
|
|
// The parts of an unclosed opening bracket that make a search state distinct.
|
|
// Compared and hashed as a whole, so everything here must matter to the search:
|
|
// two stacks that agree on these are interchangeable.
|
|
//
|
|
// Fields are ordered so the type has no padding, which
|
|
// `has_unique_object_representations` requires and `Carbon::Map` relies on to
|
|
// hash and compare every byte.
|
|
struct OpenBracketKey {
|
|
// The real opener token, or None for a synthetic opener.
|
|
TokenIndex token_index = TokenIndex::None;
|
|
// For a synthetic opener, where it would be inserted.
|
|
TokenIndex insertion_token_index = TokenIndex::None;
|
|
// Index of the real opener in the input token array, or -1 if synthetic.
|
|
int32_t token_pos = -1;
|
|
int32_t line = -1;
|
|
// Indentation of the line containing the opener.
|
|
int32_t line_indent = 0;
|
|
// Indentation of the start of the statement or declaration containing the
|
|
// opener.
|
|
int32_t effective_header_indent = 0;
|
|
BracketTokenKind kind;
|
|
bool is_synthetic = false;
|
|
bool is_struct_brace = false;
|
|
// Whether this is a paren/square bracket directly following a value-ending
|
|
// token (a call or index), rather than following a keyword like `if`. This is
|
|
// a function of the opener token, so comparing it is redundant with
|
|
// `token_index` for a real opener, and it is always false for a synthetic
|
|
// one; it's included only because excluding it would cost more than it saves.
|
|
bool is_call_paren = false;
|
|
|
|
friend auto operator==(const OpenBracketKey& lhs, const OpenBracketKey& rhs)
|
|
-> bool = default;
|
|
};
|
|
|
|
static_assert(std::has_unique_object_representations_v<OpenBracketKey>,
|
|
"Padding would leave indeterminate bytes for hashing.");
|
|
|
|
// An unclosed opening bracket on the search stack, plus the bookkeeping that
|
|
// says nothing about which state this is.
|
|
struct OpenBracketInfo : OpenBracketKey {
|
|
// For a synthetic opener, the name of the rule that proposed it, for
|
|
// debugging. Two openers that differ only in which rule proposed them are the
|
|
// same state, so this is deliberately outside the key.
|
|
llvm::StringLiteral rule_name = "";
|
|
};
|
|
|
|
} // namespace
|
|
|
|
// The `{` opening the branch that the first-on-line `else` at `else_index`
|
|
// continues. Such an `else` should have been preceded by the `}` closing that
|
|
// branch, so with the `}` missing the `else` is still inside it: the brace is
|
|
// the innermost one that nothing has closed yet. Returns -1 if there is no
|
|
// enclosing brace.
|
|
static auto FindBranchBrace(llvm::ArrayRef<MismatchedBracketToken> tokens,
|
|
int32_t else_index) -> int32_t {
|
|
int32_t depth = 0;
|
|
for (int32_t j : llvm::reverse(llvm::seq(0, else_index))) {
|
|
if (tokens[j].kind == Kind::CloseCurlyBrace) {
|
|
++depth;
|
|
} else if (tokens[j].kind == Kind::OpenCurlyBrace) {
|
|
if (depth == 0) {
|
|
return j;
|
|
}
|
|
--depth;
|
|
}
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
// Computes the associated line indentation for a token by scanning backwards,
|
|
// skipping matched parens/brackets, looking for a statement introducer.
|
|
static auto GetOuterStatementIntroducerIndent(
|
|
llvm::ArrayRef<MismatchedBracketToken> tokens,
|
|
llvm::ArrayRef<int32_t> match_partner, int32_t j) -> int32_t {
|
|
int32_t result_indent = tokens[j].line_indent;
|
|
while (j > 0) {
|
|
// An `else` continues the statement its `if` introduced, and the `}`
|
|
// closing its block lines up with that `if`, so keep walking out from the
|
|
// brace of the branch before it. A first-on-line `else`'s own column says
|
|
// nothing about where the statement starts: the `}` that should precede it
|
|
// is missing, so the `else` sits wherever the author left it.
|
|
if (tokens[j].is_else_keyword && tokens[j].line != tokens[j - 1].line) {
|
|
int32_t brace = FindBranchBrace(tokens, j);
|
|
if (brace > 0) {
|
|
j = brace;
|
|
result_indent = tokens[j].line_indent;
|
|
continue;
|
|
}
|
|
}
|
|
int32_t p = j - 1;
|
|
if ((tokens[p].kind == Kind::CloseParen ||
|
|
tokens[p].kind == Kind::CloseSquareBracket) &&
|
|
match_partner[p] != -1 && match_partner[p] < p) {
|
|
p = match_partner[p];
|
|
if (p <= 0) {
|
|
break;
|
|
}
|
|
--p;
|
|
}
|
|
if (tokens[p].kind == Kind::StatementIntroducer) {
|
|
result_indent = tokens[p].line_indent;
|
|
// Keep walking out only while the introducers stack up on one line or
|
|
// dedent; an indented token belongs to this introducer's body.
|
|
if (tokens[j].line == tokens[p].line ||
|
|
tokens[j].line_indent <= tokens[p].line_indent) {
|
|
j = p;
|
|
continue;
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
return result_indent;
|
|
}
|
|
|
|
// Computes the indentation that a token's statement or declaration starts at,
|
|
// by scanning back over the current statement to its introducer. This is what
|
|
// a `{` opened by that statement should line its `}` up with, which is often
|
|
// not the indentation of the `{` itself.
|
|
static auto ComputeAssociatedLineIndent(
|
|
llvm::ArrayRef<MismatchedBracketToken> tokens,
|
|
llvm::ArrayRef<int32_t> match_partner, int32_t token_index) -> int32_t {
|
|
if (token_index < 0 || token_index >= static_cast<int32_t>(tokens.size())) {
|
|
return 0;
|
|
}
|
|
if (tokens[token_index].kind == Kind::FileEnd) {
|
|
return 0;
|
|
}
|
|
|
|
int32_t earliest_indent = tokens[token_index].line_indent;
|
|
|
|
for (int32_t j = token_index - 1; j >= 0; --j) {
|
|
auto kind = tokens[j].kind;
|
|
|
|
if (IsClosingBracket(kind) && match_partner[j] != -1 &&
|
|
match_partner[j] < j) {
|
|
j = match_partner[j];
|
|
earliest_indent = tokens[j].line_indent;
|
|
continue;
|
|
}
|
|
|
|
if (kind == Kind::Semi || kind == Kind::OpenCurlyBrace ||
|
|
kind == Kind::CloseCurlyBrace) {
|
|
break;
|
|
}
|
|
|
|
earliest_indent = tokens[j].line_indent;
|
|
|
|
if (kind == Kind::StatementIntroducer) {
|
|
return GetOuterStatementIntroducerIndent(tokens, match_partner, j);
|
|
}
|
|
}
|
|
|
|
return earliest_indent;
|
|
}
|
|
|
|
// Determines if a token follows a statement/declaration header, so that a
|
|
// scope `{` could naturally be inserted directly before it. Only tokens that
|
|
// could start a body are considered: the first token on a line, a statement
|
|
// introducer (e.g. `return` in `if (c) return;`), or a token directly
|
|
// following a `)`/`]` that ends a header.
|
|
static auto ComputeFollowsStatementHeader(
|
|
llvm::ArrayRef<MismatchedBracketToken> tokens,
|
|
llvm::ArrayRef<int32_t> match_partner, int32_t token_index) -> bool {
|
|
if (token_index <= 0 || token_index >= static_cast<int32_t>(tokens.size())) {
|
|
return false;
|
|
}
|
|
auto curr_kind = tokens[token_index].kind;
|
|
if (curr_kind == Kind::Semi || curr_kind == Kind::OpenCurlyBrace ||
|
|
IsClosingBracket(curr_kind) || curr_kind == Kind::FileEnd) {
|
|
return false;
|
|
}
|
|
// Only tokens that could start a body are considered: the first token on a
|
|
// line, a statement introducer (`if (c) return;`), or a token directly
|
|
// after a `)` ending a header. (Not after `]`: declaration headers always
|
|
// continue after implicit parameter lists and `forall` clauses.)
|
|
bool is_first_on_line =
|
|
tokens[token_index].line != tokens[token_index - 1].line;
|
|
auto prev_kind = tokens[token_index - 1].kind;
|
|
if (!is_first_on_line && curr_kind != Kind::StatementIntroducer &&
|
|
prev_kind != Kind::CloseParen) {
|
|
return false;
|
|
}
|
|
// A body can't start with an operator like `as` or `==`, or with a `.`
|
|
// designator (a `where`-clause continuation line).
|
|
if (IsStructuralOpKind(curr_kind) || curr_kind == Kind::ComparisonOp ||
|
|
curr_kind == Kind::Period) {
|
|
return false;
|
|
}
|
|
for (int32_t j = token_index - 1; j >= 0; --j) {
|
|
auto kind = tokens[j].kind;
|
|
if ((kind == Kind::CloseParen || kind == Kind::CloseSquareBracket) &&
|
|
match_partner[j] != -1 && match_partner[j] < j) {
|
|
j = match_partner[j];
|
|
continue;
|
|
}
|
|
if (kind == Kind::Semi || kind == Kind::OpenCurlyBrace ||
|
|
kind == Kind::CloseCurlyBrace || kind == Kind::OpenParen ||
|
|
kind == Kind::OpenSquareBracket) {
|
|
return false;
|
|
}
|
|
if (kind == Kind::StatementIntroducer) {
|
|
if (curr_kind == Kind::StatementIntroducer) {
|
|
if (tokens[token_index].line == tokens[j].line) {
|
|
// On the same line, a chain of adjacent introducers (`private fn`)
|
|
// is a single header; but with other tokens in between, this is a
|
|
// body after a single-line header (`if (c) return x;`).
|
|
bool all_introducers = true;
|
|
for (int32_t k = j + 1; k < token_index; ++k) {
|
|
if (tokens[k].kind != Kind::StatementIntroducer) {
|
|
all_introducers = false;
|
|
break;
|
|
}
|
|
}
|
|
if (all_introducers) {
|
|
return false;
|
|
}
|
|
} else if (tokens[token_index].line_indent <= tokens[j].line_indent) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
// Determines if the statement/declaration header starting at token_index
|
|
// contains an OpenCurlyBrace.
|
|
static auto ComputeHeaderHasOpenCurlyBrace(
|
|
llvm::ArrayRef<MismatchedBracketToken> tokens,
|
|
llvm::ArrayRef<int32_t> match_partner, int32_t token_index) -> bool {
|
|
if (token_index <= 0 || token_index >= static_cast<int32_t>(tokens.size())) {
|
|
return false;
|
|
}
|
|
for (int32_t j = token_index; j < static_cast<int32_t>(tokens.size()); ++j) {
|
|
auto kind = tokens[j].kind;
|
|
if (kind == Kind::OpenCurlyBrace) {
|
|
return true;
|
|
}
|
|
if ((kind == Kind::OpenParen || kind == Kind::OpenSquareBracket) &&
|
|
match_partner[j] != -1 && match_partner[j] > j) {
|
|
j = match_partner[j];
|
|
continue;
|
|
}
|
|
if (kind == Kind::Semi || kind == Kind::CloseCurlyBrace ||
|
|
kind == Kind::FileEnd || kind == Kind::StatementIntroducer) {
|
|
return false;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
namespace {
|
|
|
|
struct ParentEdge {
|
|
int32_t parent_node_index;
|
|
BracketCorrection correction;
|
|
bool has_correction = false;
|
|
};
|
|
|
|
// Node in the beam search tree.
|
|
struct BeamNode {
|
|
int32_t item_index;
|
|
llvm::SmallVector<OpenBracketInfo, 4> stack;
|
|
int32_t cost;
|
|
// The kind of synthetic closer inserted directly before the current item,
|
|
// or Other if none; an inserted closer repairs illegal adjacency with the
|
|
// preceding token.
|
|
Kind closer_inserted = Kind::Other;
|
|
llvm::SmallVector<ParentEdge, 2> parent_edges;
|
|
};
|
|
|
|
// The parts of a BeamNode the search reads while expanding it. Snapshotting
|
|
// just these (rather than copying the whole node, whose `parent_edges` the
|
|
// expansion never reads) avoids copying that vector per node. A copy — not a
|
|
// reference — is needed because expanding a node pushes onto the node list,
|
|
// which may reallocate.
|
|
struct SearchState {
|
|
llvm::SmallVector<OpenBracketInfo, 4> stack;
|
|
int32_t cost;
|
|
Kind closer_inserted;
|
|
};
|
|
|
|
// What makes a search state distinct: the open-bracket stack, plus which
|
|
// closer, if any, was just inserted before the current token. Two nodes in a
|
|
// layer that agree on this are interchangeable and get merged, which is what
|
|
// `RegionSearch::layer_dedup_` looks up. This is a view of a node's state, or
|
|
// of a prospective one that has no node yet, so it must not outlive what it
|
|
// names.
|
|
//
|
|
// Only the `OpenBracketKey` part of each stack entry participates, since that's
|
|
// all `OpenBracketInfo`'s `operator==` compares and all the hash below reads.
|
|
struct StateKey {
|
|
llvm::ArrayRef<OpenBracketInfo> stack;
|
|
Kind closer_inserted;
|
|
|
|
friend auto operator==(const StateKey& lhs, const StateKey& rhs) -> bool {
|
|
return lhs.closer_inserted == rhs.closer_inserted && lhs.stack == rhs.stack;
|
|
}
|
|
|
|
// Hashes the whole of each stack entry's key, and nothing outside it, so that
|
|
// equal states always hash equal.
|
|
friend auto CarbonHashValue(const StateKey& key, uint64_t seed) -> HashCode {
|
|
Hasher hasher(seed);
|
|
hasher.Hash(key.closer_inserted, key.stack.size());
|
|
for (const OpenBracketKey& entry : key.stack) {
|
|
hasher.HashRaw(entry);
|
|
}
|
|
return static_cast<HashCode>(hasher);
|
|
}
|
|
};
|
|
|
|
// Lets the dedup table key on the search state itself while storing only a node
|
|
// index, so no stack is copied into the table. Lookups pass a `StateKey`
|
|
// directly; a stored index is translated back into one through the node list.
|
|
//
|
|
// Holds the node list by pointer, not by `ArrayRef`, because inserting a node
|
|
// reallocates it.
|
|
class LayerDedupKeyContext
|
|
: public TranslatingKeyContext<LayerDedupKeyContext> {
|
|
public:
|
|
explicit LayerDedupKeyContext(const llvm::SmallVector<BeamNode, 0>* nodes
|
|
[[clang::lifetimebound]])
|
|
: nodes_(nodes) {}
|
|
|
|
auto TranslateKey(int32_t node_index) const -> StateKey {
|
|
const BeamNode& node = (*nodes_)[node_index];
|
|
return {.stack = node.stack, .closer_inserted = node.closer_inserted};
|
|
}
|
|
|
|
private:
|
|
const llvm::SmallVector<BeamNode, 0>* nodes_;
|
|
};
|
|
|
|
} // namespace
|
|
|
|
static auto Snapshot(const BeamNode& node) -> SearchState {
|
|
return {node.stack, node.cost, node.closer_inserted};
|
|
}
|
|
|
|
// A correction that replaces a bracket token with an error token (the "give
|
|
// up on this bracket" repair).
|
|
static auto ReplaceWithError(const MismatchedBracketToken& token,
|
|
BracketDiagnosticKind diagnostic_kind,
|
|
llvm::StringLiteral rule_name)
|
|
-> BracketCorrection {
|
|
return BracketCorrection{
|
|
.diagnostic_kind = diagnostic_kind,
|
|
.diagnostic_token_index = token.token_index,
|
|
.fix_action = BracketFixAction::ReplaceWithError,
|
|
.fix_token_index = token.token_index,
|
|
.fix_token_kind = ToTokenKind(token.kind),
|
|
.rule_name = rule_name,
|
|
};
|
|
}
|
|
|
|
// Solve a damaged region using the simple greedy fallback algorithm.
|
|
static auto SolveNaive(llvm::ArrayRef<Item> items,
|
|
llvm::SmallVectorImpl<BracketCorrection>& corrections)
|
|
-> void {
|
|
llvm::SmallVector<MismatchedBracketToken> open_stack;
|
|
for (const auto& item : items) {
|
|
if (item.HasAll(Cue::CollapsedBlock)) {
|
|
continue;
|
|
}
|
|
auto kind = item.token.kind;
|
|
if (kind == Kind::Semi || kind == Kind::StatementIntroducer ||
|
|
kind == Kind::OpenCurlyBrace || kind == Kind::FileEnd) {
|
|
while (!open_stack.empty() &&
|
|
(open_stack.back().kind == Kind::OpenParen ||
|
|
open_stack.back().kind == Kind::OpenSquareBracket)) {
|
|
corrections.push_back(ReplaceWithError(
|
|
open_stack.pop_back_val(), BracketDiagnosticKind::UnmatchedOpening,
|
|
"Naive_UnclosedParenBracket"));
|
|
}
|
|
}
|
|
|
|
if (IsOpeningBracket(kind)) {
|
|
open_stack.push_back(item.token);
|
|
} else if (IsClosingBracket(kind)) {
|
|
auto search_range = llvm::reverse(open_stack);
|
|
size_t lookback = 0;
|
|
auto match_it = search_range.end();
|
|
for (auto it = search_range.begin();
|
|
it != search_range.end() && lookback < 16; ++it, ++lookback) {
|
|
if (MatchingClosingKind(it->kind) == kind) {
|
|
match_it = it;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (match_it == search_range.end()) {
|
|
corrections.push_back(ReplaceWithError(
|
|
item.token, BracketDiagnosticKind::UnmatchedClosing,
|
|
"Naive_UnmatchedClosing"));
|
|
} else {
|
|
for (auto it = search_range.begin(); it != match_it; ++it) {
|
|
corrections.push_back(
|
|
ReplaceWithError(*it, BracketDiagnosticKind::UnmatchedOpening,
|
|
"Naive_PoppedOpener"));
|
|
}
|
|
open_stack.erase(match_it.base() - 1, open_stack.end());
|
|
}
|
|
}
|
|
}
|
|
|
|
for (const auto& open : llvm::reverse(open_stack)) {
|
|
corrections.push_back(ReplaceWithError(
|
|
open, BracketDiagnosticKind::UnmatchedOpening, "Naive_UnclosedAtEnd"));
|
|
}
|
|
}
|
|
|
|
// Whether two parent edges represent the same predecessor and the same repair,
|
|
// so that keeping both would be a redundant duplicate.
|
|
static auto EdgesEqual(const ParentEdge& a, const ParentEdge& b) -> bool {
|
|
if (a.parent_node_index != b.parent_node_index ||
|
|
a.has_correction != b.has_correction) {
|
|
return false;
|
|
}
|
|
return !a.has_correction ||
|
|
(a.correction.diagnostic_token_index ==
|
|
b.correction.diagnostic_token_index &&
|
|
a.correction.fix_action == b.correction.fix_action &&
|
|
a.correction.fix_token_index == b.correction.fix_token_index &&
|
|
a.correction.fix_token_kind == b.correction.fix_token_kind);
|
|
}
|
|
|
|
// Returns true if `kind`'s matching opener appears in `stack` below the top.
|
|
static auto MatchesDeeperOpener(llvm::ArrayRef<OpenBracketInfo> stack,
|
|
Kind closing_kind) -> bool {
|
|
if (!IsClosingBracket(closing_kind) || stack.size() < 2) {
|
|
return false;
|
|
}
|
|
auto req = MatchingOpeningKind(closing_kind);
|
|
for (const OpenBracketKey& entry : stack.drop_back()) {
|
|
if (entry.kind == req) {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
// Whether the token before the current one ends a value. Unlike
|
|
// `IsValueEndingKind`, which the leaf-adjacency rules use, this also counts `]`
|
|
// and `}`, which end a value but can be followed by a leaf. `prev_token` is
|
|
// null at the start of the input.
|
|
static auto PrevIsValueLike(const MismatchedBracketToken* prev_token) -> bool {
|
|
return prev_token != nullptr &&
|
|
(IsValueEndingKind(prev_token->kind) ||
|
|
prev_token->kind == Kind::CloseSquareBracket ||
|
|
prev_token->kind == Kind::CloseCurlyBrace);
|
|
}
|
|
|
|
// Whether the top of `stack` is a synthetic opener inserted directly before
|
|
// `item` — i.e. this path just opened a bracket at this position.
|
|
static auto OpenerSynthesizedHere(llvm::ArrayRef<OpenBracketInfo> stack,
|
|
const Item& item) -> bool {
|
|
return !stack.empty() && stack.back().is_synthetic &&
|
|
stack.back().insertion_token_index == item.token.token_index;
|
|
}
|
|
|
|
// Whether a `]` or `}` was inserted directly before the current token. Such a
|
|
// closer repairs an illegal leaf adjacency (unlike `)`, these don't end a
|
|
// value). `Other` means no closer was inserted here.
|
|
static auto CloserFixesLeafAdjacency(Kind closer_inserted) -> bool {
|
|
return closer_inserted == Kind::CloseSquareBracket ||
|
|
closer_inserted == Kind::CloseCurlyBrace;
|
|
}
|
|
|
|
// The bracket-insertion rules below are expressed as data: each rule states
|
|
// the context it applies in and what that context costs, and the rules are
|
|
// tried in order so that the first (most specific) match wins. Two small
|
|
// categorical facts — a context category and the kind of the current token —
|
|
// form a bucket index, and a constexpr-built table maps each bucket to the
|
|
// bit-set of rules that can possibly apply there, so a lookup only tests the
|
|
// handful of rules relevant to the situation.
|
|
|
|
// Context categories for the closer and advance tables: the category of the
|
|
// innermost open bracket. `Struct` and `Scope` distinguish the two kinds of
|
|
// `{`. Each category has an index, which selects its row of bucket entries, and
|
|
// a bit, which is how a rule names it — so a rule can apply to several
|
|
// categories at once.
|
|
namespace {
|
|
|
|
namespace Top {
|
|
enum Index : int32_t {
|
|
ParenIndex,
|
|
SquareIndex,
|
|
StructIndex,
|
|
ScopeIndex,
|
|
// No open bracket at all. Only the advance table, which also scores tokens
|
|
// at the top level, uses this.
|
|
NoneIndex,
|
|
Count,
|
|
};
|
|
constexpr uint8_t Paren = 1 << ParenIndex;
|
|
constexpr uint8_t Square = 1 << SquareIndex;
|
|
constexpr uint8_t Struct = 1 << StructIndex;
|
|
constexpr uint8_t Scope = 1 << ScopeIndex;
|
|
constexpr uint8_t None = 1 << NoneIndex;
|
|
constexpr uint8_t ParenLike = Paren | Square;
|
|
constexpr uint8_t Any = ParenLike | Struct | Scope | None;
|
|
} // namespace Top
|
|
|
|
// Context categories for the opener table: which bracket would be inserted.
|
|
namespace Ins {
|
|
enum Index : int32_t {
|
|
ParenIndex,
|
|
SquareIndex,
|
|
ScopeBraceIndex,
|
|
StructBraceIndex,
|
|
Count,
|
|
};
|
|
constexpr uint8_t Paren = 1 << ParenIndex;
|
|
constexpr uint8_t Square = 1 << SquareIndex;
|
|
constexpr uint8_t ScopeBrace = 1 << ScopeBraceIndex;
|
|
constexpr uint8_t StructBrace = 1 << StructBraceIndex;
|
|
constexpr uint8_t ParenLike = Paren | Square;
|
|
} // namespace Ins
|
|
|
|
} // namespace
|
|
|
|
// Bucket rows are shared by all the tables, so there must be room for whichever
|
|
// set of context categories is larger.
|
|
constexpr int32_t NumContextCategories =
|
|
std::max<int32_t>(Top::Count, Ins::Count);
|
|
|
|
// The category of the innermost open bracket, which is always a real bracket.
|
|
static auto TopCategoryOf(const OpenBracketInfo& top) -> int32_t {
|
|
switch (top.kind) {
|
|
case Kind::OpenParen:
|
|
return Top::ParenIndex;
|
|
case Kind::OpenSquareBracket:
|
|
return Top::SquareIndex;
|
|
default:
|
|
return top.is_struct_brace ? Top::StructIndex : Top::ScopeIndex;
|
|
}
|
|
}
|
|
|
|
// As `TopCategoryOf`, but for a stack that may be empty.
|
|
static auto TopCategoryOfStack(llvm::ArrayRef<OpenBracketInfo> stack)
|
|
-> int32_t {
|
|
return stack.empty() ? Top::NoneIndex : TopCategoryOf(stack.back());
|
|
}
|
|
|
|
// The category of a synthetic opener of kind `kind`.
|
|
static auto InsCategoryOf(Kind kind, bool is_struct_brace) -> int32_t {
|
|
switch (kind) {
|
|
case Kind::OpenParen:
|
|
return Ins::ParenIndex;
|
|
case Kind::OpenSquareBracket:
|
|
return Ins::SquareIndex;
|
|
default:
|
|
return is_struct_brace ? Ins::StructBraceIndex : Ins::ScopeBraceIndex;
|
|
}
|
|
}
|
|
|
|
// Number of distinct `Kind` values.
|
|
constexpr int32_t NumKinds = static_cast<int32_t>(Kind::Other) + 1;
|
|
|
|
// A set of token kinds a rule can apply to. `Kind` values are the
|
|
// bit indexes, so there are no per-kind enumerators here to keep in sync with
|
|
// them; `Rule` takes the kinds themselves.
|
|
namespace {
|
|
|
|
enum class KindSet : uint32_t {
|
|
None = 0,
|
|
LLVM_MARK_AS_BITMASK_ENUM(uint32_t{1} << (NumKinds - 1))
|
|
};
|
|
|
|
} // namespace
|
|
|
|
// The set containing exactly `kinds`.
|
|
template <std::same_as<Kind>... KindT>
|
|
static constexpr auto KindSetOf(KindT... kinds) -> KindSet {
|
|
return (KindSet::None | ... |
|
|
static_cast<KindSet>(uint32_t{1} << static_cast<int>(kinds)));
|
|
}
|
|
|
|
// Sets of kinds that name a concept more than one rule shares. A rule wanting
|
|
// just one or two particular kinds names them directly instead.
|
|
namespace Kinds {
|
|
|
|
// Every kind: the default, for a rule that doesn't care.
|
|
constexpr KindSet Any = ~KindSet::None;
|
|
|
|
// A `(` or `[`, opening or closing: the bracket kinds written without a
|
|
// preceding space in formatted code.
|
|
constexpr KindSet OpenGroup =
|
|
KindSetOf(Kind::OpenParen, Kind::OpenSquareBracket);
|
|
constexpr KindSet CloseGroup =
|
|
KindSetOf(Kind::CloseParen, Kind::CloseSquareBracket);
|
|
|
|
// Any opening bracket, and everything that isn't one.
|
|
constexpr KindSet Opener = OpenGroup | KindSetOf(Kind::OpenCurlyBrace);
|
|
constexpr KindSet NonOpener = ~Opener;
|
|
|
|
// The statement-structuring operators together.
|
|
constexpr KindSet AnyStructural =
|
|
KindSetOf(Kind::Assignment, Kind::As, Kind::StructuralOp);
|
|
|
|
// A binary connector, which needs a left operand and so can't start a group.
|
|
constexpr KindSet BinaryConnector =
|
|
AnyStructural | KindSetOf(Kind::ComparisonOp);
|
|
|
|
// Everything that carries no bracket structure of its own: `Other` plus the
|
|
// operator and keyword classifications split out of it.
|
|
constexpr KindSet AnyOther =
|
|
AnyStructural |
|
|
KindSetOf(Kind::Other, Kind::ComparisonOp, Kind::ModifierKeyword);
|
|
|
|
// A token that could start a bracketed group.
|
|
constexpr KindSet GroupStarter =
|
|
Opener | KindSetOf(Kind::Leaf, Kind::Period, Kind::ModifierKeyword);
|
|
|
|
// A leaf, or a binding modifier keyword, which like a leaf can't directly
|
|
// follow a value-ending token.
|
|
constexpr KindSet LeafLike = KindSetOf(Kind::Leaf, Kind::ModifierKeyword);
|
|
|
|
// The kinds a dedent penalty can apply to: a closer is expected to dedent, and
|
|
// `FileEnd` isn't content.
|
|
constexpr KindSet Dedentable =
|
|
~(CloseGroup | KindSetOf(Kind::CloseCurlyBrace, Kind::FileEnd));
|
|
|
|
} // namespace Kinds
|
|
|
|
// A rule's `cost` when the move should not be considered at all.
|
|
constexpr int32_t DeclineCost = -1;
|
|
|
|
// A rule in a bracket-insertion table. The rule applies when the context
|
|
// category is in `ctx`, the current token's kind is in `kinds`, and all four
|
|
// cue conditions hold. Rules are tried in order and the first match wins, so
|
|
// earlier rules express stronger, more specific cues.
|
|
namespace {
|
|
|
|
struct BracketRule {
|
|
// Bit-set of context categories: `Top::` for the closer table (the category
|
|
// of the innermost open bracket), `Ins::` for the opener table (which bracket
|
|
// would be inserted).
|
|
uint8_t ctx;
|
|
// The token kinds this rule applies to.
|
|
KindSet kinds = Kinds::Any;
|
|
// Every cue listed here must hold.
|
|
CueSet when = CueSet::None;
|
|
// No cue listed here may hold.
|
|
CueSet unless = CueSet::None;
|
|
// The cues listed here must not all hold together.
|
|
CueSet not_all = CueSet::None;
|
|
// At least one cue listed here must hold, if any are listed.
|
|
CueSet any_of = CueSet::None;
|
|
// Cost of the insertion this rule proposes, or `DeclineCost`.
|
|
int32_t cost;
|
|
// See `BracketCorrection::rule_name`.
|
|
llvm::StringLiteral rule_name = "";
|
|
|
|
// Builders, so a rule reads as one sentence. Each returns an updated copy,
|
|
// and `Cost` or `Decline` finishes the rule.
|
|
template <typename... CueT>
|
|
constexpr auto When(CueT... cues) const -> BracketRule {
|
|
auto result = *this;
|
|
result.when = CueSetOf(cues...);
|
|
return result;
|
|
}
|
|
template <typename... CueT>
|
|
constexpr auto Unless(CueT... cues) const -> BracketRule {
|
|
auto result = *this;
|
|
result.unless = CueSetOf(cues...);
|
|
return result;
|
|
}
|
|
template <typename... CueT>
|
|
constexpr auto NotAll(CueT... cues) const -> BracketRule {
|
|
auto result = *this;
|
|
result.not_all = CueSetOf(cues...);
|
|
return result;
|
|
}
|
|
template <typename... CueT>
|
|
constexpr auto AnyOf(CueT... cues) const -> BracketRule {
|
|
auto result = *this;
|
|
result.any_of = CueSetOf(cues...);
|
|
return result;
|
|
}
|
|
constexpr auto Cost(int32_t rule_cost, llvm::StringLiteral name) const
|
|
-> BracketRule {
|
|
auto result = *this;
|
|
result.cost = rule_cost;
|
|
result.rule_name = name;
|
|
return result;
|
|
}
|
|
constexpr auto Decline() const -> BracketRule {
|
|
auto result = *this;
|
|
result.cost = DeclineCost;
|
|
return result;
|
|
}
|
|
};
|
|
|
|
} // namespace
|
|
|
|
// Starts a rule that applies to context categories `ctx` and token kinds
|
|
// `kinds`.
|
|
static constexpr auto Rule(uint8_t ctx, KindSet kinds = Kinds::Any)
|
|
-> BracketRule {
|
|
return BracketRule{.ctx = ctx, .kinds = kinds, .cost = DeclineCost};
|
|
}
|
|
|
|
// As above, for a rule that applies to a few particular kinds. Taking the first
|
|
// kind separately keeps the no-kinds call unambiguous.
|
|
template <std::same_as<Kind>... RestT>
|
|
static constexpr auto Rule(uint8_t ctx, Kind kind, RestT... rest)
|
|
-> BracketRule {
|
|
return Rule(ctx, KindSetOf(kind, rest...));
|
|
}
|
|
|
|
// Whether `rule`'s cue conditions hold for the cue bit-set `cues`.
|
|
static constexpr auto Matches(const BracketRule& rule, CueSet cues) -> bool {
|
|
return (cues & rule.when) == rule.when &&
|
|
(cues & rule.unless) == CueSet::None &&
|
|
(rule.not_all == CueSet::None ||
|
|
(cues & rule.not_all) != rule.not_all) &&
|
|
(rule.any_of == CueSet::None || (cues & rule.any_of) != CueSet::None);
|
|
}
|
|
|
|
// Where to insert a synthetic closing bracket, and what that costs. All costs
|
|
// are relative; see the cost model comment at the top of this file for what
|
|
// they mean and how to change one.
|
|
constexpr BracketRule CloserRules[] = {
|
|
// `(` and `[`.
|
|
//
|
|
// An empty group: the opener is directly followed by a token that can't
|
|
// start group content, so the group's closer must have been right after
|
|
// the opener. Such a token is a `,` (`f(x(), y)` becoming `f(x(, y)`); a
|
|
// binary connector like `as`/`->`/`==`, which needs a left operand
|
|
// (`f() as T` becoming `f( as T`); or a spaced `.`, which as group content
|
|
// would be written unspaced (`f(.a = 1)`).
|
|
Rule(Top::ParenLike, Kind::Comma)
|
|
.When(Cue::AfterOpenTop)
|
|
.Cost(4, "Close_EmptyGroup"),
|
|
Rule(Top::ParenLike, Kinds::BinaryConnector)
|
|
.When(Cue::AfterOpenTop)
|
|
.Cost(4, "Close_EmptyGroup"),
|
|
Rule(Top::ParenLike, Kind::Period)
|
|
.When(Cue::AfterOpenTop, Cue::LeadingSpace)
|
|
.Cost(4, "Close_EmptyGroup"),
|
|
// A `;` can't appear inside parens or square brackets at all.
|
|
Rule(Top::ParenLike, Kind::Semi).Cost(6, "Close_ParenBeforeSemi"),
|
|
Rule(Top::ParenLike).When(Cue::Cascade).Cost(3, "Close_ParenCascade"),
|
|
// A `{` starting a block means the paren should have closed: `if (c) {`,
|
|
// `while (c) {`. A struct-literal `{...}` can legitimately sit inside a
|
|
// *call* paren (`f({.x = 1})`), but not inside a keyword or grouping paren
|
|
// (whose `{` — even an empty `{}` misread as a struct — is a block). This
|
|
// is not a cue for `[`: a `]` is essentially never immediately followed by
|
|
// `{` (only in `fn [captures] {...}`, which is unimplemented), so a `[`
|
|
// should close at an earlier cue, or not here at all.
|
|
Rule(Top::Paren, Kind::OpenCurlyBrace)
|
|
.NotAll(Cue::StructBrace, Cue::CallParenTop)
|
|
.Cost(11, "Close_ParenBeforeBrace"),
|
|
// `=` can directly follow both `)` and `]`, but `->` and `as` only
|
|
// plausibly follow `)`. An unspaced structural operator is not a cue:
|
|
// formatted code spaces these operators, and an unspaced `->` is a
|
|
// pointer member access (`p->x`).
|
|
Rule(Top::Paren, Kinds::AnyStructural)
|
|
.When(Cue::LeadingSpace)
|
|
.Cost(8, "Close_ParenBeforeStructuralOp"),
|
|
Rule(Top::Square, Kind::Assignment)
|
|
.When(Cue::LeadingSpace)
|
|
.Cost(8, "Close_ParenBeforeStructuralOp"),
|
|
// A leaf directly following a value-ending token is illegal, and a `]`
|
|
// between them fixes the adjacency (unlike `)`, `]` can be directly
|
|
// followed by a leaf, as in `impl forall [...] T as ...`).
|
|
Rule(Top::Square, Kind::Leaf)
|
|
.When(Cue::PrevValueEnding)
|
|
.Cost(4, "Close_SquareAtLeafAdjacency"),
|
|
// A `.` with whitespace before it mid-line suggests a closer was deleted
|
|
// right before it: member access is written without spaces, `x.y`.
|
|
Rule(Top::ParenLike, Kind::Period)
|
|
.When(Cue::LeadingSpace, Cue::PrevValueLike)
|
|
.Unless(Cue::FirstOnLine)
|
|
.Cost(6, "Close_ParenBeforeSpacedPeriod"),
|
|
// Similarly, a `(` or `[` with whitespace before it directly following a
|
|
// value-ending token: calls and indexing are written without spaces.
|
|
Rule(Top::ParenLike, Kinds::OpenGroup)
|
|
.When(Cue::LeadingSpace, Cue::PrevValueLike)
|
|
.Unless(Cue::FirstOnLine)
|
|
.Cost(6, "Close_ParenBeforeSpacedOpen"),
|
|
// A comparison or logical operator is unlikely inside square brackets or
|
|
// call/index argument lists (but common in `if (...)` etc.).
|
|
Rule(Top::Square, Kind::ComparisonOp)
|
|
.Cost(11, "Close_ParenBeforeComparison"),
|
|
Rule(Top::Paren, Kind::ComparisonOp)
|
|
.When(Cue::CallParenTop)
|
|
.Cost(11, "Close_ParenBeforeComparison"),
|
|
// A `,` with whitespace before it: formatted code has no space before a
|
|
// comma, so a closer was likely deleted in the gap.
|
|
Rule(Top::ParenLike, Kind::Comma)
|
|
.When(Cue::LeadingSpace, Cue::PrevValueLike)
|
|
.Unless(Cue::FirstOnLine)
|
|
.Cost(6, "Close_BeforeSpacedComma"),
|
|
// Likewise a `)` or `]` with whitespace before it: formatted code has no
|
|
// space before closers either.
|
|
Rule(Top::ParenLike, Kinds::CloseGroup)
|
|
.When(Cue::LeadingSpace, Cue::PrevValueLike)
|
|
.Unless(Cue::FirstOnLine)
|
|
.Cost(6, "Close_BeforeSpacedCloser"),
|
|
Rule(Top::ParenLike, Kind::FileEnd)
|
|
.Cost(CostCloseAtEnd, "Close_ParenAtFileEnd"),
|
|
// A wide whitespace gap mid-line suggests a deleted token in the gap.
|
|
Rule(Top::ParenLike)
|
|
.When(Cue::WideGap)
|
|
.Unless(Cue::FirstOnLine)
|
|
.Cost(CostCloseAtWideGap, "Close_ParenAtWideGap"),
|
|
// A `[` group rarely spans lines except in wrapped declaration headers
|
|
// (`impl forall [...]` etc.), where the line break follows the `]`.
|
|
Rule(Top::Square)
|
|
.When(Cue::FirstOnLine, Cue::NewLineFromTop)
|
|
.Cost(10, "Close_SquareAtContinuation"),
|
|
// A block `{` can't be content of a header/grouping `[` (only an index
|
|
// `arr[...]` could hold a lambda block, but a `[` after a keyword can't):
|
|
// the `]` must close before it. A last-resort bound, below the precise
|
|
// cues above, that stops the `[` from swallowing the block. Priced above
|
|
// the precise cues (so they win) but below closing at the region end, so
|
|
// an unclosed group can't swallow a whole block.
|
|
Rule(Top::Square, Kind::OpenCurlyBrace)
|
|
.Unless(Cue::StructBrace, Cue::CallParenTop)
|
|
.Cost(14, "Close_SquareBeforeBlock"),
|
|
// No positive cue that a `(`/`[` closes here. Closing before a bare
|
|
// dedent, statement introducer, or arbitrary token was never a correct
|
|
// guess in practice (it just closes too early), so decline: the search
|
|
// will close at a real cue, at the region end, or, failing both, replace
|
|
// the unmatched opener with an error token.
|
|
Rule(Top::ParenLike).Decline(),
|
|
|
|
// Struct `{`.
|
|
Rule(Top::Struct, Kind::Semi).Cost(6, "Close_StructBeforeSemi"),
|
|
Rule(Top::Struct).When(Cue::Cascade).Cost(6, "Close_StructCascade"),
|
|
// A block `{` can't be content of a struct literal/type `{...}` (a struct
|
|
// field is `.name = value`, and a bare `{` isn't a value here): the struct
|
|
// must close before it, as in `-> {.x: i32} { body }`.
|
|
Rule(Top::Struct, Kind::OpenCurlyBrace)
|
|
.Unless(Cue::StructBrace)
|
|
.Cost(14, "Close_StructBeforeBlock"),
|
|
Rule(Top::Struct)
|
|
.When(Cue::WideGap)
|
|
.Unless(Cue::FirstOnLine)
|
|
.Cost(CostCloseAtWideGap, "Close_StructAtWideGap"),
|
|
Rule(Top::Struct, Kind::FileEnd)
|
|
.Cost(CostCloseAtEnd, "Close_StructAtFileEnd"),
|
|
Rule(Top::Struct)
|
|
.When(Cue::FirstOnLine, Cue::DedentToHeader)
|
|
.Cost(12, "Close_StructAtDedent"),
|
|
Rule(Top::Struct).Cost(40, "Close_StructBaseline"),
|
|
|
|
// Scope `{`.
|
|
// A first-on-line `else` must have been preceded by the `}` closing the
|
|
// branch before it. Ordered ahead of Close_ScopeAtDedent, which would
|
|
// otherwise match first and charge more for the same insertion.
|
|
Rule(Top::Scope)
|
|
.When(Cue::ElseKeyword, Cue::FirstOnLine)
|
|
.Cost(4, "Close_ScopeBeforeElse"),
|
|
Rule(Top::Scope)
|
|
.When(Cue::FirstOnLine, Cue::DedentToHeader)
|
|
.Cost(6, "Close_ScopeAtDedent"),
|
|
Rule(Top::Scope).When(Cue::Cascade).Cost(6, "Close_ScopeCascade"),
|
|
Rule(Top::Scope, Kind::FileEnd)
|
|
.Cost(CostCloseAtEnd, "Close_ScopeAtFileEnd"),
|
|
Rule(Top::Scope).Cost(45, "Close_ScopeBaseline"),
|
|
};
|
|
|
|
// A bucket index: rules are identified by a bit in a `uint64_t`.
|
|
static_assert(std::size(CloserRules) <= 64);
|
|
|
|
// Maps each (context category, token kind) bucket to the bit-set of rules that
|
|
// can apply in it, so a lookup tests only those rules, in table order.
|
|
using RuleIndex =
|
|
std::array<std::array<uint64_t, NumKinds>, NumContextCategories>;
|
|
|
|
template <size_t N>
|
|
static constexpr auto BuildRuleIndex(const BracketRule (&rules)[N])
|
|
-> RuleIndex {
|
|
RuleIndex index = {};
|
|
for (size_t r = 0; r != N; ++r) {
|
|
for (int32_t t = 0; t != NumContextCategories; ++t) {
|
|
if ((rules[r].ctx & (1 << t)) == 0) {
|
|
continue;
|
|
}
|
|
for (int32_t k = 0; k != NumKinds; ++k) {
|
|
if ((rules[r].kinds & KindSetOf(static_cast<Kind>(k))) ==
|
|
KindSet::None) {
|
|
continue;
|
|
}
|
|
index[t][k] |= uint64_t{1} << r;
|
|
}
|
|
}
|
|
}
|
|
return index;
|
|
}
|
|
|
|
// The bit-set of rules that could apply in a bucket. A rule is identified by
|
|
// its bit position, which is its position in the table, so scanning the bits
|
|
// from lowest to highest visits the rules in table order.
|
|
static constexpr auto CandidateRules(const RuleIndex& index,
|
|
int32_t ctx_category, Kind kind)
|
|
-> uint64_t {
|
|
return index[ctx_category][static_cast<int32_t>(kind)];
|
|
}
|
|
|
|
// Returns the first rule that applies, for a first-match table, or null if
|
|
// none does.
|
|
template <size_t N>
|
|
static auto FindMatchingRule(const BracketRule (&rules)[N],
|
|
const RuleIndex& index, int32_t ctx_category,
|
|
Kind kind, CueSet cues) -> const BracketRule* {
|
|
uint64_t candidates = CandidateRules(index, ctx_category, kind);
|
|
while (candidates != 0) {
|
|
const auto& rule = rules[std::countr_zero(candidates)];
|
|
candidates &= candidates - 1;
|
|
if (Matches(rule, cues)) {
|
|
return &rule;
|
|
}
|
|
}
|
|
return nullptr;
|
|
}
|
|
|
|
// Returns the total cost of every rule that applies, for an additive table.
|
|
template <size_t N>
|
|
static auto SumMatchingRules(const BracketRule (&rules)[N],
|
|
const RuleIndex& index, int32_t ctx_category,
|
|
Kind kind, CueSet cues) -> int32_t {
|
|
int32_t total = 0;
|
|
uint64_t candidates = CandidateRules(index, ctx_category, kind);
|
|
while (candidates != 0) {
|
|
const auto& rule = rules[std::countr_zero(candidates)];
|
|
candidates &= candidates - 1;
|
|
if (Matches(rule, cues)) {
|
|
total += rule.cost;
|
|
}
|
|
}
|
|
return total;
|
|
}
|
|
|
|
static constexpr auto CloserRuleIndex = BuildRuleIndex(CloserRules);
|
|
|
|
// The cue for the previous token having kind `kind`, if any.
|
|
static constexpr auto PrevKindCue(Kind kind) -> CueSet {
|
|
switch (kind) {
|
|
case Kind::Period:
|
|
return CueSetOf(Cue::PrevIsPeriod);
|
|
case Kind::OpenParen:
|
|
case Kind::OpenSquareBracket:
|
|
return CueSetOf(Cue::PrevIsOpenBracket);
|
|
case Kind::OpenCurlyBrace:
|
|
return CueSetOf(Cue::PrevIsOpenCurly);
|
|
case Kind::CloseParen:
|
|
return CueSetOf(Cue::PrevIsCloseParen);
|
|
case Kind::CloseCurlyBrace:
|
|
return CueSetOf(Cue::PrevIsCloseCurly);
|
|
case Kind::CloseSquareBracket:
|
|
return CueSetOf(Cue::PrevIsCloseSquare);
|
|
case Kind::Comma:
|
|
return CueSetOf(Cue::PrevIsComma);
|
|
default:
|
|
return CueSet::None;
|
|
}
|
|
}
|
|
|
|
// Returns the set holding `cue` if `holds`, and the empty set otherwise.
|
|
static constexpr auto CueIf(bool holds, Cue cue) -> CueSet {
|
|
return holds ? CueSetOf(cue) : CueSet::None;
|
|
}
|
|
|
|
// Computes every cue that depends only on an item and its neighbours.
|
|
// `prev_token` is the token directly before the item and `prev_item` the item
|
|
// directly before it, both null at the start of the input. `context_cues` holds
|
|
// the cues the caller has already determined from the surrounding token arrays
|
|
// (`Cue::CollapsedBlock`, `Cue::FirstOnLine`, and so on).
|
|
static auto ComputeItemCues(const MismatchedBracketToken& token,
|
|
const MismatchedBracketToken* prev_token,
|
|
const Item* prev_item, CueSet context_cues)
|
|
-> CueSet {
|
|
CueSet cues = context_cues |
|
|
CueIf(token.has_leading_space, Cue::LeadingSpace) |
|
|
CueIf(token.has_wide_leading_space, Cue::WideGap) |
|
|
CueIf(token.is_else_keyword, Cue::ElseKeyword) |
|
|
CueIf(token.is_struct_brace, Cue::StructBrace) |
|
|
CueIf(PrevIsValueLike(prev_token), Cue::PrevValueLike);
|
|
if (prev_token != nullptr) {
|
|
// `forall` requires a following `[`; the other paren keywords (`if`,
|
|
// `while`, `for`, `match`) are statement introducers and require a `(`.
|
|
bool wants_paren = prev_token->kind == Kind::StatementIntroducer;
|
|
cues |= CueIf(prev_token->is_paren_keyword && wants_paren,
|
|
Cue::PrevKeywordWantsParen) |
|
|
CueIf(prev_token->is_paren_keyword && !wants_paren,
|
|
Cue::PrevKeywordWantsSquare) |
|
|
CueIf(prev_token->has_leading_space, Cue::PrevHasLeadingSpace) |
|
|
CueIf(IsValueEndingKind(prev_token->kind), Cue::PrevValueEnding) |
|
|
CueIf(prev_token->kind == Kind::As ||
|
|
prev_token->kind == Kind::StructuralOp,
|
|
Cue::PrevIntroducesType) |
|
|
PrevKindCue(prev_token->kind);
|
|
}
|
|
if (prev_item != nullptr) {
|
|
// A name directly following `as` or `->` is a type, not something
|
|
// callable, so empty parens after it are implausible.
|
|
cues |= CueIf(prev_item->token.kind == Kind::Leaf, Cue::PrevItemIsLeaf) |
|
|
CueIf(prev_item->HasAll(Cue::PrevIntroducesType),
|
|
Cue::PrevItemIsTypeName);
|
|
}
|
|
return cues;
|
|
}
|
|
|
|
// Computes the cues that depend on the innermost open bracket `top` and the
|
|
// search state, to combine with `item.cues`.
|
|
static auto ComputeTopCues(const OpenBracketInfo& top, const Item& item,
|
|
llvm::ArrayRef<OpenBracketInfo> stack) -> CueSet {
|
|
const auto& token = item.token;
|
|
return CueIf(top.is_call_paren, Cue::CallParenTop) |
|
|
CueIf(MatchesDeeperOpener(stack, token.kind), Cue::Cascade) |
|
|
CueIf(top.token_pos == item.token_start_index - 1, Cue::AfterOpenTop) |
|
|
CueIf(token.line_indent <= top.effective_header_indent,
|
|
Cue::DedentToHeader) |
|
|
CueIf(token.line != top.line, Cue::NewLineFromTop);
|
|
}
|
|
|
|
// Computes the cost of inserting a synthetic closer for `top` directly before
|
|
// `item`, or nullopt if this insertion isn't worth exploring. Sets `rule_name`
|
|
// to the name of the rule that fired.
|
|
static auto ClassifyCloserInsertion(const OpenBracketInfo& top,
|
|
const Item& item,
|
|
llvm::ArrayRef<OpenBracketInfo> stack,
|
|
llvm::StringLiteral& rule_name)
|
|
-> std::optional<int32_t> {
|
|
CueSet cues = item.cues | ComputeTopCues(top, item, stack);
|
|
const auto* rule = FindMatchingRule(
|
|
CloserRules, CloserRuleIndex, TopCategoryOf(top), item.token.kind, cues);
|
|
if (rule == nullptr || rule->cost == DeclineCost) {
|
|
return std::nullopt;
|
|
}
|
|
rule_name = rule->rule_name;
|
|
return rule->cost;
|
|
}
|
|
|
|
// Where to insert a synthetic opening bracket, and what that costs. A `(` or
|
|
// `[` can always be synthesized, just expensively without a cue, so those end
|
|
// in a baseline; a brace is only proposed where a cue supports it.
|
|
constexpr BracketRule OpenerRules[] = {
|
|
// `if`/`while`/`for`/`match` (statement introducers) require a following
|
|
// `(`; `forall` (an Other token) requires a following `[`.
|
|
Rule(Ins::Paren, Kinds::NonOpener)
|
|
.When(Cue::PrevKeywordWantsParen)
|
|
.Cost(3, "Open_AfterParenKeyword"),
|
|
Rule(Ins::Square, Kinds::NonOpener)
|
|
.When(Cue::PrevKeywordWantsSquare)
|
|
.Cost(3, "Open_AfterParenKeyword"),
|
|
// A leaf or binding modifier directly following a value-ending token is
|
|
// illegal; an opener here fixes the adjacency.
|
|
Rule(Ins::ParenLike, Kinds::LeafLike)
|
|
.When(Cue::PrevValueEnding)
|
|
.Cost(3, "Open_AtLeafAdjacency"),
|
|
// A `.` with whitespace before it directly following a value-ending
|
|
// token: likely a designator argument that lost its `(`, as in
|
|
// `ImplicitAs(.Self)`.
|
|
Rule(Ins::Paren, Kind::Period)
|
|
.When(Cue::LeadingSpace, Cue::PrevValueEnding)
|
|
.Unless(Cue::FirstOnLine)
|
|
.Cost(CostCloseParenBeforeSpacedPeriod, "Open_BeforeSpacedPeriod"),
|
|
// A mid-line leaf with whitespace before it directly following an
|
|
// unspaced `.`: member access is written without spaces, `x.y`, so a
|
|
// bracket was likely deleted in the gap.
|
|
Rule(Ins::ParenLike, Kind::Leaf)
|
|
.When(Cue::LeadingSpace, Cue::PrevIsPeriod)
|
|
.Unless(Cue::FirstOnLine, Cue::PrevHasLeadingSpace)
|
|
.Cost(4, "Open_AfterPeriodGap"),
|
|
// A mid-line leaf with whitespace before it directly following an
|
|
// opener: formatted code has no space after `(` or `[`, so a bracket was
|
|
// likely deleted in the gap.
|
|
Rule(Ins::ParenLike, Kind::Leaf)
|
|
.When(Cue::LeadingSpace, Cue::PrevIsOpenBracket)
|
|
.Unless(Cue::FirstOnLine)
|
|
.Cost(4, "Open_AfterOpenGap"),
|
|
// A wide whitespace gap before a token that could start a group suggests
|
|
// an opener was deleted in the gap.
|
|
Rule(Ins::ParenLike, Kinds::GroupStarter)
|
|
.When(Cue::WideGap)
|
|
.Unless(Cue::FirstOnLine)
|
|
.Cost(CostCloseAtWideGap, "Open_AtWideGap"),
|
|
// An empty group directly after a name: `Op()`. Only applies after a
|
|
// leaf (a call of a just-computed value, `f(x)()`, is much rarer than a
|
|
// call of a name), and not when the name is a type after `as` or `->`,
|
|
// where a parenthesized group is more plausible than an empty call.
|
|
Rule(Ins::Paren, Kind::CloseParen)
|
|
.When(Cue::PrevItemIsLeaf, Cue::LeadingSpace)
|
|
.Unless(Cue::PrevItemIsTypeName)
|
|
.Cost(5, "Open_EmptyParens"),
|
|
// An empty `[]` is rarer than empty parens.
|
|
Rule(Ins::Square, Kind::CloseSquareBracket)
|
|
.When(Cue::PrevItemIsLeaf, Cue::LeadingSpace)
|
|
.Unless(Cue::PrevItemIsTypeName)
|
|
.Cost(12, "Open_EmptySquares"),
|
|
// An empty call of a just-computed value: `T.(Default.Op)()`. Only
|
|
// trusted when the `)` is spaced, marking the deletion gap. This looks at
|
|
// the last token of the previous item, so it fires after a collapsed
|
|
// `(...)` block too.
|
|
Rule(Ins::Paren, Kind::CloseParen)
|
|
.When(Cue::PrevIsCloseParen, Cue::LeadingSpace)
|
|
.Unless(Cue::FirstOnLine)
|
|
.Cost(8, "Open_EmptyParensAfterClose"),
|
|
// A `(` or `[` anywhere else an expression could start.
|
|
Rule(Ins::ParenLike).Cost(70, "Open_ParenBaseline"),
|
|
// A scope `{` is never inserted directly before another opener: the body
|
|
// it would open starts with that opener, so the `{` belongs before it only
|
|
// via one of the rules above.
|
|
Rule(Ins::ScopeBrace, Kinds::Opener).Decline(),
|
|
// A scope `{` between an unbraced declaration or statement header and its
|
|
// body, as in `if (c) return;`.
|
|
Rule(Ins::ScopeBrace)
|
|
.When(Cue::FollowsStatementHeader)
|
|
.Unless(Cue::HeaderHasOpenCurly)
|
|
.Cost(8, "Open_ScopeAfterHeader"),
|
|
Rule(Ins::ScopeBrace).Cost(30, "Open_ScopeBaseline"),
|
|
// A struct `{` before a `.` designator that isn't a member access.
|
|
Rule(Ins::StructBrace, Kind::Period)
|
|
.Unless(Cue::PrevValueEnding)
|
|
.Cost(5, "Open_StructBeforeDesignator"),
|
|
// A struct literal `{...}` that lost its `{`, leaving content directly
|
|
// before the `}`. Real content is required, so a stray `}` is reported as
|
|
// an error instead. Priced above Open_ScopeAfterHeader: when a single-line
|
|
// body lost its `{`, inserting it before the body beats making empty
|
|
// braces at the `}`.
|
|
Rule(Ins::StructBrace, Kind::CloseCurlyBrace)
|
|
.Unless(Cue::FirstOnLine)
|
|
.AnyOf(Cue::PrevValueEnding, Cue::PrevIsCloseCurly,
|
|
Cue::PrevIsCloseSquare, Cue::PrevIsComma)
|
|
.Cost(10, "Open_StructEmptyBraces"),
|
|
// Any other struct brace has no cue at all, and isn't worth proposing.
|
|
Rule(Ins::StructBrace).Decline(),
|
|
};
|
|
|
|
static_assert(std::size(OpenerRules) <= 64);
|
|
|
|
static constexpr auto OpenerRuleIndex = BuildRuleIndex(OpenerRules);
|
|
|
|
// Computes the cost of inserting a synthetic opener directly before `item`, or
|
|
// nullopt if this insertion isn't worth exploring. Sets `rule_name` to the name
|
|
// of the rule that fired.
|
|
static auto ClassifyOpenerInsertion(Kind kind, bool is_struct_brace,
|
|
const Item& item,
|
|
llvm::StringLiteral& rule_name)
|
|
-> std::optional<int32_t> {
|
|
const auto* rule = FindMatchingRule(OpenerRules, OpenerRuleIndex,
|
|
InsCategoryOf(kind, is_struct_brace),
|
|
item.token.kind, item.cues);
|
|
if (rule == nullptr || rule->cost == DeclineCost) {
|
|
return std::nullopt;
|
|
}
|
|
rule_name = rule->rule_name;
|
|
return rule->cost;
|
|
}
|
|
|
|
// Penalties for advancing over an item in a context where it doesn't belong.
|
|
// These make "close the group before this token" win over "swallow this token
|
|
// into the group". Unlike the tables above, this one is additive: every
|
|
// matching rule contributes, and the penalties sum.
|
|
constexpr BracketRule AdvanceRules[] = {
|
|
// A line that dedents to at-or-before the indentation of the enclosing
|
|
// brace's statement header, or of the enclosing paren's own line, while
|
|
// the bracket is still open. Closing brackets are excluded, since a `}`
|
|
// that closes its group is expected to dedent, and `FileEnd` isn't
|
|
// content at all.
|
|
Rule(Top::Struct | Top::Scope, Kinds::Dedentable)
|
|
.When(Cue::FirstOnLine, Cue::DedentToHeader)
|
|
.Cost(40, "Adv_DedentInScope"),
|
|
Rule(Top::ParenLike, Kinds::Dedentable)
|
|
.When(Cue::FirstOnLine, Cue::DedentToOpenerLine)
|
|
.Cost(25, "Adv_DedentInParen"),
|
|
// A scope `{` inside parens or a struct brace is a lambda, which is rare;
|
|
// a struct `{` there is a struct literal argument, which is common. The
|
|
// first rule covers a collapsed block that contains a scope brace.
|
|
Rule(Top::ParenLike | Top::Struct, Kinds::Opener)
|
|
.When(Cue::CollapsedBlock, Cue::ContainsScopeBrace)
|
|
.Cost(40, "Adv_ScopeBlockInParen"),
|
|
Rule(Top::ParenLike | Top::Struct, Kind::OpenCurlyBrace)
|
|
.Unless(Cue::CollapsedBlock, Cue::StructBrace)
|
|
.Cost(40, "Adv_ScopeBraceInParen"),
|
|
Rule(Top::ParenLike | Top::Struct, Kind::OpenCurlyBrace)
|
|
.When(Cue::StructBrace)
|
|
.Unless(Cue::CollapsedBlock)
|
|
.Cost(5, "Adv_StructBraceInParen"),
|
|
// A spaced `(` or `[` directly after a value: calls and indexing are
|
|
// written without spaces, so a bracket was likely deleted in the gap —
|
|
// unless this path already inserted a closer there.
|
|
Rule(Top::Any, Kinds::OpenGroup)
|
|
.When(Cue::LeadingSpace, Cue::PrevValueLike)
|
|
.Unless(Cue::FirstOnLine, Cue::CollapsedBlock, Cue::CloserInserted)
|
|
.Cost(10, "Adv_SpacedOpenAfterValue"),
|
|
// Formatted code has no space before a closer either.
|
|
Rule(Top::Any, Kinds::CloseGroup)
|
|
.When(Cue::LeadingSpace, Cue::PrevValueLike)
|
|
.Unless(Cue::FirstOnLine, Cue::BracketInsertedHere)
|
|
.Cost(8, "Adv_SpacedCloserUnexplained"),
|
|
// A `;` can't appear inside parens, square brackets, or a struct brace.
|
|
Rule(Top::ParenLike | Top::Struct, Kind::Semi).Cost(100, "Adv_SemiInParen"),
|
|
// A `,` at statement level: directly in a scope brace or at the top level.
|
|
Rule(Top::None | Top::Scope, Kind::Comma)
|
|
.Cost(50, "Adv_CommaAtStatementLevel"),
|
|
// A `,` directly following a still-open `(`/`[` is illegal.
|
|
Rule(Top::ParenLike | Top::Struct, Kind::Comma)
|
|
.When(Cue::AfterOpenTop)
|
|
.Cost(50, "Adv_CommaAfterOpen"),
|
|
// Formatted code has no space before a `,`, so a bracket was likely
|
|
// deleted in the gap.
|
|
Rule(Top::ParenLike | Top::Struct, Kind::Comma)
|
|
.When(Cue::LeadingSpace, Cue::PrevValueLike)
|
|
.Unless(Cue::FirstOnLine, Cue::CloserInserted, Cue::AfterOpenTop)
|
|
.Cost(8, "Adv_SpacedCommaInParen"),
|
|
// A statement introducer keyword inside parens or a struct brace.
|
|
Rule(Top::ParenLike | Top::Struct, Kind::StatementIntroducer)
|
|
.Cost(60, "Adv_IntroducerInParen"),
|
|
// A leaf, or a binding modifier keyword, directly following a value-ending
|
|
// token is an illegal adjacency — unless an opener synthesized here, or a
|
|
// `]`/`}` inserted here, repairs it.
|
|
Rule(Top::Any, Kinds::LeafLike)
|
|
.When(Cue::PrevValueEnding)
|
|
.Unless(Cue::OpenerHere, Cue::CloserFixesAdjacency)
|
|
.Cost(60, "Adv_LeafAdjacency"),
|
|
// `=`, `->`, or `as` inside parens or square brackets. These *can* occur
|
|
// there (default arguments, function types, casts), so this is mild; it
|
|
// serves to prefer the earliest sensible close point. Casts `(x as T)` are
|
|
// common enough that `as` keeps only a nominal preference.
|
|
Rule(Top::ParenLike, Kind::Assignment, Kind::StructuralOp)
|
|
.When(Cue::LeadingSpace)
|
|
.Cost(5, "Adv_StructuralOpInParen"),
|
|
Rule(Top::ParenLike, Kind::As)
|
|
.When(Cue::LeadingSpace)
|
|
.Cost(1, "Adv_AsOpInParen"),
|
|
// A comparison or logical operator inside square brackets or a call.
|
|
Rule(Top::Square, Kind::ComparisonOp).Cost(8, "Adv_ComparisonInSquare"),
|
|
Rule(Top::Paren, Kind::ComparisonOp)
|
|
.When(Cue::CallParenTop)
|
|
.Cost(8, "Adv_ComparisonInCall"),
|
|
// A wide mid-line whitespace gap suggests a deleted bracket that this path
|
|
// hasn't repaired.
|
|
Rule(Top::Any, Kinds::AnyOther)
|
|
.When(Cue::WideGap)
|
|
.Unless(Cue::FirstOnLine, Cue::BracketInsertedHere)
|
|
.Cost(10, "Adv_WideGapUnexplained"),
|
|
// A mid-line `.` with whitespace before it suggests a deleted bracket:
|
|
// member access is written without spaces. Prefer closing an open group
|
|
// before it, or opening one.
|
|
Rule(Top::Any, Kind::Period)
|
|
.When(Cue::LeadingSpace)
|
|
.Unless(Cue::FirstOnLine, Cue::BracketInsertedHere)
|
|
.AnyOf(Cue::PrevValueLike, Cue::PrevIsOpenBracket, Cue::PrevIsOpenCurly)
|
|
.Cost(10, "Adv_SpacedPeriodInParen"),
|
|
};
|
|
|
|
static_assert(std::size(AdvanceRules) <= 64);
|
|
|
|
// Declining is meaningless in an additive table: every matching rule
|
|
// contributes its cost, so a declining rule would subtract from the penalty.
|
|
// This also catches a rule that forgot its `Cost`.
|
|
static_assert([] {
|
|
for (const BracketRule& rule : AdvanceRules) {
|
|
if (rule.cost == DeclineCost) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}());
|
|
|
|
static constexpr auto AdvanceRuleIndex = BuildRuleIndex(AdvanceRules);
|
|
|
|
// Computes the cues for advancing over `item` in search state `node`.
|
|
static auto ComputeAdvanceCues(const SearchState& node, const Item& item)
|
|
-> CueSet {
|
|
bool opener_here = OpenerSynthesizedHere(node.stack, item);
|
|
bool closer_here = node.closer_inserted != Kind::Other;
|
|
CueSet cues = item.cues | CueIf(closer_here, Cue::CloserInserted) |
|
|
CueIf(CloserFixesLeafAdjacency(node.closer_inserted),
|
|
Cue::CloserFixesAdjacency) |
|
|
CueIf(opener_here, Cue::OpenerHere) |
|
|
CueIf(closer_here || opener_here, Cue::BracketInsertedHere);
|
|
if (node.stack.empty()) {
|
|
return cues;
|
|
}
|
|
const auto& top = node.stack.back();
|
|
return cues | CueIf(top.is_call_paren, Cue::CallParenTop) |
|
|
CueIf(top.token_pos == item.token_start_index - 1, Cue::AfterOpenTop) |
|
|
CueIf(item.token.line_indent <= top.effective_header_indent,
|
|
Cue::DedentToHeader) |
|
|
CueIf(item.token.line_indent <= top.line_indent,
|
|
Cue::DedentToOpenerLine);
|
|
}
|
|
|
|
// Computes the total penalty for advancing over `item` in search state `node`,
|
|
// summing every `AdvanceRules` entry that matches.
|
|
static auto AdvancePenalty(const SearchState& node, const Item& item)
|
|
-> int32_t {
|
|
return SumMatchingRules(AdvanceRules, AdvanceRuleIndex,
|
|
TopCategoryOfStack(node.stack), item.token.kind,
|
|
ComputeAdvanceCues(node, item));
|
|
}
|
|
|
|
// Enumerates the distinct optimal repairs, by walking parent edges from each
|
|
// goal node back to the root and collecting the corrections along the way. At
|
|
// most `MaxOptimalPaths` are returned, so a correction that only a path beyond
|
|
// that cap would dispute stays untied.
|
|
static auto EnumerateOptimalPaths(llvm::ArrayRef<BeamNode> nodes,
|
|
llvm::ArrayRef<int32_t> goal_node_indices)
|
|
-> llvm::SmallVector<llvm::SmallVector<BracketCorrection>> {
|
|
llvm::SmallVector<llvm::SmallVector<BracketCorrection>> all_paths;
|
|
llvm::SmallVector<BracketCorrection> current_path;
|
|
struct StackFrame {
|
|
int32_t node_index;
|
|
int32_t edge_index = 0;
|
|
};
|
|
llvm::SmallVector<StackFrame> stack;
|
|
for (int32_t goal_idx : goal_node_indices) {
|
|
stack.push_back({.node_index = goal_idx, .edge_index = 0});
|
|
}
|
|
while (!stack.empty()) {
|
|
auto& frame = stack.back();
|
|
const auto& node = nodes[frame.node_index];
|
|
if (node.parent_edges.empty()) {
|
|
// The root, so `current_path` is now a complete repair, in reverse.
|
|
all_paths.push_back(current_path);
|
|
std::reverse(all_paths.back().begin(), all_paths.back().end());
|
|
stack.pop_back();
|
|
if (all_paths.size() >= MaxOptimalPaths) {
|
|
break;
|
|
}
|
|
continue;
|
|
}
|
|
if (frame.edge_index > 0) {
|
|
const auto& prev_edge = node.parent_edges[frame.edge_index - 1];
|
|
if (prev_edge.has_correction) {
|
|
current_path.pop_back();
|
|
}
|
|
}
|
|
if (static_cast<size_t>(frame.edge_index) < node.parent_edges.size()) {
|
|
const auto& edge = node.parent_edges[frame.edge_index];
|
|
++frame.edge_index;
|
|
if (edge.has_correction) {
|
|
current_path.push_back(edge.correction);
|
|
}
|
|
stack.push_back({.node_index = edge.parent_node_index, .edge_index = 0});
|
|
} else {
|
|
stack.pop_back();
|
|
}
|
|
}
|
|
return all_paths;
|
|
}
|
|
|
|
// Finds which of the first optimal path's corrections the optimal repairs
|
|
// disagree about. Each of its corrections is matched against an `equivalent`
|
|
// one in every other path; one with no counterpart in some path is tied.
|
|
static auto FindTiedCorrections(
|
|
llvm::ArrayRef<llvm::SmallVector<BracketCorrection>> all_paths,
|
|
llvm::function_ref<bool(const BracketCorrection&, const BracketCorrection&)>
|
|
equivalent) -> llvm::BitVector {
|
|
llvm::ArrayRef<BracketCorrection> baseline_path = all_paths.front();
|
|
llvm::BitVector tied(baseline_path.size());
|
|
llvm::BitVector used;
|
|
for (const auto& path : all_paths) {
|
|
used.reset();
|
|
used.resize(path.size());
|
|
for (auto [corr_idx, corr] : llvm::enumerate(baseline_path)) {
|
|
bool found = false;
|
|
for (auto [path_idx, path_corr] : llvm::enumerate(path)) {
|
|
if (!used[path_idx] && equivalent(path_corr, corr)) {
|
|
used.set(path_idx);
|
|
found = true;
|
|
break;
|
|
}
|
|
}
|
|
if (!found) {
|
|
tied.set(corr_idx);
|
|
}
|
|
}
|
|
}
|
|
return tied;
|
|
}
|
|
|
|
// From the optimal goal nodes, reconstructs the repair corrections and appends
|
|
// them to `corrections`. Every optimal repair is enumerated (up to a cap); a
|
|
// correction that the optimal repairs disagree about is marked tied, so the
|
|
// caller downgrades it to an error token rather than guessing. Falls back to
|
|
// naive recovery if no path can be reconstructed.
|
|
static auto ReconstructCorrections(
|
|
llvm::ArrayRef<BeamNode> nodes, llvm::ArrayRef<int32_t> goal_node_indices,
|
|
llvm::ArrayRef<Item> items, TokenIndex region_end_token,
|
|
llvm::SmallVectorImpl<BracketCorrection>& corrections) -> void {
|
|
auto all_paths = EnumerateOptimalPaths(nodes, goal_node_indices);
|
|
if (all_paths.empty()) {
|
|
SolveNaive(items, corrections);
|
|
return;
|
|
}
|
|
|
|
// Two insertions of the same bracket kind are equivalent if every token
|
|
// between their insertion points is that same kind: inserting a `)` on
|
|
// either side of an existing `)` produces the same token sequence.
|
|
Map<int32_t, int32_t> token_to_item;
|
|
for (auto [idx, region_item] : llvm::enumerate(items)) {
|
|
token_to_item.Update(region_item.token.token_index.index,
|
|
static_cast<int32_t>(idx));
|
|
}
|
|
token_to_item.Update(region_end_token.index,
|
|
static_cast<int32_t>(items.size()));
|
|
// Compares only the fixes, not the diagnosed brackets: two paths that
|
|
// blame different brackets but repair the token stream identically don't
|
|
// disagree about the repair.
|
|
auto corrections_equivalent = [&](const BracketCorrection& a,
|
|
const BracketCorrection& b) -> bool {
|
|
if (a.fix_action != b.fix_action || a.fix_token_kind != b.fix_token_kind) {
|
|
return false;
|
|
}
|
|
if (a.fix_token_index == b.fix_token_index) {
|
|
return true;
|
|
}
|
|
if (a.fix_action != BracketFixAction::InsertBefore) {
|
|
return false;
|
|
}
|
|
int32_t* a_item = token_to_item[a.fix_token_index.index];
|
|
int32_t* b_item = token_to_item[b.fix_token_index.index];
|
|
if (a_item == nullptr || b_item == nullptr) {
|
|
return false;
|
|
}
|
|
auto [lo, hi] = std::minmax(*a_item, *b_item);
|
|
for (const Item& between : items.slice(lo, hi - lo)) {
|
|
if (between.HasAll(Cue::CollapsedBlock) ||
|
|
ToTokenKind(between.token.kind) != a.fix_token_kind) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
};
|
|
|
|
llvm::BitVector tied = FindTiedCorrections(all_paths, corrections_equivalent);
|
|
for (auto [corr_idx, corr] : llvm::enumerate(all_paths.front())) {
|
|
corrections.push_back(corr);
|
|
corrections.back().is_tied = tied[corr_idx];
|
|
}
|
|
}
|
|
|
|
// Whether matching the real closer `item` directly against `top` is strictly
|
|
// better than synthesizing a duplicate closer in front of it. That's normally
|
|
// the case, but not when the closer isn't allowed to match `top` directly, and
|
|
// not when it has suspicious whitespace before it suggesting a closer was
|
|
// deleted in the gap.
|
|
static auto DirectMatchPreferred(const OpenBracketInfo& top, const Item& item)
|
|
-> bool {
|
|
auto kind = item.token.kind;
|
|
bool direct_match_ok = kind != Kind::CloseCurlyBrace || top.is_struct_brace ||
|
|
item.token.line == top.line ||
|
|
item.token.line_indent >= top.effective_header_indent;
|
|
bool spaced_suspicious = kind != Kind::CloseCurlyBrace &&
|
|
!item.HasAll(Cue::FirstOnLine) &&
|
|
item.HasAll(Cue::LeadingSpace, Cue::PrevValueLike);
|
|
return direct_match_ok && !spaced_suspicious;
|
|
}
|
|
|
|
namespace {
|
|
|
|
// A flavor of synthetic opening bracket the search can insert.
|
|
struct SyntheticOpener {
|
|
Kind kind;
|
|
bool is_struct_brace;
|
|
};
|
|
|
|
} // namespace
|
|
|
|
// The synthetic openers the search considers inserting before a token, in the
|
|
// order they're proposed. Each is offered to `ClassifyOpenerInsertion`, which
|
|
// decides whether the context supports it.
|
|
constexpr SyntheticOpener SyntheticOpeners[] = {
|
|
{Kind::OpenParen, /*is_struct_brace=*/false},
|
|
{Kind::OpenSquareBracket, /*is_struct_brace=*/false},
|
|
{Kind::OpenCurlyBrace, /*is_struct_brace=*/false},
|
|
{Kind::OpenCurlyBrace, /*is_struct_brace=*/true},
|
|
};
|
|
|
|
// The stack entry for the real opening bracket `item`.
|
|
static auto RealOpener(const Item& item) -> OpenBracketInfo {
|
|
return OpenBracketInfo{OpenBracketKey{
|
|
.token_index = item.token.token_index,
|
|
.token_pos = item.token_start_index,
|
|
.line = item.token.line,
|
|
.line_indent = item.token.line_indent,
|
|
.effective_header_indent = item.effective_header_indent,
|
|
.kind = item.token.kind,
|
|
.is_struct_brace = item.token.is_struct_brace,
|
|
.is_call_paren = item.HasAll(Cue::PrevValueEnding),
|
|
}};
|
|
}
|
|
|
|
// The extra cost of matching the real closer `item` against `top`, or nullopt
|
|
// if the match isn't allowed. A multi-line scope close must not be dedented
|
|
// past its header, and pays for indentation disagreement with it.
|
|
static auto MatchClosePenalty(const OpenBracketInfo& top, const Item& item)
|
|
-> std::optional<int32_t> {
|
|
if (item.token.kind != Kind::CloseCurlyBrace || top.is_struct_brace ||
|
|
item.token.line == top.line) {
|
|
return 0;
|
|
}
|
|
if (item.token.line_indent < top.effective_header_indent) {
|
|
return std::nullopt;
|
|
}
|
|
if (item.HasAll(Cue::FirstOnLine) &&
|
|
item.token.line_indent != top.effective_header_indent) {
|
|
return CostBraceIndentMismatchBase +
|
|
CostBraceIndentMismatchPerColumn *
|
|
std::abs(top.effective_header_indent - item.token.line_indent);
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
// The correction that records where the synthetic opener `opener`, which the
|
|
// real closer `closer` has just matched, would be inserted.
|
|
static auto InsertOpenerCorrection(const OpenBracketInfo& opener,
|
|
const MismatchedBracketToken& closer)
|
|
-> BracketCorrection {
|
|
return BracketCorrection{
|
|
.diagnostic_kind = BracketDiagnosticKind::UnmatchedClosing,
|
|
.diagnostic_token_index = closer.token_index,
|
|
.fix_action = BracketFixAction::InsertBefore,
|
|
.fix_token_index = opener.insertion_token_index,
|
|
.fix_token_kind = ToTokenKind(opener.kind),
|
|
.rule_name = opener.rule_name,
|
|
};
|
|
}
|
|
|
|
// The cost of closing an open bracket at the end of the file or region.
|
|
static auto CostToCloseAtEnd(const OpenBracketInfo& open) -> int32_t {
|
|
if (open.kind != Kind::OpenCurlyBrace) {
|
|
return CostCloseParenAtEnd;
|
|
}
|
|
return open.is_struct_brace ? CostCloseStructAtEnd : CostCloseAtEnd;
|
|
}
|
|
|
|
// A layered beam search over one damaged region.
|
|
//
|
|
// Each layer holds the states that survive just before one item. Expanding a
|
|
// layer first proposes bracket insertions before the item — epsilon moves,
|
|
// which stay within the layer — and then advances over the item into the next
|
|
// layer, pruning each layer back to `MaxBeamWidth`. States in a layer that
|
|
// agree on their open-bracket stack merge into one node, which keeps every
|
|
// cheapest way of reaching it so that ties can be found afterwards.
|
|
namespace {
|
|
|
|
class RegionSearch {
|
|
public:
|
|
// `region_end_token` is the token directly after the region, where any
|
|
// still-unclosed brackets are closed.
|
|
RegionSearch(llvm::ArrayRef<Item> items, TokenIndex region_end_token)
|
|
: items_(items), region_end_token_(region_end_token) {
|
|
nodes_.reserve(256);
|
|
}
|
|
|
|
// Runs the search and appends the corrections for the cheapest repair to
|
|
// `corrections`.
|
|
auto Solve(llvm::SmallVectorImpl<BracketCorrection>& corrections) -> void;
|
|
|
|
private:
|
|
// Adds the state reached by `edge` at total cost `next_cost` to `layer`,
|
|
// merging it into an equal state already there if there is one.
|
|
// `next_item_idx` is the item the state sits before. If `worklist` is given,
|
|
// any node that was added or became cheaper is appended to it, so that a
|
|
// further epsilon move can be applied to it.
|
|
auto AddToLayer(llvm::SmallVectorImpl<int32_t>& layer, int32_t next_item_idx,
|
|
llvm::SmallVector<OpenBracketInfo, 4> next_stack,
|
|
Kind closer_inserted, int32_t next_cost, ParentEdge edge,
|
|
llvm::SmallVectorImpl<int32_t>* worklist = nullptr) -> void;
|
|
|
|
// Merges a newly-found way of reaching the state already in `nodes_[idx]`: a
|
|
// cheaper cost replaces what's recorded there, and an equal cost adds another
|
|
// parent edge, so that every cheapest path stays available.
|
|
auto MergeIntoNode(int32_t idx, int32_t cost, const ParentEdge& edge,
|
|
llvm::SmallVectorImpl<int32_t>* worklist) -> void;
|
|
|
|
// Keeps a layer within the beam width by discarding the costliest states.
|
|
auto PruneBeam(llvm::SmallVectorImpl<int32_t>& layer) -> void;
|
|
|
|
// Adds to the current layer every state reachable by inserting brackets
|
|
// directly before item `item_idx`.
|
|
auto InsertBracketsBefore(int32_t item_idx) -> void;
|
|
auto InsertSyntheticClosers(int32_t item_idx) -> void;
|
|
auto InsertSyntheticOpeners(int32_t item_idx) -> void;
|
|
|
|
// Advances the current layer over item `item_idx`, returning the layer that
|
|
// results.
|
|
auto AdvanceOverItem(int32_t item_idx) -> llvm::SmallVector<int32_t>;
|
|
auto AdvanceNode(int32_t node_idx, int32_t item_idx,
|
|
llvm::SmallVectorImpl<int32_t>& next_layer) -> void;
|
|
|
|
// Closes whatever is still open at the end of the region, returning the goal
|
|
// nodes of the cheapest complete repairs.
|
|
auto CloseAtRegionEnd() -> llvm::SmallVector<int32_t>;
|
|
|
|
llvm::ArrayRef<Item> items_;
|
|
TokenIndex region_end_token_;
|
|
|
|
// Every state the search has reached, in creation order; a layer names its
|
|
// states by their index here. Nodes are never removed, so pruning a layer
|
|
// leaves its states behind, unreachable.
|
|
llvm::SmallVector<BeamNode, 0> nodes_;
|
|
// The cost of the cheapest complete repair found so far, which bounds the
|
|
// cost of any state still worth expanding.
|
|
int32_t min_goal_cost_ = std::numeric_limits<int32_t>::max();
|
|
llvm::SmallVector<int32_t> current_layer_;
|
|
// The states in the layer currently being built, named by their index in
|
|
// `nodes_` and keyed on the state itself, so that reaching a state already in
|
|
// the layer merges into it. Kept across layers only to reuse its allocation.
|
|
Set<int32_t, /*SmallSize=*/0, LayerDedupKeyContext> layer_dedup_;
|
|
};
|
|
|
|
} // namespace
|
|
|
|
auto RegionSearch::AddToLayer(llvm::SmallVectorImpl<int32_t>& layer,
|
|
int32_t next_item_idx,
|
|
llvm::SmallVector<OpenBracketInfo, 4> next_stack,
|
|
Kind closer_inserted, int32_t next_cost,
|
|
ParentEdge edge,
|
|
llvm::SmallVectorImpl<int32_t>* worklist)
|
|
-> void {
|
|
if (next_cost > min_goal_cost_) {
|
|
return;
|
|
}
|
|
LayerDedupKeyContext key_context(&nodes_);
|
|
StateKey key = {.stack = next_stack, .closer_inserted = closer_inserted};
|
|
if (auto existing = layer_dedup_.Lookup(key, key_context)) {
|
|
MergeIntoNode(existing.key(), next_cost, edge, worklist);
|
|
return;
|
|
}
|
|
auto new_idx = static_cast<int32_t>(nodes_.size());
|
|
nodes_.push_back(BeamNode{
|
|
.item_index = next_item_idx,
|
|
.stack = std::move(next_stack),
|
|
.cost = next_cost,
|
|
.closer_inserted = closer_inserted,
|
|
.parent_edges = {edge},
|
|
});
|
|
layer.push_back(new_idx);
|
|
layer_dedup_.Insert(new_idx, key_context);
|
|
if (worklist) {
|
|
worklist->push_back(new_idx);
|
|
}
|
|
}
|
|
|
|
auto RegionSearch::MergeIntoNode(int32_t idx, int32_t cost,
|
|
const ParentEdge& edge,
|
|
llvm::SmallVectorImpl<int32_t>* worklist)
|
|
-> void {
|
|
BeamNode& node = nodes_[idx];
|
|
if (cost < node.cost) {
|
|
node.cost = cost;
|
|
node.parent_edges.clear();
|
|
node.parent_edges.push_back(edge);
|
|
if (worklist) {
|
|
worklist->push_back(idx);
|
|
}
|
|
} else if (cost == node.cost &&
|
|
llvm::none_of(node.parent_edges, [&](const ParentEdge& e) {
|
|
return EdgesEqual(e, edge);
|
|
})) {
|
|
node.parent_edges.push_back(edge);
|
|
}
|
|
}
|
|
|
|
auto RegionSearch::PruneBeam(llvm::SmallVectorImpl<int32_t>& layer) -> void {
|
|
if (layer.size() > MaxBeamWidth) {
|
|
llvm::stable_sort(layer, [&](int32_t a, int32_t b) {
|
|
return nodes_[a].cost < nodes_[b].cost;
|
|
});
|
|
layer.resize(MaxBeamWidth);
|
|
}
|
|
}
|
|
|
|
auto RegionSearch::InsertBracketsBefore(int32_t item_idx) -> void {
|
|
// Seed the dedup table with the states already in the layer, so that an
|
|
// insertion reaching one of them merges into it instead of duplicating it.
|
|
for (int32_t idx : current_layer_) {
|
|
layer_dedup_.Insert(idx, LayerDedupKeyContext(&nodes_));
|
|
}
|
|
InsertSyntheticClosers(item_idx);
|
|
InsertSyntheticOpeners(item_idx);
|
|
layer_dedup_.Clear();
|
|
PruneBeam(current_layer_);
|
|
}
|
|
|
|
auto RegionSearch::InsertSyntheticClosers(int32_t item_idx) -> void {
|
|
const Item& item = items_[item_idx];
|
|
// A worklist rather than one pass over the layer, so that several groups can
|
|
// be closed at the same point.
|
|
llvm::SmallVector<int32_t> worklist = current_layer_;
|
|
for (size_t head = 0; head < worklist.size(); ++head) {
|
|
const SearchState current = Snapshot(nodes_[worklist[head]]);
|
|
if (current.cost > min_goal_cost_ || current.stack.empty()) {
|
|
continue;
|
|
}
|
|
const auto& top = current.stack.back();
|
|
// Synthetic openers exist only to consume real closers; closing one
|
|
// synthetically would insert a pointless empty pair.
|
|
if (top.is_synthetic) {
|
|
continue;
|
|
}
|
|
if (item.token.kind == MatchingClosingKind(top.kind) &&
|
|
DirectMatchPreferred(top, item)) {
|
|
continue;
|
|
}
|
|
llvm::StringLiteral rule_name = "";
|
|
auto cost = ClassifyCloserInsertion(top, item, current.stack, rule_name);
|
|
if (!cost) {
|
|
continue;
|
|
}
|
|
|
|
auto next_stack = current.stack;
|
|
auto popped = next_stack.pop_back_val();
|
|
auto closer_kind = MatchingClosingKind(popped.kind);
|
|
AddToLayer(
|
|
current_layer_, item_idx, std::move(next_stack), closer_kind,
|
|
current.cost + *cost,
|
|
ParentEdge{
|
|
.parent_node_index = worklist[head],
|
|
.correction =
|
|
BracketCorrection{
|
|
.diagnostic_kind = BracketDiagnosticKind::UnmatchedOpening,
|
|
.diagnostic_token_index = popped.token_index,
|
|
.fix_action = BracketFixAction::InsertBefore,
|
|
.fix_token_index = item.token.token_index,
|
|
.fix_token_kind = ToTokenKind(closer_kind),
|
|
.rule_name = rule_name,
|
|
},
|
|
.has_correction = true,
|
|
},
|
|
&worklist);
|
|
}
|
|
}
|
|
|
|
auto RegionSearch::InsertSyntheticOpeners(int32_t item_idx) -> void {
|
|
const Item& item = items_[item_idx];
|
|
// Iterate only over the states present after the closer phase, so that
|
|
// synthetic openers don't chain onto each other. The bound is taken before
|
|
// the loop because the loop appends to the layer.
|
|
for (size_t idx : llvm::seq<size_t>(0, current_layer_.size())) {
|
|
int32_t node_idx = current_layer_[idx];
|
|
const SearchState current = Snapshot(nodes_[node_idx]);
|
|
if (current.cost > min_goal_cost_ ||
|
|
current.stack.size() >= MaxSearchStackDepth) {
|
|
continue;
|
|
}
|
|
for (auto [open_kind, is_struct_brace] : SyntheticOpeners) {
|
|
llvm::StringLiteral rule_name = "";
|
|
auto cost =
|
|
ClassifyOpenerInsertion(open_kind, is_struct_brace, item, rule_name);
|
|
if (!cost) {
|
|
continue;
|
|
}
|
|
auto next_stack = current.stack;
|
|
next_stack.push_back(OpenBracketInfo{
|
|
OpenBracketKey{
|
|
.insertion_token_index = item.token.token_index,
|
|
.line = item.token.line,
|
|
.line_indent = item.token.line_indent,
|
|
.effective_header_indent = item.effective_header_indent,
|
|
.kind = open_kind,
|
|
.is_synthetic = true,
|
|
.is_struct_brace = is_struct_brace,
|
|
},
|
|
rule_name,
|
|
});
|
|
AddToLayer(current_layer_, item_idx, std::move(next_stack),
|
|
current.closer_inserted, current.cost + *cost,
|
|
ParentEdge{.parent_node_index = node_idx});
|
|
}
|
|
}
|
|
}
|
|
|
|
auto RegionSearch::AdvanceOverItem(int32_t item_idx)
|
|
-> llvm::SmallVector<int32_t> {
|
|
llvm::SmallVector<int32_t> next_layer;
|
|
for (int32_t node_idx : current_layer_) {
|
|
AdvanceNode(node_idx, item_idx, next_layer);
|
|
}
|
|
layer_dedup_.Clear();
|
|
PruneBeam(next_layer);
|
|
return next_layer;
|
|
}
|
|
|
|
auto RegionSearch::AdvanceNode(int32_t node_idx, int32_t item_idx,
|
|
llvm::SmallVectorImpl<int32_t>& next_layer)
|
|
-> void {
|
|
const SearchState current = Snapshot(nodes_[node_idx]);
|
|
if (current.cost > min_goal_cost_) {
|
|
return;
|
|
}
|
|
const Item& item = items_[item_idx];
|
|
auto kind = item.token.kind;
|
|
|
|
auto advance = [&](llvm::SmallVector<OpenBracketInfo, 4> next_stack,
|
|
int32_t add_cost, BracketCorrection correction = {},
|
|
bool has_correction = false) {
|
|
AddToLayer(next_layer, item_idx + 1, std::move(next_stack), Kind::Other,
|
|
current.cost + add_cost,
|
|
ParentEdge{.parent_node_index = node_idx,
|
|
.correction = correction,
|
|
.has_correction = has_correction});
|
|
};
|
|
|
|
// What it costs to advance over this item in this state, per the
|
|
// `AdvanceRules` table.
|
|
int32_t penalty = AdvancePenalty(current, item);
|
|
|
|
if (item.HasAll(Cue::CollapsedBlock)) {
|
|
advance(current.stack, penalty);
|
|
return;
|
|
}
|
|
|
|
if (IsOpeningBracket(kind)) {
|
|
// Advance, pushing the opener onto the stack.
|
|
if (current.stack.size() < MaxSearchStackDepth) {
|
|
auto next_stack = current.stack;
|
|
next_stack.push_back(RealOpener(item));
|
|
advance(std::move(next_stack), penalty);
|
|
}
|
|
// Advance without pushing: replace the unmatched opener with an error
|
|
// token.
|
|
advance(
|
|
current.stack, CostReplaceOpening,
|
|
ReplaceWithError(item.token, BracketDiagnosticKind::UnmatchedOpening,
|
|
"Adv_ReplaceOpener"),
|
|
/*has_correction=*/true);
|
|
return;
|
|
}
|
|
|
|
if (IsClosingBracket(kind)) {
|
|
// Advance, popping the opener this closer matches.
|
|
if (!current.stack.empty() &&
|
|
current.stack.back().kind == MatchingOpeningKind(kind)) {
|
|
if (auto match_cost = MatchClosePenalty(current.stack.back(), item)) {
|
|
auto next_stack = current.stack;
|
|
auto popped = next_stack.pop_back_val();
|
|
// Matching a synthetic opener finally pins down where it goes, so this
|
|
// is where it becomes a correction.
|
|
advance(std::move(next_stack), penalty + *match_cost,
|
|
popped.is_synthetic ? InsertOpenerCorrection(popped, item.token)
|
|
: BracketCorrection{},
|
|
popped.is_synthetic);
|
|
}
|
|
}
|
|
// Advance without matching: replace the unmatched closer with an error
|
|
// token.
|
|
advance(
|
|
current.stack, CostReplaceClosing,
|
|
ReplaceWithError(item.token, BracketDiagnosticKind::UnmatchedClosing,
|
|
"Adv_ReplaceCloser"),
|
|
/*has_correction=*/true);
|
|
return;
|
|
}
|
|
|
|
// Any other token.
|
|
advance(current.stack, penalty);
|
|
}
|
|
|
|
auto RegionSearch::CloseAtRegionEnd() -> llvm::SmallVector<int32_t> {
|
|
llvm::SmallVector<int32_t> goal_node_indices;
|
|
for (int32_t node_idx : current_layer_) {
|
|
const SearchState current = Snapshot(nodes_[node_idx]);
|
|
if (current.cost > min_goal_cost_) {
|
|
continue;
|
|
}
|
|
// A synthetic opener that never matched a real closer is a meaningless
|
|
// insertion; reject such states rather than dropping it silently.
|
|
if (llvm::any_of(current.stack, [](const OpenBracketInfo& open) {
|
|
return open.is_synthetic;
|
|
})) {
|
|
continue;
|
|
}
|
|
// Close what's left innermost-first, one node per closer, so that each is
|
|
// reconstructed as its own correction.
|
|
int32_t finish_cost = current.cost;
|
|
int32_t parent = node_idx;
|
|
for (const auto& open : llvm::reverse(current.stack)) {
|
|
finish_cost += CostToCloseAtEnd(open);
|
|
nodes_.push_back(BeamNode{
|
|
.item_index = static_cast<int32_t>(items_.size()),
|
|
.stack = {},
|
|
.cost = finish_cost,
|
|
.parent_edges = {{
|
|
.parent_node_index = parent,
|
|
.correction =
|
|
BracketCorrection{
|
|
.diagnostic_kind =
|
|
BracketDiagnosticKind::UnmatchedOpening,
|
|
.diagnostic_token_index = open.token_index,
|
|
.fix_action = BracketFixAction::InsertBefore,
|
|
.fix_token_index = region_end_token_,
|
|
.fix_token_kind =
|
|
ToTokenKind(MatchingClosingKind(open.kind)),
|
|
.rule_name = "Close_RegionEnd"},
|
|
.has_correction = true,
|
|
}},
|
|
});
|
|
parent = static_cast<int32_t>(nodes_.size()) - 1;
|
|
}
|
|
|
|
if (finish_cost < min_goal_cost_) {
|
|
min_goal_cost_ = finish_cost;
|
|
goal_node_indices.clear();
|
|
}
|
|
if (finish_cost == min_goal_cost_) {
|
|
goal_node_indices.push_back(parent);
|
|
}
|
|
}
|
|
return goal_node_indices;
|
|
}
|
|
|
|
auto RegionSearch::Solve(llvm::SmallVectorImpl<BracketCorrection>& corrections)
|
|
-> void {
|
|
nodes_.push_back(BeamNode{
|
|
.item_index = 0,
|
|
.stack = {},
|
|
.cost = 0,
|
|
.parent_edges = {},
|
|
});
|
|
current_layer_ = {0};
|
|
|
|
for (auto [item_index, item] : llvm::enumerate(items_)) {
|
|
auto i = static_cast<int32_t>(item_index);
|
|
// Nothing is inserted before the end of the file; brackets still open there
|
|
// are closed by `CloseAtRegionEnd` instead.
|
|
if (item.token.kind != Kind::FileEnd) {
|
|
InsertBracketsBefore(i);
|
|
}
|
|
current_layer_ = AdvanceOverItem(i);
|
|
}
|
|
|
|
llvm::SmallVector<int32_t> goal_node_indices = CloseAtRegionEnd();
|
|
if (goal_node_indices.empty()) {
|
|
SolveNaive(items_, corrections);
|
|
return;
|
|
}
|
|
ReconstructCorrections(nodes_, goal_node_indices, items_, region_end_token_,
|
|
corrections);
|
|
}
|
|
|
|
// Solve a damaged region using layered beam search with tie detection.
|
|
// `region_end_token` is the token directly after the region, where any
|
|
// still-unclosed brackets are closed.
|
|
static auto SolveRegionCostBased(
|
|
llvm::ArrayRef<Item> items, TokenIndex region_end_token,
|
|
llvm::SmallVectorImpl<BracketCorrection>& corrections) -> void {
|
|
if (items.size() > static_cast<size_t>(MaxRegionItemsForSearch)) {
|
|
SolveNaive(items, corrections);
|
|
return;
|
|
}
|
|
RegionSearch(items, region_end_token).Solve(corrections);
|
|
}
|
|
|
|
// The passes below run in order over the whole token sequence, each consuming
|
|
// the results of the ones before it: analyze the tokens, decide which matched
|
|
// bracket pairs can be trusted and collapsed, build the item sequence from
|
|
// that, then split it into regions and search the damaged ones.
|
|
|
|
// Finds the stack position of the opener in `open_stack` that the closing
|
|
// bracket `tokens[closer_index]` should pair with, or -1 if none is plausible.
|
|
// A `}` matches a `{` only when they're on the same line, the `{` is a struct
|
|
// brace, or their line indentation agrees.
|
|
static auto FindMatchingOpener(llvm::ArrayRef<MismatchedBracketToken> tokens,
|
|
llvm::ArrayRef<int32_t> open_stack,
|
|
int32_t closer_index) -> int32_t {
|
|
const auto& closer = tokens[closer_index];
|
|
auto num_open = static_cast<int32_t>(open_stack.size());
|
|
for (int32_t s : llvm::reverse(llvm::seq(0, num_open))) {
|
|
const auto& opener = tokens[open_stack[s]];
|
|
if (closer.kind != Kind::CloseCurlyBrace) {
|
|
if (opener.kind == MatchingOpeningKind(closer.kind)) {
|
|
return s;
|
|
}
|
|
} else if (opener.kind == Kind::OpenCurlyBrace &&
|
|
(opener.line == closer.line || opener.is_struct_brace ||
|
|
opener.line_indent == closer.line_indent)) {
|
|
return s;
|
|
}
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
// Pairs up brackets by a stack walk, returning for each token the index of the
|
|
// bracket it pairs with, or -1 if it has none. A closer that doesn't match the
|
|
// top of the stack pops through to a plausible match if one exists (leaving the
|
|
// popped brackets unmatched), and is otherwise left unmatched without
|
|
// disturbing the stack.
|
|
static auto MatchBracketPairs(llvm::ArrayRef<MismatchedBracketToken> tokens)
|
|
-> llvm::SmallVector<int32_t> {
|
|
auto num_tokens = static_cast<int32_t>(tokens.size());
|
|
llvm::SmallVector<int32_t> match_partner(num_tokens, -1);
|
|
llvm::SmallVector<int32_t> open_stack;
|
|
for (int32_t i : llvm::seq(0, num_tokens)) {
|
|
auto kind = tokens[i].kind;
|
|
if (IsOpeningBracket(kind)) {
|
|
open_stack.push_back(i);
|
|
} else if (IsClosingBracket(kind)) {
|
|
int32_t match_s = FindMatchingOpener(tokens, open_stack, i);
|
|
if (match_s != -1) {
|
|
match_partner[open_stack[match_s]] = i;
|
|
match_partner[i] = open_stack[match_s];
|
|
open_stack.resize(match_s);
|
|
}
|
|
}
|
|
}
|
|
return match_partner;
|
|
}
|
|
|
|
namespace {
|
|
|
|
// The statement segments of the token sequence, which are separated by `;`,
|
|
// `{`, and `}`.
|
|
struct Segments {
|
|
// The id of each token's segment. Segments are numbered in order.
|
|
llvm::SmallVector<int32_t> id;
|
|
// The index of the first token of each token's segment.
|
|
llvm::SmallVector<int32_t> first;
|
|
};
|
|
|
|
// The indexes of the unmatched `(`, `[`, `)`, and `]` tokens, each list in
|
|
// increasing order so it can be binary-searched.
|
|
struct UnmatchedGroupBrackets {
|
|
llvm::SmallVector<int32_t> open_parens;
|
|
llvm::SmallVector<int32_t> open_squares;
|
|
llvm::SmallVector<int32_t> close_parens;
|
|
llvm::SmallVector<int32_t> close_squares;
|
|
|
|
// The openers and closers of the bracket kind that `open_kind` opens.
|
|
auto OpenersOfKind(Kind open_kind) const -> llvm::ArrayRef<int32_t> {
|
|
return open_kind == Kind::OpenParen ? open_parens : open_squares;
|
|
}
|
|
auto ClosersOfKind(Kind open_kind) const -> llvm::ArrayRef<int32_t> {
|
|
return open_kind == Kind::OpenParen ? close_parens : close_squares;
|
|
}
|
|
};
|
|
|
|
// Everything the later passes need to know about the token sequence as a whole,
|
|
// computed once up front by `AnalyzeTokens`. The vectors are all indexed by
|
|
// token index.
|
|
struct TokenAnalysis {
|
|
// The index of the bracket each token pairs with, or -1. See
|
|
// `MatchBracketPairs`.
|
|
llvm::SmallVector<int32_t> match_partner;
|
|
Segments segments;
|
|
UnmatchedGroupBrackets unmatched;
|
|
// See `ComputeAssociatedLineIndent`.
|
|
llvm::SmallVector<int32_t> effective_header_indent;
|
|
llvm::BitVector is_first_on_line;
|
|
// See `ComputeFollowsStatementHeader` and `ComputeHeaderHasOpenCurlyBrace`.
|
|
// The latter is only computed where the former holds.
|
|
llvm::BitVector follows_statement_header;
|
|
llvm::BitVector header_has_open_curly_brace;
|
|
};
|
|
|
|
} // namespace
|
|
|
|
static auto ComputeSegments(llvm::ArrayRef<MismatchedBracketToken> tokens)
|
|
-> Segments {
|
|
auto num_tokens = static_cast<int32_t>(tokens.size());
|
|
Segments segments = {.id = llvm::SmallVector<int32_t>(num_tokens, 0),
|
|
.first = llvm::SmallVector<int32_t>(num_tokens, 0)};
|
|
for (int32_t i : llvm::seq(1, num_tokens)) {
|
|
auto prev_kind = tokens[i - 1].kind;
|
|
bool new_seg = prev_kind == Kind::Semi ||
|
|
prev_kind == Kind::OpenCurlyBrace ||
|
|
prev_kind == Kind::CloseCurlyBrace;
|
|
segments.id[i] = segments.id[i - 1] + (new_seg ? 1 : 0);
|
|
segments.first[i] = new_seg ? i : segments.first[i - 1];
|
|
}
|
|
return segments;
|
|
}
|
|
|
|
static auto FindUnmatchedGroupBrackets(
|
|
llvm::ArrayRef<MismatchedBracketToken> tokens,
|
|
llvm::ArrayRef<int32_t> match_partner) -> UnmatchedGroupBrackets {
|
|
UnmatchedGroupBrackets unmatched;
|
|
for (auto [i, token] : llvm::enumerate(tokens)) {
|
|
if (match_partner[i] != -1) {
|
|
continue;
|
|
}
|
|
auto index = static_cast<int32_t>(i);
|
|
switch (token.kind) {
|
|
case Kind::OpenParen:
|
|
unmatched.open_parens.push_back(index);
|
|
break;
|
|
case Kind::OpenSquareBracket:
|
|
unmatched.open_squares.push_back(index);
|
|
break;
|
|
case Kind::CloseParen:
|
|
unmatched.close_parens.push_back(index);
|
|
break;
|
|
case Kind::CloseSquareBracket:
|
|
unmatched.close_squares.push_back(index);
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
return unmatched;
|
|
}
|
|
|
|
static auto AnalyzeTokens(llvm::ArrayRef<MismatchedBracketToken> tokens)
|
|
-> TokenAnalysis {
|
|
auto num_tokens = static_cast<int32_t>(tokens.size());
|
|
auto match_partner = MatchBracketPairs(tokens);
|
|
auto segments = ComputeSegments(tokens);
|
|
auto unmatched = FindUnmatchedGroupBrackets(tokens, match_partner);
|
|
|
|
llvm::SmallVector<int32_t> effective_header_indent(num_tokens, 0);
|
|
llvm::BitVector is_first_on_line(num_tokens);
|
|
llvm::BitVector follows_statement_header(num_tokens);
|
|
llvm::BitVector header_has_open_curly_brace(num_tokens);
|
|
for (int32_t i : llvm::seq(0, num_tokens)) {
|
|
is_first_on_line[i] = (i == 0 || tokens[i].line != tokens[i - 1].line);
|
|
effective_header_indent[i] =
|
|
ComputeAssociatedLineIndent(tokens, match_partner, i);
|
|
follows_statement_header[i] =
|
|
ComputeFollowsStatementHeader(tokens, match_partner, i);
|
|
if (follows_statement_header[i]) {
|
|
header_has_open_curly_brace[i] =
|
|
ComputeHeaderHasOpenCurlyBrace(tokens, match_partner, i);
|
|
}
|
|
}
|
|
|
|
return TokenAnalysis{
|
|
.match_partner = std::move(match_partner),
|
|
.segments = std::move(segments),
|
|
.unmatched = std::move(unmatched),
|
|
.effective_header_indent = std::move(effective_header_indent),
|
|
.is_first_on_line = std::move(is_first_on_line),
|
|
.follows_statement_header = std::move(follows_statement_header),
|
|
.header_has_open_curly_brace = std::move(header_has_open_curly_brace),
|
|
};
|
|
}
|
|
|
|
// Whether the sorted list `list` contains an element in [lo, hi].
|
|
static auto ContainsInRange(llvm::ArrayRef<int32_t> list, int32_t lo,
|
|
int32_t hi) -> bool {
|
|
const auto* it = std::lower_bound(list.begin(), list.end(), lo);
|
|
return it != list.end() && *it <= hi;
|
|
}
|
|
|
|
// Whether a matched `{`...`}` pair is trustworthy, judging by how the `}` lines
|
|
// up and by the separators directly inside the pair.
|
|
static auto BracePairIsClean(llvm::ArrayRef<MismatchedBracketToken> tokens,
|
|
const TokenAnalysis& analysis, int32_t open_idx,
|
|
int32_t close_idx) -> bool {
|
|
const auto& open = tokens[open_idx];
|
|
int32_t header_indent = analysis.effective_header_indent[open_idx];
|
|
if (open.line != tokens[close_idx].line) {
|
|
if (!open.is_struct_brace &&
|
|
(!open.is_at_end_of_line ||
|
|
header_indent != tokens[close_idx].line_indent)) {
|
|
return false;
|
|
}
|
|
if (open.is_struct_brace && tokens[close_idx].line_indent < header_indent) {
|
|
return false;
|
|
}
|
|
}
|
|
// A `;` directly inside a struct brace, or a `,` directly inside a scope
|
|
// brace, is illegal; the brace pairing has likely captured too much.
|
|
auto bad_kind = open.is_struct_brace ? Kind::Semi : Kind::Comma;
|
|
int32_t depth = 0;
|
|
for (int32_t j : llvm::seq(open_idx + 1, close_idx)) {
|
|
if (IsOpeningBracket(tokens[j].kind)) {
|
|
++depth;
|
|
} else if (IsClosingBracket(tokens[j].kind)) {
|
|
--depth;
|
|
} else if (tokens[j].kind == bad_kind && depth == 0) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// Whether a matched `(`...`)` or `[`...`]` pair is trustworthy. An unmatched
|
|
// opener of the same kind earlier in the same statement segment could really
|
|
// own our closer, and an unmatched closer of the same kind later in the same
|
|
// segment could really own our opener; both make the pairing suspect.
|
|
static auto GroupPairIsClean(llvm::ArrayRef<MismatchedBracketToken> tokens,
|
|
const TokenAnalysis& analysis, int32_t open_idx,
|
|
int32_t close_idx) -> bool {
|
|
auto kind = tokens[open_idx].kind;
|
|
if (ContainsInRange(analysis.unmatched.OpenersOfKind(kind),
|
|
analysis.segments.first[open_idx], open_idx - 1)) {
|
|
return false;
|
|
}
|
|
// A scan for an unmatched closer would be bounded by the end of the closer's
|
|
// segment, so it's enough to check whether the first one after `close_idx` is
|
|
// still in that segment.
|
|
llvm::ArrayRef<int32_t> closers = analysis.unmatched.ClosersOfKind(kind);
|
|
const auto* it = std::upper_bound(closers.begin(), closers.end(), close_idx);
|
|
return it == closers.end() ||
|
|
analysis.segments.id[*it] != analysis.segments.id[close_idx];
|
|
}
|
|
|
|
// Whether everything between a matched pair is itself safe to collapse: every
|
|
// bracket inside pairs up within the pair and is clean, and for a scope brace,
|
|
// no line inside is dedented to or past the header.
|
|
static auto PairInteriorIsClean(llvm::ArrayRef<MismatchedBracketToken> tokens,
|
|
const TokenAnalysis& analysis,
|
|
const llvm::BitVector& is_clean_range,
|
|
int32_t open_idx, int32_t close_idx) -> bool {
|
|
const auto& open = tokens[open_idx];
|
|
for (int32_t j : llvm::seq(open_idx + 1, close_idx)) {
|
|
int32_t partner = analysis.match_partner[j];
|
|
if (partner == -1) {
|
|
// An unmatched bracket inside means the pair can't be trusted.
|
|
if (IsOpeningBracket(tokens[j].kind) ||
|
|
IsClosingBracket(tokens[j].kind)) {
|
|
return false;
|
|
}
|
|
} else if (partner < open_idx || partner > close_idx) {
|
|
return false;
|
|
}
|
|
if (IsOpeningBracket(tokens[j].kind) && !is_clean_range[j]) {
|
|
return false;
|
|
}
|
|
if (open.kind == Kind::OpenCurlyBrace && !open.is_struct_brace &&
|
|
analysis.is_first_on_line[j] &&
|
|
tokens[j].line != tokens[close_idx].line &&
|
|
tokens[j].line_indent <= analysis.effective_header_indent[open_idx]) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// Marks the matched pairs that can be trusted, so that the search can collapse
|
|
// each into a single item instead of reconsidering brackets that clearly pair
|
|
// up. Pairs are visited in reverse order, so an inner pair is decided before
|
|
// the pairs enclosing it.
|
|
//
|
|
// Note: an illegal leaf adjacency inside a *matched* pair is treated as invalid
|
|
// user code, not a bracket error; the pair is still trusted and collapsed. The
|
|
// adjacency cue only guides where to insert brackets in regions that are
|
|
// already unbalanced.
|
|
static auto MarkCleanRanges(llvm::ArrayRef<MismatchedBracketToken> tokens,
|
|
const TokenAnalysis& analysis) -> llvm::BitVector {
|
|
auto num_tokens = static_cast<int32_t>(tokens.size());
|
|
llvm::BitVector is_clean_range(num_tokens);
|
|
for (int32_t i : llvm::reverse(llvm::seq(0, num_tokens))) {
|
|
// Only consider a pair from its opener; this also skips the unmatched
|
|
// tokens, whose partner is -1.
|
|
int32_t close_idx = analysis.match_partner[i];
|
|
if (close_idx <= i) {
|
|
continue;
|
|
}
|
|
bool clean = tokens[i].kind == Kind::OpenCurlyBrace
|
|
? BracePairIsClean(tokens, analysis, i, close_idx)
|
|
: GroupPairIsClean(tokens, analysis, i, close_idx);
|
|
is_clean_range[i] =
|
|
clean &&
|
|
PairInteriorIsClean(tokens, analysis, is_clean_range, i, close_idx);
|
|
}
|
|
return is_clean_range;
|
|
}
|
|
|
|
// Builds the item sequence the search runs over: one item per token, except
|
|
// that each clean matched pair collapses into a single item spanning it.
|
|
static auto BuildItems(llvm::ArrayRef<MismatchedBracketToken> tokens,
|
|
const TokenAnalysis& analysis,
|
|
const llvm::BitVector& is_clean_range)
|
|
-> llvm::SmallVector<Item> {
|
|
auto num_tokens = static_cast<int32_t>(tokens.size());
|
|
llvm::SmallVector<Item> items;
|
|
auto make_item = [&](int32_t start, int32_t end, bool collapsed,
|
|
bool has_scope) {
|
|
// Cues the surrounding analysis has already determined; the rest follow
|
|
// from the tokens themselves.
|
|
CueSet context_cues =
|
|
CueIf(collapsed, Cue::CollapsedBlock) |
|
|
CueIf(has_scope, Cue::ContainsScopeBrace) |
|
|
CueIf(analysis.is_first_on_line[start], Cue::FirstOnLine) |
|
|
CueIf(analysis.follows_statement_header[start],
|
|
Cue::FollowsStatementHeader) |
|
|
CueIf(analysis.header_has_open_curly_brace[start],
|
|
Cue::HeaderHasOpenCurly);
|
|
items.push_back(Item{
|
|
.token_start_index = start,
|
|
.token_end_index = end,
|
|
.token = tokens[start],
|
|
.effective_header_indent = analysis.effective_header_indent[start],
|
|
.cues = ComputeItemCues(
|
|
tokens[start], start > 0 ? &tokens[start - 1] : nullptr,
|
|
items.empty() ? nullptr : &items.back(), context_cues),
|
|
});
|
|
};
|
|
for (int32_t i = 0; i < num_tokens;) {
|
|
if (!is_clean_range[i]) {
|
|
make_item(i, i, /*collapsed=*/false,
|
|
tokens[i].kind == Kind::OpenCurlyBrace ||
|
|
tokens[i].kind == Kind::CloseCurlyBrace);
|
|
++i;
|
|
continue;
|
|
}
|
|
int32_t close_idx = analysis.match_partner[i];
|
|
bool has_scope = llvm::any_of(llvm::seq(i, close_idx + 1), [&](int32_t j) {
|
|
return tokens[j].kind == Kind::OpenCurlyBrace &&
|
|
!tokens[j].is_struct_brace;
|
|
});
|
|
make_item(i, close_idx, /*collapsed=*/true, has_scope);
|
|
i = close_idx + 1;
|
|
}
|
|
return items;
|
|
}
|
|
|
|
// Finds the item indexes where each independently-solved region starts,
|
|
// beginning with 0 and ending with the number of items, so that consecutive
|
|
// boundaries delimit the regions. A region ends at a top-level declaration
|
|
// boundary: a statement introducer at zero indentation whose predecessor ended
|
|
// a statement. This bounds how far a single mistake can smear, and gives
|
|
// unclosed brackets a natural place to be closed (the region end).
|
|
static auto FindRegionBoundaries(llvm::ArrayRef<MismatchedBracketToken> tokens,
|
|
llvm::ArrayRef<Item> items)
|
|
-> llvm::SmallVector<int32_t> {
|
|
llvm::SmallVector<int32_t> boundaries = {0};
|
|
for (auto [item_index, item] : llvm::enumerate(items)) {
|
|
auto i = static_cast<int32_t>(item_index);
|
|
// Note: line_indent is a 1-based column number, so top-level tokens have
|
|
// line_indent 1.
|
|
if (i == 0 || item.HasAll(Cue::CollapsedBlock) ||
|
|
item.token.kind != Kind::StatementIntroducer ||
|
|
item.token.is_else_keyword || item.token.line_indent > 1 ||
|
|
!item.HasAll(Cue::FirstOnLine)) {
|
|
continue;
|
|
}
|
|
auto prev_end_kind = tokens[items[i - 1].token_end_index].kind;
|
|
if ((prev_end_kind == Kind::Semi ||
|
|
prev_end_kind == Kind::CloseCurlyBrace) &&
|
|
boundaries.back() != i) {
|
|
boundaries.push_back(i);
|
|
}
|
|
}
|
|
if (boundaries.back() != static_cast<int32_t>(items.size())) {
|
|
boundaries.push_back(static_cast<int32_t>(items.size()));
|
|
}
|
|
return boundaries;
|
|
}
|
|
|
|
// Whether a region's loose (non-collapsed) brackets already form a balanced,
|
|
// well-nested sequence. Such a region needs no solving: it has no unmatched
|
|
// bracket, so the search would simply match everything and emit no corrections.
|
|
// Skipping it avoids running the beam search over the many regions whose
|
|
// matched pairs merely weren't collapsed.
|
|
static auto RegionIsBalanced(llvm::ArrayRef<Item> region) -> bool {
|
|
llvm::SmallVector<Kind> open_kinds;
|
|
for (const Item& item : region) {
|
|
if (item.HasAll(Cue::CollapsedBlock)) {
|
|
continue;
|
|
}
|
|
auto kind = item.token.kind;
|
|
if (IsOpeningBracket(kind)) {
|
|
open_kinds.push_back(kind);
|
|
} else if (IsClosingBracket(kind)) {
|
|
if (open_kinds.empty() ||
|
|
MatchingClosingKind(open_kinds.back()) != kind) {
|
|
return false;
|
|
}
|
|
open_kinds.pop_back();
|
|
}
|
|
}
|
|
return open_kinds.empty();
|
|
}
|
|
|
|
auto FixMismatchedBrackets(llvm::ArrayRef<MismatchedBracketToken> tokens)
|
|
-> llvm::SmallVector<BracketCorrection> {
|
|
llvm::SmallVector<BracketCorrection> corrections;
|
|
if (tokens.empty()) {
|
|
return corrections;
|
|
}
|
|
|
|
TokenAnalysis analysis = AnalyzeTokens(tokens);
|
|
llvm::BitVector is_clean_range = MarkCleanRanges(tokens, analysis);
|
|
llvm::SmallVector<Item> items = BuildItems(tokens, analysis, is_clean_range);
|
|
llvm::SmallVector<int32_t> region_boundaries =
|
|
FindRegionBoundaries(tokens, items);
|
|
|
|
// Each region runs between consecutive boundaries.
|
|
for (auto [start, end] :
|
|
llvm::zip(region_boundaries, llvm::drop_begin(region_boundaries))) {
|
|
if (start >= end) {
|
|
continue;
|
|
}
|
|
auto region = llvm::ArrayRef<Item>(items).slice(start, end - start);
|
|
if (RegionIsBalanced(region)) {
|
|
continue;
|
|
}
|
|
// Any bracket still open at the end of the region is closed before the
|
|
// token the next region starts at, or before the final `FileEnd`.
|
|
TokenIndex region_end_token = end < static_cast<int32_t>(items.size())
|
|
? items[end].token.token_index
|
|
: tokens.back().token_index;
|
|
SolveRegionCostBased(region, region_end_token, corrections);
|
|
}
|
|
|
|
llvm::stable_sort(
|
|
corrections, [](const BracketCorrection& a, const BracketCorrection& b) {
|
|
if (a.diagnostic_token_index != b.diagnostic_token_index) {
|
|
return a.diagnostic_token_index < b.diagnostic_token_index;
|
|
}
|
|
return a.fix_token_index < b.fix_token_index;
|
|
});
|
|
|
|
return corrections;
|
|
}
|
|
|
|
} // namespace Carbon::Lex
|