mirror of
https://github.com/carbon-language/carbon-lang.git
synced 2026-10-05 22:02:55 +01:00
Richard Smith
3bbc03f527
Adds an algorithm to compute where to insert brackets to repair bracketing mismatches during lexing. This takes indentation, as well as a number of other cues, into account to predict where the brackets should have gone. Detects when there is ambiguity between solutions and makes no suggestion in that case. Reduces the problem by splitting on properly bracketed top-level constructs, then uses a beam search to find good candidate solutions quickly. This includes both a fuzzer and an eval tool that can be used to determine how well the algorithm fares against a given corpus of valid Carbon code, by damaging it in various ways and seeing whether the algorithm can correctly fix it. On all the eval modes, this algorithm can correctly infer the positions for over 80% of lost brackets (and can correctly restore 95+% of brackets in some modes), with low rates of incorrect suggestions. See added documentation for full details. Assisted-by: Gemini via Antigravity, Claude via Claude Code
50 lines
1.8 KiB
C++
50 lines
1.8 KiB
C++
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
|
|
// Exceptions. See /LICENSE for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
|
|
#ifndef CARBON_TOOLCHAIN_LEX_LEX_H_
|
|
#define CARBON_TOOLCHAIN_LEX_LEX_H_
|
|
|
|
#include "toolchain/base/shared_value_stores.h"
|
|
#include "toolchain/diagnostics/emitter.h"
|
|
#include "toolchain/lex/mismatched_brackets.h"
|
|
#include "toolchain/lex/tokenized_buffer.h"
|
|
#include "toolchain/source/source_buffer.h"
|
|
|
|
namespace Carbon::Lex {
|
|
|
|
struct LexOptions {
|
|
// Options must be set individually, not through initialization.
|
|
explicit LexOptions() = default;
|
|
|
|
// If set, a consumer for diagnostics. Otherwise, diagnostics go to stderr.
|
|
Diagnostics::Consumer* consumer = nullptr;
|
|
|
|
// If set, enables verbose output.
|
|
llvm::raw_ostream* vlog_stream = nullptr;
|
|
|
|
// If set, tokens will be dumped to this.
|
|
llvm::raw_ostream* dump_stream = nullptr;
|
|
|
|
// When dumping, whether to omit `FileStart` and `FileEnd` in output.
|
|
bool omit_file_boundary_tokens = false;
|
|
|
|
// If set, points to a caller-owned vector that `Lex` overwrites with the
|
|
// bracket corrections recovery made; it need only outlive the `Lex` call.
|
|
// Their token indexes refer to the returned buffer, so a correction that
|
|
// inserted a bracket names the inserted token itself.
|
|
llvm::SmallVector<BracketCorrection>* bracket_corrections = nullptr;
|
|
};
|
|
|
|
// Lexes a buffer of source code into a tokenized buffer.
|
|
//
|
|
// The provided source buffer must outlive any returned `TokenizedBuffer`
|
|
// which will refer into the source.
|
|
auto Lex(SharedValueStores& value_stores,
|
|
SourceBuffer& source [[clang::lifetimebound]], LexOptions options)
|
|
-> TokenizedBuffer;
|
|
|
|
} // namespace Carbon::Lex
|
|
|
|
#endif // CARBON_TOOLCHAIN_LEX_LEX_H_
|