Improve lexer and parser diagnostic locations (#7990)

Lexer diagnostics used a single `const char*` as their source location,
which only allowed pointing at a single character rather than a run of
characters. This limited the underlining of unrecognized tokens, and
also made escape sequence diagnostics particularly strange by stopping
at whatever point the lexer discovered the error and pointing only at
that location. For example: `"\w"` was reported at the `w`, and `"\x"`
at the closing `"`.

This change tries to always keep the primary location of a diagnostic
the point at which typing (inserting or deleting) might fix the error.
So for `"\x"`, `\x` is underlined, but the position is at the `"` as
that's where you would type a hex code.

The lexer's emitter now takes a special lexing source location that
represents either a single position _between_ two characters, a single
character, or a range of the source buffer. The lexer diagnostics are
updated to construct these for the character range as appropriate.

Parser diagnostics for a missing token were located at the _next_ token,
which is particularly confusing for terminators like missing `;`s. Now
they are located at a position after the previous token, with or without
a space depending on the token, but typically on the same line.
Diagnostics still sort by how far the parser has read.

The parser diagnostics for unclosed groups also now attaches a label to
the opening token when it is on a separate line. And the diagnostics for
mismatched indent in a multi-line string attaches a label to the closing
`'''` that establishes that indent.

There are likely more location improvements we should do here, but this
seemed like a reasonable incremental improvement and is what Claude
found when doing an initial audit. One example where I think we should
make further improvements is around suggesting raw identifiers when we
find an unexpected `Cpp` or `Core` or `base`. We do this consistently
now, but raw identifiers should be rare. A TODO is added for these
cases.

Assisted-by: Claude Code

---------

Co-authored-by: Richard Smith <richard@metafoo.co.uk>
This commit is contained in:
Chandler CarruthandRichard Smith authored and GitHub committed 2026-11-01 06:22:00 +00:00
1 parent 611e6a210e
commit 68569d234d
173 files changed
+1782 -689

No files matched your search

+42 -14
View File
@@ -55,11 +55,20 @@ auto Context::VlogPopState(llvm::StringLiteral label) const -> void {
state_stack_.back());
}
auto Context::OpeningBracketInsertionPoint() const -> DiagnosticLoc {
auto previous_token = *(position_ - 1);
if (tokens().GetEndLoc(previous_token).first ==
tokens().GetLine(*position_)) {
return DiagnosticLoc::Before(*position_);
}
return DiagnosticLoc::After(previous_token);
}
auto Context::DiagnoseExpectedOpenCurlyBrace(Lex::TokenIndex default_token,
NodeKind start_kind) -> void {
CARBON_DIAGNOSTIC(ExpectedCurlyBraceAfter, Error, "expected `{` after `{0}`",
Lex::TokenKind);
emitter_.Emit(*position_, ExpectedCurlyBraceAfter,
emitter_.Emit(OpeningBracketInsertionPoint(), ExpectedCurlyBraceAfter,
tokens().GetKind(default_token));
AddLeafNode(start_kind, default_token, /*has_error=*/true);
}
@@ -68,7 +77,7 @@ auto Context::DiagnoseExpectedOpenParen(Lex::TokenIndex default_token,
NodeKind start_kind) -> void {
CARBON_DIAGNOSTIC(ExpectedParenAfter, Error, "expected `(` after `{0}`",
Lex::TokenKind);
emitter_.Emit(*position_, ExpectedParenAfter,
emitter_.Emit(OpeningBracketInsertionPoint(), ExpectedParenAfter,
tokens().GetKind(default_token));
AddLeafNode(start_kind, default_token, /*has_error=*/true);
}
@@ -81,14 +90,20 @@ auto Context::DiagnoseAndSkipToCloseSymbol(State state, NodeKind close_kind)
return;
}
// TODO: Include the location of the matching opening delimiter in the
// diagnostic.
CARBON_DIAGNOSTIC(ExpectedCloseSymbol, Error,
"unexpected tokens before `{0}`", Lex::TokenKind);
emitter_.Emit(*position_, ExpectedCloseSymbol,
open_token_kind.closing_symbol());
CARBON_DIAGNOSTIC_LABEL(GroupOpenedHere, Info, "matching this `{0}`",
Lex::TokenKind);
auto matched_close = tokens().GetMatchedClosingToken(state.token);
emitter_
.Build(DiagnosticLoc::Tokens(
{.begin = *position_,
.end = Lex::TokenIndex(matched_close.index - 1)}),
ExpectedCloseSymbol, open_token_kind.closing_symbol())
.Attach(state.token, GroupOpenedHere, open_token_kind)
.Emit();
SkipTo(tokens().GetMatchedClosingToken(state.token));
SkipTo(matched_close);
AddNode(close_kind, Consume(), /*has_error=*/true);
}
@@ -184,13 +199,18 @@ auto Context::DiagnoseInvalidOperatorFixity(OperatorFixity fixity) -> void {
"whitespace missing {0:=-1:before|=0:around|=1:after} "
"binary operator",
Diagnostics::IntAsSelect);
// Whitespace missing on one side goes at that side. Whitespace missing
// around the operator is about the operator.
Diagnostics::IntAsSelect pos(0);
DiagnosticLoc loc = *position_;
if (tokens().HasLeadingWhitespace(*position_)) {
pos.value = 1;
loc = DiagnosticLoc::After(*position_);
} else if (tokens().HasTrailingWhitespace(*position_)) {
pos.value = -1;
loc = DiagnosticLoc::Before(*position_);
}
emitter_.Emit(*position_, BinaryOperatorRequiresWhitespace, pos);
emitter_.Emit(loc, BinaryOperatorRequiresWhitespace, pos);
} else {
bool prefix = fixity == OperatorFixity::Prefix;
@@ -209,7 +229,9 @@ auto Context::DiagnoseInvalidOperatorFixity(OperatorFixity fixity) -> void {
UnaryOperatorRequiresWhitespace, Error,
"whitespace is required {0:before|after} this unary operator",
Diagnostics::BoolAsSelect);
emitter_.Emit(*position_, UnaryOperatorRequiresWhitespace, prefix);
emitter_.Emit(prefix ? DiagnosticLoc::Before(*position_)
: DiagnosticLoc::After(*position_),
UnaryOperatorRequiresWhitespace, prefix);
}
}
}
@@ -218,9 +240,13 @@ auto Context::RecoverFromInvalidListToken(Lex::TokenKind close_kind,
bool already_has_error) -> void {
// Don't error a second time on the same element.
if (!already_has_error) {
// TODO: Also label the bracket that opened the list, to show which list
// this is. That needs the opening token, which callers hold in
// `state.token` but don't pass down.
CARBON_DIAGNOSTIC(UnexpectedTokenAfterListElement, Error,
"expected `,` or `{0}`", Lex::TokenKind);
emitter_.Emit(*position_, UnexpectedTokenAfterListElement, close_kind);
emitter_.Emit(PointAfterPreviousToken(), UnexpectedTokenAfterListElement,
close_kind);
ReturnErrorOnState();
}
@@ -279,8 +305,9 @@ auto Context::ParseLibraryName(bool accept_default)
"expected `default` or a string literal to specify the library name");
CARBON_DIAGNOSTIC(ExpectedLibraryName, Error,
"expected a string literal to specify the library name");
emitter().Emit(*position(), accept_default ? ExpectedLibraryNameOrDefault
: ExpectedLibraryName);
emitter().Emit(PointAfterPreviousToken(), accept_default
? ExpectedLibraryNameOrDefault
: ExpectedLibraryName);
return std::nullopt;
}
@@ -298,7 +325,7 @@ auto Context::ParseLibrarySpecifier(bool accept_default)
auto Context::DiagnoseExpectedDeclSemi(Lex::TokenKind expected_kind) -> void {
CARBON_DIAGNOSTIC(ExpectedDeclSemi, Error,
"`{0}` declarations must end with a `;`", Lex::TokenKind);
emitter().Emit(*position(), ExpectedDeclSemi, expected_kind);
emitter().Emit(PointAfterPreviousToken(), ExpectedDeclSemi, expected_kind);
}
auto Context::DiagnoseExpectedDeclSemiOrDefinition(Lex::TokenKind expected_kind)
@@ -307,7 +334,8 @@ auto Context::DiagnoseExpectedDeclSemiOrDefinition(Lex::TokenKind expected_kind)
"`{0}` declarations must either end with a `;` or "
"have a `{{ ... }` block for a definition",
Lex::TokenKind);
emitter().Emit(*position(), ExpectedDeclSemiOrDefinition, expected_kind);
emitter().Emit(PointAfterPreviousToken(), ExpectedDeclSemiOrDefinition,
expected_kind);
}
// Returns whether we are currently parsing in a scope in which function