Files
carbon-lang/toolchain/parse/handle_pattern_list.cpp
T
Chandler CarruthandRichard Smith 68569d234d Improve lexer and parser diagnostic locations (#7990)
Lexer diagnostics used a single `const char*` as their source location,
which only allowed pointing at a single character rather than a run of
characters. This limited the underlining of unrecognized tokens, and
also made escape sequence diagnostics particularly strange by stopping
at whatever point the lexer discovered the error and pointing only at
that location. For example: `"\w"` was reported at the `w`, and `"\x"`
at the closing `"`.

This change tries to always keep the primary location of a diagnostic
the point at which typing (inserting or deleting) might fix the error.
So for `"\x"`, `\x` is underlined, but the position is at the `"` as
that's where you would type a hex code.

The lexer's emitter now takes a special lexing source location that
represents either a single position _between_ two characters, a single
character, or a range of the source buffer. The lexer diagnostics are
updated to construct these for the character range as appropriate.

Parser diagnostics for a missing token were located at the _next_ token,
which is particularly confusing for terminators like missing `;`s. Now
they are located at a position after the previous token, with or without
a space depending on the token, but typically on the same line.
Diagnostics still sort by how far the parser has read.

The parser diagnostics for unclosed groups also now attaches a label to
the opening token when it is on a separate line. And the diagnostics for
mismatched indent in a multi-line string attaches a label to the closing
`'''` that establishes that indent.

There are likely more location improvements we should do here, but this
seemed like a reasonable incremental improvement and is what Claude
found when doing an initial audit. One example where I think we should
make further improvements is around suggesting raw identifiers when we
find an unexpected `Cpp` or `Core` or `base`. We do this consistently
now, but raw identifiers should be rare. A TODO is added for these
cases.

Assisted-by: Claude Code

---------

Co-authored-by: Richard Smith <richard@metafoo.co.uk>
2026-11-01 06:22:00 +00:00

374 lines
15 KiB
C++

// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
// Exceptions. See /LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#include "toolchain/parse/context.h"
#include "toolchain/parse/handle.h"
namespace Carbon::Parse {
// Handles PatternListElementAs(Tuple|Explicit|Implicit).
static auto HandlePatternListElement(Context& context, StateKind pattern_state,
StateKind finish_state_kind) -> void {
auto state = context.PopState();
context.PushStateForPattern(finish_state_kind, state.in_var_pattern,
state.in_unused_pattern,
state.in_field_shorthand_pattern,
state.binding_context, state.ambient_precedence);
// For patterns inside of list elements we normally check for a default value.
// Parsing a struct designator is special case as that also is looking for a
// required `=` sign, so we handle looking for an optional default value in
// that state.
if (pattern_state != StateKind::StructPatternFieldAfterDesignator) {
context.PushStateForPattern(
StateKind::PatternListElementCheckForDefaultValue, state.in_var_pattern,
state.in_unused_pattern, state.in_field_shorthand_pattern,
state.binding_context, state.ambient_precedence);
}
context.PushStateForPattern(pattern_state, state.in_var_pattern,
state.in_unused_pattern,
state.in_field_shorthand_pattern,
state.binding_context, state.ambient_precedence);
}
auto HandlePatternListElementAsTuple(Context& context) -> void {
HandlePatternListElement(context, StateKind::Pattern,
StateKind::PatternListElementFinishAsTuple);
}
auto HandlePatternListElementAsStruct(Context& context) -> void {
switch (context.PositionKind()) {
case Lex::TokenKind::Period:
HandlePatternListElement(context,
StateKind::StructPatternFieldAfterDesignator,
StateKind::PatternListElementFinishAsStruct);
context.PushState(StateKind::PeriodAsStruct);
break;
case Lex::TokenKind::Underscore:
// If this ends up as part of a binding pattern, it will look like a
// shorthand pattern such as `{_: i32}`.
// Set flag on state which will be popped in HandlePatternListElement.
context.state_stack().back().in_field_shorthand_pattern = true;
HandlePatternListElement(context, StateKind::StructPatternUnderscore,
StateKind::PatternListElementFinishAsStruct);
break;
default:
// Set flag on state which will be popped in HandlePatternListElement.
context.state_stack().back().in_field_shorthand_pattern = true;
HandlePatternListElement(context, StateKind::Pattern,
StateKind::PatternListElementFinishAsStruct);
break;
}
}
auto HandlePatternListElementAsExplicit(Context& context) -> void {
HandlePatternListElement(context, StateKind::Pattern,
StateKind::PatternListElementFinishAsExplicit);
}
auto HandlePatternListElementAsImplicit(Context& context) -> void {
HandlePatternListElement(context, StateKind::Pattern,
StateKind::PatternListElementFinishAsImplicit);
}
auto HandleStructPatternDesignatedFieldFinish(Context& context) -> void {
auto state = context.PopState();
context.AddNode(NodeKind::StructPatternDesignatedField, state.token,
state.has_error);
}
auto HandleStructPatternFieldAfterDesignator(Context& context) -> void {
auto state = context.PopState();
auto skip_to_recovery_position = [&](bool add_invalid_parse) {
auto recovery_pos =
context.FindNextOf({Lex::TokenKind::Equal, Lex::TokenKind::Comma,
Lex::TokenKind::CloseCurlyBrace});
if (add_invalid_parse) {
if (context.tokens().GetKind(*recovery_pos) != Lex::TokenKind::Equal) {
context.AddInvalidParse(*context.position());
}
}
context.SkipTo(*recovery_pos);
};
if (state.has_error) {
// recover from error returned when parsing designator
skip_to_recovery_position(/*add_invalid_parse=*/false);
}
if (!context.PositionIs(Lex::TokenKind::Equal)) {
if (!state.has_error) {
state.has_error = true;
CARBON_DIAGNOSTIC(ExpectedStructPatternDesignatedField, Error,
"expected `= value` after `.field`");
context.emitter().Emit(context.PointAfterPreviousToken(),
ExpectedStructPatternDesignatedField);
}
skip_to_recovery_position(/*add_invalid_parse=*/true);
if (context.PositionIs(Lex::TokenKind::Comma) ||
context.PositionIs(Lex::TokenKind::CloseCurlyBrace)) {
context.PushState(state, StateKind::StructPatternDesignatedFieldFinish);
return;
}
}
state.token = context.ConsumeChecked(Lex::TokenKind::Equal);
context.PushState(state, StateKind::StructPatternDesignatedFieldFinish);
context.PushStateForPattern(StateKind::Pattern, state.in_var_pattern,
state.in_unused_pattern,
state.in_field_shorthand_pattern,
state.binding_context, state.ambient_precedence);
}
auto HandleStructPatternUnderscore(Context& context) -> void {
auto state = context.PopState();
if (context.PositionKind(Lookahead::NextToken)
.is_binding_pattern_operator()) {
context.PushStateForPattern(
StateKind::BindingPattern, state.in_var_pattern,
state.in_unused_pattern, state.in_field_shorthand_pattern,
state.binding_context, state.ambient_precedence);
return;
}
auto underscore = context.ConsumeChecked(Lex::TokenKind::Underscore);
bool is_last = context.PositionIs(Lex::TokenKind::CloseCurlyBrace);
if (!is_last) {
CARBON_DIAGNOSTIC(
ExpectedCloseAfterUnderscore, Error,
"unexpected token `{0}` after `_` in struct pattern, expected `}`",
Lex::TokenKind);
context.emitter().Emit(*context.position(), ExpectedCloseAfterUnderscore,
context.PositionKind());
state.has_error = true;
}
context.AddNode(NodeKind::UnderscoreName, underscore, state.has_error);
if (state.has_error) {
context.ReturnErrorOnState();
}
}
auto HandlePatternListElementCheckForDefaultValue(Context& context) -> void {
auto state = context.PopState();
if (state.has_error) {
context.ReturnErrorOnState();
}
// Check for the optional `=` indicating a default value.
auto equals_token = context.ConsumeIf(Lex::TokenKind::Equal);
if (!equals_token) {
return;
}
// Add the first virtual node surrounding the value expr node, to facilitate
// handling in check.
context.AddLeafNode(NodeKind::DefaultValueExprStart, *equals_token);
state.token = *equals_token;
state.kind = StateKind::PatternListElementFinishDefaultValue;
context.PushState(state);
// Check for the underscore `_` indicating the default value is unspecified.
auto underscore = context.ConsumeIf(Lex::TokenKind::Underscore);
if (underscore) {
context.AddLeafNode(NodeKind::DefaultValueUnspecified, *underscore);
return;
}
context.PushStateForExpr(state.ambient_precedence);
}
auto HandlePatternListElementFinishDefaultValue(Context& context) -> void {
auto state = context.PopState();
// Propagate any expr parsing errors upwards to initiate error recovery
// at the pattern list level.
if (state.has_error) {
context.ReturnErrorOnState();
}
context.AddNode(NodeKind::DefaultValuePattern, state.token, state.has_error);
}
// Handles PatternListElementFinishAs(Tuple|Struct|Explicit|Implicit).
static auto HandlePatternListElementFinish(Context& context,
Lex::TokenKind close_token,
StateKind param_state_kind) -> void {
auto state = context.PopState();
if (state.has_error) {
context.ReturnErrorOnState();
}
auto list_token_kind = context.ConsumeListToken(NodeKind::PatternListComma,
close_token, state.has_error);
// If we have a comma, the parent is now a tuple pattern not a parenthesized
// pattern.
if (list_token_kind != Context::ListTokenKind::Close &&
param_state_kind == StateKind::PatternListElementAsTuple) {
auto parent_state = context.PopState();
CARBON_CHECK(parent_state.kind == StateKind::PatternListFinishAsTuple ||
parent_state.kind == StateKind::PatternListFinishAsParen);
context.PushState(parent_state, StateKind::PatternListFinishAsTuple);
}
if (list_token_kind == Context::ListTokenKind::Comma) {
context.PushStateForPattern(
param_state_kind, state.in_var_pattern, state.in_unused_pattern,
/*in_field_shorthand_pattern=*/false, state.binding_context,
state.ambient_precedence);
}
}
auto HandlePatternListElementFinishAsTuple(Context& context) -> void {
HandlePatternListElementFinish(context, Lex::TokenKind::CloseParen,
StateKind::PatternListElementAsTuple);
}
auto HandlePatternListElementFinishAsStruct(Context& context) -> void {
HandlePatternListElementFinish(context, Lex::TokenKind::CloseCurlyBrace,
StateKind::PatternListElementAsStruct);
}
auto HandlePatternListElementFinishAsExplicit(Context& context) -> void {
HandlePatternListElementFinish(context, Lex::TokenKind::CloseParen,
StateKind::PatternListElementAsExplicit);
}
auto HandlePatternListElementFinishAsImplicit(Context& context) -> void {
HandlePatternListElementFinish(context, Lex::TokenKind::CloseSquareBracket,
StateKind::PatternListElementAsImplicit);
}
// Handles PatternListAs(Tuple|Struct|Explicit|Implicit).
static auto HandlePatternList(Context& context, NodeKind node_kind,
Lex::TokenKind open_token_kind,
Lex::TokenKind close_token_kind,
StateKind param_state,
StateKind finish_state_empty,
StateKind finish_state_nonempty) -> void {
auto state = context.PopState();
if (state.in_field_shorthand_pattern) {
if (node_kind == NodeKind::TuplePatternStart) {
CARBON_DIAGNOSTIC(NestedTuplePatternInStructPatternShortField, Error,
"tuple pattern in shorthand struct pattern field; use "
"`.field = (...)` instead");
context.emitter().Emit(*context.position(),
NestedTuplePatternInStructPatternShortField);
context.ReturnErrorOnState();
} else if (node_kind == NodeKind::StructPatternStart) {
CARBON_DIAGNOSTIC(NestedStructPatternStructPatternShortField, Error,
"struct pattern in shorthand struct pattern field; use "
"`.field = {{...}` instead");
context.emitter().Emit(*context.position(),
NestedStructPatternStructPatternShortField);
context.ReturnErrorOnState();
}
}
auto open_token = context.ConsumeChecked(open_token_kind);
bool empty = context.PositionIs(close_token_kind);
context.PushStateForPattern(
empty ? finish_state_empty : finish_state_nonempty, state.in_var_pattern,
state.in_unused_pattern, state.in_field_shorthand_pattern,
state.binding_context, state.ambient_precedence);
context.AddLeafNode(node_kind, open_token);
if (!empty) {
context.PushStateForPattern(
param_state, state.in_var_pattern, state.in_unused_pattern,
state.in_field_shorthand_pattern, state.binding_context,
PrecedenceGroup::ForTopLevelExpr());
}
}
auto HandlePatternListAsTuple(Context& context) -> void {
// If the list is nonempty, use PatternListFinishAsParen as the parent. This
// will be replaced by PatternListFinishAsTuple if we see a comma.
HandlePatternList(
context, NodeKind::TuplePatternStart, Lex::TokenKind::OpenParen,
Lex::TokenKind::CloseParen, StateKind::PatternListElementAsTuple,
StateKind::PatternListFinishAsTuple, StateKind::PatternListFinishAsParen);
}
auto HandlePatternListAsStruct(Context& context) -> void {
HandlePatternList(
context, NodeKind::StructPatternStart, Lex::TokenKind::OpenCurlyBrace,
Lex::TokenKind::CloseCurlyBrace, StateKind::PatternListElementAsStruct,
StateKind::PatternListFinishAsStruct,
StateKind::PatternListFinishAsStruct);
}
auto HandlePatternListAsExplicit(Context& context) -> void {
HandlePatternList(context, NodeKind::ExplicitParamListStart,
Lex::TokenKind::OpenParen, Lex::TokenKind::CloseParen,
StateKind::PatternListElementAsExplicit,
StateKind::PatternListFinishAsExplicit,
StateKind::PatternListFinishAsExplicit);
}
auto HandlePatternListAsImplicit(Context& context) -> void {
HandlePatternList(context, NodeKind::ImplicitParamListStart,
Lex::TokenKind::OpenSquareBracket,
Lex::TokenKind::CloseSquareBracket,
StateKind::PatternListElementAsImplicit,
StateKind::PatternListFinishAsImplicit,
StateKind::PatternListFinishAsImplicit);
}
// Handles PatternListFinishAs(Paren|Tuple|Explicit|Implicit).
static auto HandlePatternListFinish(Context& context, NodeKind node_kind,
Lex::TokenKind token_kind) -> void {
auto state = context.PopState();
context.AddNode(node_kind, context.ConsumeChecked(token_kind),
state.has_error);
}
auto HandlePatternListFinishAsParen(Context& context) -> void {
HandlePatternListFinish(context, NodeKind::ParenPattern,
Lex::TokenKind::CloseParen);
}
auto HandlePatternListFinishAsTuple(Context& context) -> void {
HandlePatternListFinish(context, NodeKind::TuplePattern,
Lex::TokenKind::CloseParen);
}
auto HandlePatternListFinishAsStruct(Context& context) -> void {
HandlePatternListFinish(context, NodeKind::StructPattern,
Lex::TokenKind::CloseCurlyBrace);
}
auto HandlePatternListFinishAsExplicit(Context& context) -> void {
HandlePatternListFinish(context, NodeKind::ExplicitParamList,
Lex::TokenKind::CloseParen);
}
auto HandlePatternListFinishAsImplicit(Context& context) -> void {
HandlePatternListFinish(context, NodeKind::ImplicitParamList,
Lex::TokenKind::CloseSquareBracket);
}
} // namespace Carbon::Parse