carbon-language/carbon-lang
1
0
Fork 0
mirror of https://github.com/carbon-language/carbon-lang.git synced 2026-10-05 22:02:55 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Improve diagnostics and error recovery for invalid identifiers. (#7249)

Browse Source 
If a keyword or a sized type literal (eg, `f2`) is used in a context
where we are confident that we are expecting an identifier -- either
before a `:` in a binding pattern or after a `.` in a member access or
designator -- then recover as if a raw identifier was used.

This appears to be a particular stumbling block for coding agents, so
seems worth paying special attention to.

Add a mechanism to the tokenized buffer to track additional tokens
synthesized for error recovery so that we can keep the lexed token
sequence immutable and still satisfy the invariants throughout the rest
of the toolchain for recovery tokens. Thanks to chandlerc for suggesting
this approach!
This commit is contained in:
Richard Smith
2026-05-22 23:53:19 +00:00
committed by GitHub
parent 64e890c246
commit 49024c8d83
14 changed files with 163 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
toolchain/parse/testdata/basics/fail_invalid_designators.carbon
Vendored
+2 -2
View File
@@ -15,7 +15,7 @@ fn F() {
// CHECK:STDERR: ^
// CHECK:STDERR:
a.;
// CHECK:STDERR: fail_invalid_designators.carbon:[[@LINE+4]]:5: error: expected identifier after `.` [ExpectedIdentifierAfterPeriodOrArrow]
// CHECK:STDERR: fail_invalid_designators.carbon:[[@LINE+4]]:5: error: expected identifier after `.`; prefix reserved word with `r#` to form a valid identifier [ExpectedIdentifierAfterPeriodOrArrow]
// CHECK:STDERR: a.fn;
// CHECK:STDERR: ^~
// CHECK:STDERR:
@@ -35,7 +35,7 @@ fn F() {
// CHECK:STDOUT: {kind: 'MemberAccessExpr', text: '.', subtree_size: 3},
// CHECK:STDOUT: {kind: 'ExprStatement', text: ';', has_error: yes, subtree_size: 4},
// CHECK:STDOUT: {kind: 'IdentifierNameExpr', text: 'a'},
// CHECK:STDOUT: {kind: 'IdentifierNameNotBeforeSignature', text: 'fn', has_error: yes},
// CHECK:STDOUT: {kind: 'IdentifierNameNotBeforeSignature', text: 'fn'},
// CHECK:STDOUT: {kind: 'MemberAccessExpr', text: '.', subtree_size: 3},
// CHECK:STDOUT: {kind: 'ExprStatement', text: ';', subtree_size: 4},
// CHECK:STDOUT: {kind: 'FunctionDefinition', text: '}', subtree_size: 14},
toolchain/parse/testdata/member_access/fail_keyword.carbon
Vendored
+12 -12
View File
@@ -9,32 +9,32 @@
// TIP: bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/parse/testdata/member_access/fail_keyword.carbon
fn F() {
// CHECK:STDERR: fail_keyword.carbon:[[@LINE+4]]:5: error: expected identifier after `.` [ExpectedIdentifierAfterPeriodOrArrow]
// CHECK:STDERR: fail_keyword.carbon:[[@LINE+4]]:5: error: expected identifier after `.`; prefix reserved word with `r#` to form a valid identifier [ExpectedIdentifierAfterPeriodOrArrow]
// CHECK:STDERR: a.self;
// CHECK:STDERR: ^~~~
// CHECK:STDERR:
a.self;
// CHECK:STDERR: fail_keyword.carbon:[[@LINE+4]]:5: error: expected identifier after `.` [ExpectedIdentifierAfterPeriodOrArrow]
// CHECK:STDERR: fail_keyword.carbon:[[@LINE+4]]:5: error: expected identifier after `.`; prefix reserved word with `r#` to form a valid identifier [ExpectedIdentifierAfterPeriodOrArrow]
// CHECK:STDERR: a.Self;
// CHECK:STDERR: ^~~~
// CHECK:STDERR:
a.Self;
// CHECK:STDERR: fail_keyword.carbon:[[@LINE+4]]:5: error: expected identifier after `.` [ExpectedIdentifierAfterPeriodOrArrow]
// CHECK:STDERR: fail_keyword.carbon:[[@LINE+4]]:5: error: expected identifier after `.`; prefix reserved word with `r#` to form a valid identifier [ExpectedIdentifierAfterPeriodOrArrow]
// CHECK:STDERR: a.for;
// CHECK:STDERR: ^~~
// CHECK:STDERR:
a.for;
// CHECK:STDERR: fail_keyword.carbon:[[@LINE+4]]:6: error: expected identifier after `->` [ExpectedIdentifierAfterPeriodOrArrow]
// CHECK:STDERR: fail_keyword.carbon:[[@LINE+4]]:6: error: expected identifier after `->`; prefix reserved word with `r#` to form a valid identifier [ExpectedIdentifierAfterPeriodOrArrow]
// CHECK:STDERR: p->self;
// CHECK:STDERR: ^~~~
// CHECK:STDERR:
p->self;
// CHECK:STDERR: fail_keyword.carbon:[[@LINE+4]]:6: error: expected identifier after `->` [ExpectedIdentifierAfterPeriodOrArrow]
// CHECK:STDERR: fail_keyword.carbon:[[@LINE+4]]:6: error: expected identifier after `->`; prefix reserved word with `r#` to form a valid identifier [ExpectedIdentifierAfterPeriodOrArrow]
// CHECK:STDERR: p->Self;
// CHECK:STDERR: ^~~~
// CHECK:STDERR:
p->Self;
// CHECK:STDERR: fail_keyword.carbon:[[@LINE+4]]:6: error: expected identifier after `->` [ExpectedIdentifierAfterPeriodOrArrow]
// CHECK:STDERR: fail_keyword.carbon:[[@LINE+4]]:6: error: expected identifier after `->`; prefix reserved word with `r#` to form a valid identifier [ExpectedIdentifierAfterPeriodOrArrow]
// CHECK:STDERR: p->while;
// CHECK:STDERR: ^~~~~
// CHECK:STDERR:
@@ -50,27 +50,27 @@ fn F() {
// CHECK:STDOUT: {kind: 'ExplicitParamList', text: ')', subtree_size: 2},
// CHECK:STDOUT: {kind: 'FunctionDefinitionStart', text: '{', subtree_size: 5},
// CHECK:STDOUT: {kind: 'IdentifierNameExpr', text: 'a'},
// CHECK:STDOUT: {kind: 'IdentifierNameNotBeforeSignature', text: 'self', has_error: yes},
// CHECK:STDOUT: {kind: 'IdentifierNameNotBeforeSignature', text: 'self'},
// CHECK:STDOUT: {kind: 'MemberAccessExpr', text: '.', subtree_size: 3},
// CHECK:STDOUT: {kind: 'ExprStatement', text: ';', subtree_size: 4},
// CHECK:STDOUT: {kind: 'IdentifierNameExpr', text: 'a'},
// CHECK:STDOUT: {kind: 'IdentifierNameNotBeforeSignature', text: 'Self', has_error: yes},
// CHECK:STDOUT: {kind: 'IdentifierNameNotBeforeSignature', text: 'Self'},
// CHECK:STDOUT: {kind: 'MemberAccessExpr', text: '.', subtree_size: 3},
// CHECK:STDOUT: {kind: 'ExprStatement', text: ';', subtree_size: 4},
// CHECK:STDOUT: {kind: 'IdentifierNameExpr', text: 'a'},
// CHECK:STDOUT: {kind: 'IdentifierNameNotBeforeSignature', text: 'for', has_error: yes},
// CHECK:STDOUT: {kind: 'IdentifierNameNotBeforeSignature', text: 'for'},
// CHECK:STDOUT: {kind: 'MemberAccessExpr', text: '.', subtree_size: 3},
// CHECK:STDOUT: {kind: 'ExprStatement', text: ';', subtree_size: 4},
// CHECK:STDOUT: {kind: 'IdentifierNameExpr', text: 'p'},
// CHECK:STDOUT: {kind: 'IdentifierNameNotBeforeSignature', text: 'self', has_error: yes},
// CHECK:STDOUT: {kind: 'IdentifierNameNotBeforeSignature', text: 'self'},
// CHECK:STDOUT: {kind: 'PointerMemberAccessExpr', text: '->', subtree_size: 3},
// CHECK:STDOUT: {kind: 'ExprStatement', text: ';', subtree_size: 4},
// CHECK:STDOUT: {kind: 'IdentifierNameExpr', text: 'p'},
// CHECK:STDOUT: {kind: 'IdentifierNameNotBeforeSignature', text: 'Self', has_error: yes},
// CHECK:STDOUT: {kind: 'IdentifierNameNotBeforeSignature', text: 'Self'},
// CHECK:STDOUT: {kind: 'PointerMemberAccessExpr', text: '->', subtree_size: 3},
// CHECK:STDOUT: {kind: 'ExprStatement', text: ';', subtree_size: 4},
// CHECK:STDOUT: {kind: 'IdentifierNameExpr', text: 'p'},
// CHECK:STDOUT: {kind: 'IdentifierNameNotBeforeSignature', text: 'while', has_error: yes},
// CHECK:STDOUT: {kind: 'IdentifierNameNotBeforeSignature', text: 'while'},
// CHECK:STDOUT: {kind: 'PointerMemberAccessExpr', text: '->', subtree_size: 3},
// CHECK:STDOUT: {kind: 'ExprStatement', text: ';', subtree_size: 4},
// CHECK:STDOUT: {kind: 'FunctionDefinition', text: '}', subtree_size: 30},
toolchain/parse/testdata/package_expr/fail_in_name.carbon
Vendored
+2 -2
View File
@@ -15,7 +15,7 @@
var package.n: i32;
// `val` is a keyword, but during error recovery we treat it as an identifier.
// CHECK:STDERR: fail_in_name.carbon:[[@LINE+8]]:13: error: expected identifier after `.` [ExpectedIdentifierAfterPeriodOrArrow]
// CHECK:STDERR: fail_in_name.carbon:[[@LINE+8]]:13: error: expected identifier after `.`; prefix reserved word with `r#` to form a valid identifier [ExpectedIdentifierAfterPeriodOrArrow]
// CHECK:STDERR: var package.val: i32;
// CHECK:STDERR: ^~~
// CHECK:STDERR:
@@ -49,7 +49,7 @@ class package.C {
// CHECK:STDOUT: {kind: 'VariableDecl', text: ';', subtree_size: 6},
// CHECK:STDOUT: {kind: 'VariableIntroducer', text: 'var'},
// CHECK:STDOUT: {kind: 'PackageExpr', text: 'package'},
// CHECK:STDOUT: {kind: 'IdentifierNameNotBeforeSignature', text: 'val', has_error: yes},
// CHECK:STDOUT: {kind: 'IdentifierNameNotBeforeSignature', text: 'val'},
// CHECK:STDOUT: {kind: 'MemberAccessExpr', text: '.', subtree_size: 3},
// CHECK:STDOUT: {kind: 'VariablePattern', text: 'var', has_error: yes, subtree_size: 4},
// CHECK:STDOUT: {kind: 'VariableDecl', text: ';', subtree_size: 6},
toolchain/parse/testdata/var/fail_bad_name.carbon
Vendored
+24
View File
@@ -14,6 +14,18 @@
// CHECK:STDERR:
var *;
// CHECK:STDERR: fail_bad_name.carbon:[[@LINE+4]]:11: error: expected name in binding pattern; prefix reserved word with `r#` to form a valid identifier [ExpectedBindingPattern]
// CHECK:STDERR: var choice: i32;
// CHECK:STDERR: ^
// CHECK:STDERR:
var choice: i32;
// CHECK:STDERR: fail_bad_name.carbon:[[@LINE+4]]:7: error: expected name in binding pattern; prefix reserved word with `r#` to form a valid identifier [ExpectedBindingPattern]
// CHECK:STDERR: var f2: f32;
// CHECK:STDERR: ^
// CHECK:STDERR:
var f2: f32;
// CHECK:STDOUT: - filename: fail_bad_name.carbon
// CHECK:STDOUT: parse_tree: [
// CHECK:STDOUT: {kind: 'FileStart', text: ''},
@@ -22,5 +34,17 @@ var *;
// CHECK:STDOUT: {kind: 'PrefixOperatorStar', text: '*', has_error: yes, subtree_size: 2},
// CHECK:STDOUT: {kind: 'VariablePattern', text: 'var', subtree_size: 3},
// CHECK:STDOUT: {kind: 'VariableDecl', text: ';', subtree_size: 5},
// CHECK:STDOUT: {kind: 'VariableIntroducer', text: 'var'},
// CHECK:STDOUT: {kind: 'IdentifierNameNotBeforeSignature', text: 'choice'},
// CHECK:STDOUT: {kind: 'IntTypeLiteral', text: 'i32'},
// CHECK:STDOUT: {kind: 'VarBindingPattern', text: ':', subtree_size: 3},
// CHECK:STDOUT: {kind: 'VariablePattern', text: 'var', subtree_size: 4},
// CHECK:STDOUT: {kind: 'VariableDecl', text: ';', subtree_size: 6},
// CHECK:STDOUT: {kind: 'VariableIntroducer', text: 'var'},
// CHECK:STDOUT: {kind: 'IdentifierNameNotBeforeSignature', text: 'f2'},
// CHECK:STDOUT: {kind: 'FloatTypeLiteral', text: 'f32'},
// CHECK:STDOUT: {kind: 'VarBindingPattern', text: ':', subtree_size: 3},
// CHECK:STDOUT: {kind: 'VariablePattern', text: 'var', subtree_size: 4},
// CHECK:STDOUT: {kind: 'VariableDecl', text: ';', subtree_size: 6},
// CHECK:STDOUT: {kind: 'FileEnd', text: ''},
// CHECK:STDOUT: ]