carbon-language/carbon-lang
1
0
Fork 0
mirror of https://github.com/carbon-language/carbon-lang.git synced 2026-10-05 22:02:55 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Add a start-of-file token and parse node. (#3263)

Browse Source 
This removes a (very) hot branch in the lexer where we need to special
case when a token is the first token and can't look at its previous
token. It also seems like a generally nice change to the structure of
both the token buffer and parse tree as there are now bracketing
elements for both ends and we should be able to avoid similar branching
in the future.

Mostly mechanical updates to the lexer and parser code to handle this,
but also needed to special case the location information in the
autoupdate code. And then the usual large body of auto-updated tests.

No benchmark data for this change alone as in isolation and in the
current lexer structure it doesn't make a big difference. But this
branch was particularly difficult to handle when trying to update the
whitespace skipping code to be faster, and so I think it is worth
systematically avoiding the special case here.
This commit is contained in:
Chandler Carruth
2023-10-04 23:36:35 +00:00
committed by GitHub
parent 412e4fb461
commit a46ca6bf7a
208 changed files with 482 additions and 209 deletions
Download Patch File Download Diff File Expand all files Collapse all files
toolchain/parse/testdata/function/declaration/addr.carbon
Vendored
+1
View File
@@ -8,6 +8,7 @@ fn foo(addr a: i32*);
// CHECK:STDOUT: - filename: addr.carbon
// CHECK:STDOUT: parse_tree: [
// CHECK:STDOUT: {kind: 'FileStart', text: ''},
// CHECK:STDOUT: {kind: 'FunctionIntroducer', text: 'fn'},
// CHECK:STDOUT: {kind: 'Name', text: 'foo'},
// CHECK:STDOUT: {kind: 'ParameterListStart', text: '('},
toolchain/parse/testdata/function/declaration/basic.carbon
Vendored
+1
View File
@@ -8,6 +8,7 @@ fn F();
// CHECK:STDOUT: - filename: basic.carbon
// CHECK:STDOUT: parse_tree: [
// CHECK:STDOUT: {kind: 'FileStart', text: ''},
// CHECK:STDOUT: {kind: 'FunctionIntroducer', text: 'fn'},
// CHECK:STDOUT: {kind: 'Name', text: 'F'},
// CHECK:STDOUT: {kind: 'ParameterListStart', text: '('},
toolchain/parse/testdata/function/declaration/deduced_empty.carbon
Vendored
+1
View File
@@ -8,6 +8,7 @@ fn foo[]();
// CHECK:STDOUT: - filename: deduced_empty.carbon
// CHECK:STDOUT: parse_tree: [
// CHECK:STDOUT: {kind: 'FileStart', text: ''},
// CHECK:STDOUT: {kind: 'FunctionIntroducer', text: 'fn'},
// CHECK:STDOUT: {kind: 'Name', text: 'foo'},
// CHECK:STDOUT: {kind: 'DeducedParameterListStart', text: '['},
toolchain/parse/testdata/function/declaration/deduced_params.carbon
Vendored
+1
View File
@@ -8,6 +8,7 @@ fn foo[a: i32, b: i32]();
// CHECK:STDOUT: - filename: deduced_params.carbon
// CHECK:STDOUT: parse_tree: [
// CHECK:STDOUT: {kind: 'FileStart', text: ''},
// CHECK:STDOUT: {kind: 'FunctionIntroducer', text: 'fn'},
// CHECK:STDOUT: {kind: 'Name', text: 'foo'},
// CHECK:STDOUT: {kind: 'DeducedParameterListStart', text: '['},
toolchain/parse/testdata/function/declaration/fail_identifier_instead_of_sig.carbon
Vendored
+1
View File
@@ -11,6 +11,7 @@ fn foo bar;
// CHECK:STDOUT: - filename: fail_identifier_instead_of_sig.carbon
// CHECK:STDOUT: parse_tree: [
// CHECK:STDOUT: {kind: 'FileStart', text: ''},
// CHECK:STDOUT: {kind: 'FunctionIntroducer', text: 'fn'},
// CHECK:STDOUT: {kind: 'Name', text: 'foo'},
// CHECK:STDOUT: {kind: 'FunctionDeclaration', text: ';', has_error: yes, subtree_size: 3},
toolchain/parse/testdata/function/declaration/fail_missing_deduced_close.carbon
Vendored
+2 -1
View File
@@ -13,11 +13,12 @@
// CHECK:STDERR: ^
fn Div[();
// CHECK:STDERR: fail_missing_deduced_close.carbon:[[@LINE+14]]:21: ERROR: A `(` for parameters is required after deduced parameters.
// CHECK:STDERR: fail_missing_deduced_close.carbon:[[@LINE+15]]:21: ERROR: A `(` for parameters is required after deduced parameters.
// CHECK:STDERR: // CHECK:STDOUT: ]
// CHECK:STDERR: ^
// CHECK:STDOUT: - filename: fail_missing_deduced_close.carbon
// CHECK:STDOUT: parse_tree: [
// CHECK:STDOUT: {kind: 'FileStart', text: ''},
// CHECK:STDOUT: {kind: 'FunctionIntroducer', text: 'fn'},
// CHECK:STDOUT: {kind: 'Name', text: 'Div'},
// CHECK:STDOUT: {kind: 'DeducedParameterListStart', text: '['},
toolchain/parse/testdata/function/declaration/fail_missing_name.carbon
Vendored
+1
View File
@@ -11,6 +11,7 @@ fn ();
// CHECK:STDOUT: - filename: fail_missing_name.carbon
// CHECK:STDOUT: parse_tree: [
// CHECK:STDOUT: {kind: 'FileStart', text: ''},
// CHECK:STDOUT: {kind: 'FunctionIntroducer', text: 'fn'},
// CHECK:STDOUT: {kind: 'InvalidParse', text: '('},
// CHECK:STDOUT: {kind: 'FunctionDeclaration', text: ';', has_error: yes, subtree_size: 3},
toolchain/parse/testdata/function/declaration/fail_no_sig_or_semi.carbon
Vendored
+2 -1
View File
@@ -6,11 +6,12 @@
fn foo
// CHECK:STDERR: fail_no_sig_or_semi.carbon:[[@LINE+9]]:21: ERROR: `fn` requires a `(` for parameters.
// CHECK:STDERR: fail_no_sig_or_semi.carbon:[[@LINE+10]]:21: ERROR: `fn` requires a `(` for parameters.
// CHECK:STDERR: // CHECK:STDOUT: ]
// CHECK:STDERR: ^
// CHECK:STDOUT: - filename: fail_no_sig_or_semi.carbon
// CHECK:STDOUT: parse_tree: [
// CHECK:STDOUT: {kind: 'FileStart', text: ''},
// CHECK:STDOUT: {kind: 'FunctionIntroducer', text: 'fn'},
// CHECK:STDOUT: {kind: 'Name', text: 'foo'},
// CHECK:STDOUT: {kind: 'FunctionDeclaration', text: 'fn', has_error: yes, subtree_size: 3},
toolchain/parse/testdata/function/declaration/fail_only_fn_and_semi.carbon
Vendored
+1
View File
@@ -11,6 +11,7 @@ fn;
// CHECK:STDOUT: - filename: fail_only_fn_and_semi.carbon
// CHECK:STDOUT: parse_tree: [
// CHECK:STDOUT: {kind: 'FileStart', text: ''},
// CHECK:STDOUT: {kind: 'FunctionIntroducer', text: 'fn'},
// CHECK:STDOUT: {kind: 'InvalidParse', text: ';'},
// CHECK:STDOUT: {kind: 'FunctionDeclaration', text: ';', has_error: yes, subtree_size: 3},
toolchain/parse/testdata/function/declaration/fail_repeated_fn_and_semi.carbon
Vendored
+1
View File
@@ -11,6 +11,7 @@ fn fn;
// CHECK:STDOUT: - filename: fail_repeated_fn_and_semi.carbon
// CHECK:STDOUT: parse_tree: [
// CHECK:STDOUT: {kind: 'FileStart', text: ''},
// CHECK:STDOUT: {kind: 'FunctionIntroducer', text: 'fn'},
// CHECK:STDOUT: {kind: 'InvalidParse', text: 'fn'},
// CHECK:STDOUT: {kind: 'FunctionDeclaration', text: ';', has_error: yes, subtree_size: 3},
toolchain/parse/testdata/function/declaration/fail_skip_indented_newline_until_outdent.carbon
Vendored
+1
View File
@@ -14,6 +14,7 @@ fn F();
// CHECK:STDOUT: - filename: fail_skip_indented_newline_until_outdent.carbon
// CHECK:STDOUT: parse_tree: [
// CHECK:STDOUT: {kind: 'FileStart', text: ''},
// CHECK:STDOUT: {kind: 'FunctionIntroducer', text: 'fn'},
// CHECK:STDOUT: {kind: 'InvalidParse', text: '('},
// CHECK:STDOUT: {kind: 'FunctionDeclaration', text: 'fn', has_error: yes, subtree_size: 3},
toolchain/parse/testdata/function/declaration/fail_skip_indented_newline_with_semi.carbon
Vendored
+1
View File
@@ -14,6 +14,7 @@ fn F();
// CHECK:STDOUT: - filename: fail_skip_indented_newline_with_semi.carbon
// CHECK:STDOUT: parse_tree: [
// CHECK:STDOUT: {kind: 'FileStart', text: ''},
// CHECK:STDOUT: {kind: 'FunctionIntroducer', text: 'fn'},
// CHECK:STDOUT: {kind: 'InvalidParse', text: '('},
// CHECK:STDOUT: {kind: 'FunctionDeclaration', text: ';', has_error: yes, subtree_size: 3},
toolchain/parse/testdata/function/declaration/fail_skip_indented_newline_without_semi.carbon
Vendored
+1
View File
@@ -14,6 +14,7 @@ fn F();
// CHECK:STDOUT: - filename: fail_skip_indented_newline_without_semi.carbon
// CHECK:STDOUT: parse_tree: [
// CHECK:STDOUT: {kind: 'FileStart', text: ''},
// CHECK:STDOUT: {kind: 'FunctionIntroducer', text: 'fn'},
// CHECK:STDOUT: {kind: 'InvalidParse', text: '('},
// CHECK:STDOUT: {kind: 'FunctionDeclaration', text: 'fn', has_error: yes, subtree_size: 3},
toolchain/parse/testdata/function/declaration/fail_skip_to_newline_without_semi.carbon
Vendored
+1
View File
@@ -12,6 +12,7 @@ fn F();
// CHECK:STDOUT: - filename: fail_skip_to_newline_without_semi.carbon
// CHECK:STDOUT: parse_tree: [
// CHECK:STDOUT: {kind: 'FileStart', text: ''},
// CHECK:STDOUT: {kind: 'FunctionIntroducer', text: 'fn'},
// CHECK:STDOUT: {kind: 'InvalidParse', text: '('},
// CHECK:STDOUT: {kind: 'FunctionDeclaration', text: 'fn', has_error: yes, subtree_size: 3},
toolchain/parse/testdata/function/declaration/fail_skip_without_semi_to_curly.carbon
Vendored
+1
View File
@@ -12,6 +12,7 @@ fn F();
// CHECK:STDOUT: - filename: fail_skip_without_semi_to_curly.carbon
// CHECK:STDOUT: parse_tree: [
// CHECK:STDOUT: {kind: 'FileStart', text: ''},
// CHECK:STDOUT: {kind: 'EmptyDeclaration', text: 'struct', has_error: yes},
// CHECK:STDOUT: {kind: 'FunctionIntroducer', text: 'fn'},
// CHECK:STDOUT: {kind: 'Name', text: 'F'},
toolchain/parse/testdata/function/declaration/fail_with_identifier_as_param.carbon
Vendored
+1
View File
@@ -11,6 +11,7 @@ fn foo(bar);
// CHECK:STDOUT: - filename: fail_with_identifier_as_param.carbon
// CHECK:STDOUT: parse_tree: [
// CHECK:STDOUT: {kind: 'FileStart', text: ''},
// CHECK:STDOUT: {kind: 'FunctionIntroducer', text: 'fn'},
// CHECK:STDOUT: {kind: 'Name', text: 'foo'},
// CHECK:STDOUT: {kind: 'ParameterListStart', text: '('},
toolchain/parse/testdata/function/declaration/fail_without_name_and_many_tokens_in_params.carbon
Vendored
+1
View File
@@ -11,6 +11,7 @@ fn (a tokens c d e f g h i j k l m n o p q r s t u v w x y z);
// CHECK:STDOUT: - filename: fail_without_name_and_many_tokens_in_params.carbon
// CHECK:STDOUT: parse_tree: [
// CHECK:STDOUT: {kind: 'FileStart', text: ''},
// CHECK:STDOUT: {kind: 'FunctionIntroducer', text: 'fn'},
// CHECK:STDOUT: {kind: 'InvalidParse', text: '('},
// CHECK:STDOUT: {kind: 'FunctionDeclaration', text: ';', has_error: yes, subtree_size: 3},
toolchain/parse/testdata/function/declaration/params.carbon
Vendored
+1
View File
@@ -8,6 +8,7 @@ fn foo(a: i32, b: i32);
// CHECK:STDOUT: - filename: params.carbon
// CHECK:STDOUT: parse_tree: [
// CHECK:STDOUT: {kind: 'FileStart', text: ''},
// CHECK:STDOUT: {kind: 'FunctionIntroducer', text: 'fn'},
// CHECK:STDOUT: {kind: 'Name', text: 'foo'},
// CHECK:STDOUT: {kind: 'ParameterListStart', text: '('},
toolchain/parse/testdata/function/declaration/with_return_type.carbon
Vendored
+1
View File
@@ -8,6 +8,7 @@ fn foo() -> u32;
// CHECK:STDOUT: - filename: with_return_type.carbon
// CHECK:STDOUT: parse_tree: [
// CHECK:STDOUT: {kind: 'FileStart', text: ''},
// CHECK:STDOUT: {kind: 'FunctionIntroducer', text: 'fn'},
// CHECK:STDOUT: {kind: 'Name', text: 'foo'},
// CHECK:STDOUT: {kind: 'ParameterListStart', text: '('},
toolchain/parse/testdata/function/definition/basic.carbon
Vendored
+1
View File
@@ -9,6 +9,7 @@ fn F() {
// CHECK:STDOUT: - filename: basic.carbon
// CHECK:STDOUT: parse_tree: [
// CHECK:STDOUT: {kind: 'FileStart', text: ''},
// CHECK:STDOUT: {kind: 'FunctionIntroducer', text: 'fn'},
// CHECK:STDOUT: {kind: 'Name', text: 'F'},
// CHECK:STDOUT: {kind: 'ParameterListStart', text: '('},
toolchain/parse/testdata/function/definition/fail_identifier_in_statements.carbon
Vendored
+1
View File
@@ -15,6 +15,7 @@ fn F() {
// CHECK:STDOUT: - filename: fail_identifier_in_statements.carbon
// CHECK:STDOUT: parse_tree: [
// CHECK:STDOUT: {kind: 'FileStart', text: ''},
// CHECK:STDOUT: {kind: 'FunctionIntroducer', text: 'fn'},
// CHECK:STDOUT: {kind: 'Name', text: 'F'},
// CHECK:STDOUT: {kind: 'ParameterListStart', text: '('},
toolchain/parse/testdata/function/definition/with_params.carbon
Vendored
+1
View File
@@ -10,6 +10,7 @@ fn foo(bar: i64, baz: i64) {
// CHECK:STDOUT: - filename: with_params.carbon
// CHECK:STDOUT: parse_tree: [
// CHECK:STDOUT: {kind: 'FileStart', text: ''},
// CHECK:STDOUT: {kind: 'FunctionIntroducer', text: 'fn'},
// CHECK:STDOUT: {kind: 'Name', text: 'foo'},
// CHECK:STDOUT: {kind: 'ParameterListStart', text: '('},
toolchain/parse/testdata/function/definition/with_return_type.carbon
Vendored
+1
View File
@@ -10,6 +10,7 @@ fn foo() -> f64 {
// CHECK:STDOUT: - filename: with_return_type.carbon
// CHECK:STDOUT: parse_tree: [
// CHECK:STDOUT: {kind: 'FileStart', text: ''},
// CHECK:STDOUT: {kind: 'FunctionIntroducer', text: 'fn'},
// CHECK:STDOUT: {kind: 'Name', text: 'foo'},
// CHECK:STDOUT: {kind: 'ParameterListStart', text: '('},