carbon-language/carbon-lang
1
0
Fork 0
mirror of https://github.com/carbon-language/carbon-lang.git synced 2026-10-05 22:02:55 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Align on FileStart/FileEnd for naming. (#3428)

Browse Source 
The lexer has been using EndOfFile form (stemming from EOF), parser went
to FileEnd form. This consolidates on FileEnd form.
This commit is contained in:
Jon Ross-Perkins
2023-11-29 16:36:57 +00:00
committed by GitHub
parent 239f8030a4
commit 3f208e27f9
28 changed files with 147 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files
toolchain/lex/lex.cpp
+8 -9
View File
@@ -157,9 +157,9 @@ class [[clang::internal_linkage]] Lexer {
auto LexError(llvm::StringRef source_text, ssize_t& position) -> LexResult;
auto LexStartOfFile(llvm::StringRef source_text, ssize_t& position) -> void;
auto LexFileStart(llvm::StringRef source_text, ssize_t& position) -> void;
auto LexEndOfFile(llvm::StringRef source_text, ssize_t position) -> void;
auto LexFileEnd(llvm::StringRef source_text, ssize_t position) -> void;
// The main entry point for dispatching through the lexer's table. This method
// should always fully consume the source text.
@@ -647,7 +647,7 @@ static auto DispatchNext(Lexer& lexer, llvm::StringRef source_text,
// When we finish the source text, stop recursing. We also hint this so that
// the tail-dispatch is optimized as that's essentially the loop back-edge
// and this is the loop exit.
lexer.LexEndOfFile(source_text, position);
lexer.LexFileEnd(source_text, position);
}
auto Lexer::Lex() && -> TokenizedBuffer {
@@ -657,7 +657,7 @@ auto Lexer::Lex() && -> TokenizedBuffer {
CreateLines(source_text);
ssize_t position = 0;
LexStartOfFile(source_text, position);
LexFileStart(source_text, position);
// Manually enter the dispatch loop. This call will tail-recurse through the
// dispatch table until everything from source_text is consumed.
@@ -1263,12 +1263,12 @@ auto Lexer::LexError(llvm::StringRef source_text, ssize_t& position)
return token;
}
auto Lexer::LexStartOfFile(llvm::StringRef source_text, ssize_t& position)
auto Lexer::LexFileStart(llvm::StringRef source_text, ssize_t& position)
-> void {
// Before lexing any source text, add the start-of-file token so that code
// can assume a non-empty token buffer for the rest of lexing. Note that the
// start-of-file always has trailing space because it *is* whitespace.
buffer_.AddToken({.kind = TokenKind::StartOfFile,
buffer_.AddToken({.kind = TokenKind::FileStart,
.has_trailing_space = true,
.token_line = current_line(),
.column = 0});
@@ -1281,8 +1281,7 @@ auto Lexer::LexStartOfFile(llvm::StringRef source_text, ssize_t& position)
line_info->indent = position;
}
auto Lexer::LexEndOfFile(llvm::StringRef source_text, ssize_t position)
-> void {
auto Lexer::LexFileEnd(llvm::StringRef source_text, ssize_t position) -> void {
CARBON_CHECK(position == static_cast<ssize_t>(source_text.size()));
// Check if the last line is empty and not the first line (and only). If so,
// re-pin the last line to be the prior one so that diagnostics and editors
@@ -1307,7 +1306,7 @@ auto Lexer::LexEndOfFile(llvm::StringRef source_text, ssize_t position)
CloseInvalidOpenGroups(TokenKind::Error, position);
}
buffer_.AddToken({.kind = TokenKind::EndOfFile,
buffer_.AddToken({.kind = TokenKind::FileEnd,
.token_line = current_line(),
.column = ComputeColumn(position)});
}
toolchain/lex/testdata/basic_syntax.carbon
Vendored
+2 -2
View File
@@ -5,7 +5,7 @@
// AUTOUPDATE
// CHECK:STDOUT: - filename: basic_syntax.carbon
// CHECK:STDOUT: tokens: [
// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
fn run(String program) {
// CHECK:STDOUT: { index: 1, kind: 'Fn', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'fn', has_trailing_space: true },
@@ -22,5 +22,5 @@ fn run(String program) {
}
// CHECK:STDOUT: { index: 11, kind: 'CloseCurlyBrace', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '}', opening_token: 7, has_trailing_space: true },
// CHECK:STDOUT: { index: 12, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: { index: 12, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: ]
toolchain/lex/testdata/fail_bad_comment_introducers.carbon
Vendored
+2 -2
View File
@@ -5,7 +5,7 @@
// AUTOUPDATE
// CHECK:STDOUT: - filename: fail_bad_comment_introducers.carbon
// CHECK:STDOUT: tokens: [
// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
//
// Comments have to have whitespace after `//` currently.
@@ -50,5 +50,5 @@
// An extra un-indented comment line to anchor the end of the file checks.
// CHECK:STDOUT: { index: 1, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: { index: 1, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: ]
toolchain/lex/testdata/fail_bad_comment_introducers_mid_block_indent_change.carbon
Vendored
+2 -2
View File
@@ -30,7 +30,7 @@
// CHECK:STDERR: ^
// CHECK:STDOUT: - filename: fail_bad_comment_introducers_mid_block_indent_change.carbon
// CHECK:STDOUT: tokens: [
// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
// CHECK:STDOUT: { index: 1, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: { index: 1, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: ]
toolchain/lex/testdata/fail_bad_raw_identifier.carbon
Vendored
+2 -2
View File
@@ -5,7 +5,7 @@
// AUTOUPDATE
// CHECK:STDOUT: - filename: fail_bad_raw_identifier.carbon
// CHECK:STDOUT: tokens: [
// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
// Missing the character after `#`.
// CHECK:STDERR: fail_bad_raw_identifier.carbon:[[@LINE+3]]:2: ERROR: Encountered unrecognized characters while parsing.
@@ -32,5 +32,5 @@ r#á
// CHECK:STDOUT: { index: 6, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'r', identifier: 0 },
// CHECK:STDOUT: { index: 7, kind: 'Error', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: '#á', has_trailing_space: true },
// CHECK:STDOUT: { index: 8, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: { index: 8, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: ]
toolchain/lex/testdata/fail_block_string_second_line.carbon
Vendored
+2 -2
View File
@@ -12,7 +12,7 @@
// CHECK:STDERR: ^
var s: String = '''
error here: '''
// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: 1, column: 1, indent: 1, spelling: '', has_trailing_space: true },
// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: 1, column: 1, indent: 1, spelling: '', has_trailing_space: true },
// CHECK:STDOUT: { index: 1, kind: 'Var', line: {{ *}}[[@LINE-3]], column: 1, indent: 1, spelling: 'var', has_trailing_space: true },
// CHECK:STDOUT: { index: 2, kind: 'Identifier', line: {{ *}}[[@LINE-4]], column: 5, indent: 1, spelling: 's', identifier: 0 },
// CHECK:STDOUT: { index: 3, kind: 'Colon', line: {{ *}}[[@LINE-5]], column: 6, indent: 1, spelling: ':', has_trailing_space: true },
@@ -21,5 +21,5 @@ var s: String = '''
// CHECK:STDOUT: { index: 6, kind: 'StringLiteral', line: {{ *}}[[@LINE-8]], column: 17, indent: 1, spelling: ''''
// CHECK:STDOUT: error here: '''', value: `error here: `, has_trailing_space: true },
// CHECK:STDOUT: { index: 7, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: { index: 7, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: ]
toolchain/lex/testdata/fail_multifile.carbon
Vendored
+6 -6
View File
@@ -7,24 +7,24 @@
// --- a.carbon
// CHECK:STDOUT: - filename: a.carbon
// CHECK:STDOUT: tokens: [
// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
// CHECK:STDERR: a.carbon:[[@LINE+3]]:3: ERROR: Empty digit sequence in numeric literal.
// CHECK:STDERR: 1.a
// CHECK:STDERR: ^
1.a
// CHECK:STDOUT: { index: 1, kind: 'Error', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '1.a', has_trailing_space: true },
// CHECK:STDOUT: { index: 1, kind: 'Error', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '1.a', has_trailing_space: true },
// CHECK:STDOUT: { index: 2, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: { index: 2, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: ]
// --- b.carbon
// CHECK:STDOUT: - filename: b.carbon
// CHECK:STDOUT: tokens: [
// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
// CHECK:STDERR: b.carbon:[[@LINE+3]]:3: ERROR: Empty digit sequence in numeric literal.
// CHECK:STDERR: 2.b
// CHECK:STDERR: ^
2.b
// CHECK:STDOUT: { index: 1, kind: 'Error', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '2.b', has_trailing_space: true },
// CHECK:STDOUT: { index: 1, kind: 'Error', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '2.b', has_trailing_space: true },
// CHECK:STDOUT: { index: 2, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: { index: 2, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: ]
toolchain/lex/testdata/fail_trailing_comments.carbon
Vendored
+2 -2
View File
@@ -23,7 +23,7 @@ var c: i32 = 0.4; // still more trailing comment
// CHECK:STDERR: ^
// CHECK:STDOUT: - filename: fail_trailing_comments.carbon
// CHECK:STDOUT: tokens: [
// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
// CHECK:STDOUT: { index: 1, kind: 'Var', line: {{ *}}[[@LINE-19]], column: 1, indent: 1, spelling: 'var', has_trailing_space: true },
// CHECK:STDOUT: { index: 2, kind: 'Identifier', line: {{ *}}[[@LINE-20]], column: 5, indent: 1, spelling: 'a', identifier: 0 },
// CHECK:STDOUT: { index: 3, kind: 'Colon', line: {{ *}}[[@LINE-21]], column: 6, indent: 1, spelling: ':', has_trailing_space: true },
@@ -46,5 +46,5 @@ var c: i32 = 0.4; // still more trailing comment
// CHECK:STDOUT: { index: 20, kind: 'RealLiteral', line: {{ *}}[[@LINE-36]], column: 14, indent: 1, spelling: '0.4', value: `4*10^-1` },
// CHECK:STDOUT: { index: 21, kind: 'Semi', line: {{ *}}[[@LINE-37]], column: 17, indent: 1, spelling: ';', has_trailing_space: true },
// CHECK:STDOUT: { index: 22, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: { index: 22, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: ]
toolchain/lex/testdata/keywords.carbon
Vendored
+2 -2
View File
@@ -5,7 +5,7 @@
// AUTOUPDATE
// CHECK:STDOUT: - filename: keywords.carbon
// CHECK:STDOUT: tokens: [
// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
abstract
// CHECK:STDOUT: { index: 1, kind: 'Abstract', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'abstract', has_trailing_space: true },
@@ -133,5 +133,5 @@ while
notakeyword
// CHECK:STDOUT: { index: 62, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'notakeyword', identifier: 0, has_trailing_space: true },
// CHECK:STDOUT: { index: 63, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: { index: 63, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: ]
toolchain/lex/testdata/multifile.carbon
Vendored
+10 -10
View File
@@ -7,23 +7,23 @@
// --- a.carbon
// CHECK:STDOUT: - filename: a.carbon
// CHECK:STDOUT: tokens: [
// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
a;
// CHECK:STDOUT: { index: 1, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'a', identifier: 0 },
// CHECK:STDOUT: { index: 2, kind: 'Semi', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: ';', has_trailing_space: true },
// CHECK:STDOUT: { index: 1, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'a', identifier: 0 },
// CHECK:STDOUT: { index: 2, kind: 'Semi', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: ';', has_trailing_space: true },
// CHECK:STDOUT: { index: 3, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: { index: 3, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: ]
// --- b.carbon
// CHECK:STDOUT: - filename: b.carbon
// CHECK:STDOUT: tokens: [
// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
b;
// CHECK:STDOUT: { index: 1, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'b', identifier: 0 },
// CHECK:STDOUT: { index: 2, kind: 'Semi', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: ';', has_trailing_space: true },
// CHECK:STDOUT: { index: 1, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'b', identifier: 0 },
// CHECK:STDOUT: { index: 2, kind: 'Semi', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: ';', has_trailing_space: true },
a;
// CHECK:STDOUT: { index: 3, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'a', identifier: 1 },
// CHECK:STDOUT: { index: 4, kind: 'Semi', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: ';', has_trailing_space: true },
// CHECK:STDOUT: { index: 3, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'a', identifier: 1 },
// CHECK:STDOUT: { index: 4, kind: 'Semi', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: ';', has_trailing_space: true },
// CHECK:STDOUT: { index: 5, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: { index: 5, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: ]
toolchain/lex/testdata/numeric_literals.carbon
Vendored
+2 -2
View File
@@ -5,7 +5,7 @@
// AUTOUPDATE
// CHECK:STDOUT: - filename: numeric_literals.carbon
// CHECK:STDOUT: tokens: [
// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
fn F() {
// CHECK:STDOUT: { index: 1, kind: 'Fn', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'fn', has_trailing_space: true },
@@ -82,5 +82,5 @@ fn F() {
}
// CHECK:STDOUT: { index: 54, kind: 'CloseCurlyBrace', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '}', opening_token: 5, has_trailing_space: true },
// CHECK:STDOUT: { index: 55, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: { index: 55, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: ]
toolchain/lex/testdata/printing_digit_padding.carbon
Vendored
+6 -6
View File
@@ -5,16 +5,16 @@
// AUTOUPDATE
// CHECK:STDOUT: - filename: printing_digit_padding.carbon
// CHECK:STDOUT: tokens: [
// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
;
// CHECK:STDOUT: { index: 1, kind: 'Semi', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: ';', has_trailing_space: true },
// CHECK:STDOUT: { index: 1, kind: 'Semi', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: ';', has_trailing_space: true },
// Test digit padding with values of 9, 10, and 11.
;;;
// CHECK:STDOUT: { index: 2, kind: 'Semi', line: {{ *}}[[@LINE-1]], column: 9, indent: 9, spelling: ';' },
// CHECK:STDOUT: { index: 3, kind: 'Semi', line: {{ *}}[[@LINE-2]], column: 10, indent: 9, spelling: ';' },
// CHECK:STDOUT: { index: 4, kind: 'Semi', line: {{ *}}[[@LINE-3]], column: 11, indent: 9, spelling: ';', has_trailing_space: true },
// CHECK:STDOUT: { index: 2, kind: 'Semi', line: {{ *}}[[@LINE-1]], column: 9, indent: 9, spelling: ';' },
// CHECK:STDOUT: { index: 3, kind: 'Semi', line: {{ *}}[[@LINE-2]], column: 10, indent: 9, spelling: ';' },
// CHECK:STDOUT: { index: 4, kind: 'Semi', line: {{ *}}[[@LINE-3]], column: 11, indent: 9, spelling: ';', has_trailing_space: true },
// CHECK:STDOUT: { index: 5, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: { index: 5, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: ]
toolchain/lex/testdata/printing_integer_literal.carbon
Vendored
+2 -2
View File
@@ -5,10 +5,10 @@
// AUTOUPDATE
// CHECK:STDOUT: - filename: printing_integer_literal.carbon
// CHECK:STDOUT: tokens: [
// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
123
// CHECK:STDOUT: { index: 1, kind: 'IntegerLiteral', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '123', value: `123`, has_trailing_space: true },
// CHECK:STDOUT: { index: 2, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: { index: 2, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: ]
toolchain/lex/testdata/printing_real_literal.carbon
Vendored
+2 -2
View File
@@ -5,10 +5,10 @@
// AUTOUPDATE
// CHECK:STDOUT: - filename: printing_real_literal.carbon
// CHECK:STDOUT: tokens: [
// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
2.5
// CHECK:STDOUT: { index: 1, kind: 'RealLiteral', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '2.5', value: `25*10^-1`, has_trailing_space: true },
// CHECK:STDOUT: { index: 2, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: { index: 2, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: ]
toolchain/lex/testdata/printing_token.carbon
Vendored
+2 -2
View File
@@ -5,10 +5,10 @@
// AUTOUPDATE
// CHECK:STDOUT: - filename: printing_token.carbon
// CHECK:STDOUT: tokens: [
// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
0x9
// CHECK:STDOUT: { index: 1, kind: 'IntegerLiteral', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '0x9', value: `9`, has_trailing_space: true },
// CHECK:STDOUT: { index: 2, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: { index: 2, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: ]
toolchain/lex/testdata/raw_identifier.carbon
Vendored
+2 -2
View File
@@ -5,7 +5,7 @@
// AUTOUPDATE
// CHECK:STDOUT: - filename: raw_identifier.carbon
// CHECK:STDOUT: tokens: [
// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true },
// A non-keyword identifier.
r#foo
@@ -31,5 +31,5 @@ r#i32
i32
// CHECK:STDOUT: { index: 6, kind: 'IntegerTypeLiteral', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'i32', has_trailing_space: true },
// CHECK:STDOUT: { index: 7, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: { index: 7, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' },
// CHECK:STDOUT: ]
toolchain/lex/token_kind.def
+2 -2
View File
@@ -213,8 +213,8 @@ CARBON_TOKEN(StringLiteral)
CARBON_TOKEN(IntegerTypeLiteral)
CARBON_TOKEN(UnsignedIntegerTypeLiteral)
CARBON_TOKEN(FloatingPointTypeLiteral)
CARBON_TOKEN(StartOfFile)
CARBON_TOKEN(EndOfFile)
CARBON_TOKEN(FileStart)
CARBON_TOKEN(FileEnd)
#undef CARBON_TOKEN
#undef CARBON_TOKEN_WITH_VIRTUAL_NODE
toolchain/lex/tokenized_buffer.cpp
+2 -2
View File
@@ -81,8 +81,8 @@ auto TokenizedBuffer::GetTokenText(Token token) const -> llvm::StringRef {
return llvm::StringRef(suffix.data() - 1, suffix.size() + 1);
}
if (token_info.kind == TokenKind::StartOfFile ||
token_info.kind == TokenKind::EndOfFile) {
if (token_info.kind == TokenKind::FileStart ||
token_info.kind == TokenKind::FileEnd) {
return llvm::StringRef();
}
toolchain/lex/tokenized_buffer.h
+1 -1
View File
@@ -41,7 +41,7 @@ class TokenizedBuffer;
// All other APIs to query a `Token` are on the `TokenizedBuffer`.
struct Token : public ComparableIndexBase {
static const Token Invalid;
// Comments aren't tokenized, so this is the first token after StartOfFile.
// Comments aren't tokenized, so this is the first token after FileStart.
static const Token FirstNonCommentToken;
using ComparableIndexBase::ComparableIndexBase;
};
toolchain/lex/tokenized_buffer_test.cpp
+72 -73
View File
@@ -60,7 +60,7 @@ TEST_F(LexerTest, HandlesEmptyBuffer) {
auto buffer = Lex("");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile}, {TokenKind::EndOfFile}}));
{TokenKind::FileStart}, {TokenKind::FileEnd}}));
}
TEST_F(LexerTest, TracksLinesAndColumns) {
@@ -69,7 +69,7 @@ TEST_F(LexerTest, TracksLinesAndColumns) {
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::StartOfFile,
{.kind = TokenKind::FileStart,
.line = 1,
.column = 1,
.indent_column = 1},
@@ -96,7 +96,7 @@ TEST_F(LexerTest, TracksLinesAndColumns) {
.column = 6,
.indent_column = 11,
.text = "y"},
{.kind = TokenKind::EndOfFile, .line = 6, .column = 7},
{.kind = TokenKind::FileEnd, .line = 6, .column = 7},
}));
}
@@ -105,7 +105,7 @@ TEST_F(LexerTest, HandlesNumericLiteral) {
EXPECT_FALSE(buffer.has_errors());
ASSERT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::StartOfFile, .line = 1, .column = 1},
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::IntegerLiteral,
.line = 1,
.column = 1,
@@ -150,7 +150,7 @@ TEST_F(LexerTest, HandlesNumericLiteral) {
.column = 1,
.indent_column = 1,
.text = "1.5e9"},
{.kind = TokenKind::EndOfFile, .line = 6, .column = 6},
{.kind = TokenKind::FileEnd, .line = 6, .column = 6},
}));
auto token_start = buffer.tokens().begin();
auto token_12 = token_start + 1;
@@ -190,7 +190,7 @@ TEST_F(LexerTest, HandlesInvalidNumericLiterals) {
EXPECT_TRUE(buffer.has_errors());
ASSERT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::StartOfFile, .line = 1, .column = 1},
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::Error,
.line = 1,
.column = 1,
@@ -216,7 +216,7 @@ TEST_F(LexerTest, HandlesInvalidNumericLiterals) {
.column = 28,
.indent_column = 1,
.text = "0ops"},
{.kind = TokenKind::EndOfFile, .line = 1, .column = 32},
{.kind = TokenKind::FileEnd, .line = 1, .column = 32},
}));
}
@@ -239,7 +239,7 @@ TEST_F(LexerTest, SplitsNumericLiteralsProperly) {
auto buffer = Lex(source_text);
EXPECT_TRUE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::StartOfFile},
{.kind = TokenKind::FileStart},
{.kind = TokenKind::IntegerLiteral, .text = "1"},
{.kind = TokenKind::Period},
// newline
@@ -288,7 +288,7 @@ TEST_F(LexerTest, SplitsNumericLiteralsProperly) {
{.kind = TokenKind::Period},
{.kind = TokenKind::Underscore},
// newline
{.kind = TokenKind::EndOfFile},
{.kind = TokenKind::FileEnd},
}));
}
@@ -299,7 +299,7 @@ TEST_F(LexerTest, HandlesGarbageCharacters) {
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::StartOfFile, .line = 1, .column = 1},
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::Error,
.line = 1,
.column = 1,
@@ -322,7 +322,7 @@ TEST_F(LexerTest, HandlesGarbageCharacters) {
{.kind = TokenKind::Error, .line = 3, .column = 2, .text = "\"\\"},
// newline
{.kind = TokenKind::Error, .line = 4, .column = 1, .text = "\"x"},
{.kind = TokenKind::EndOfFile, .line = 4, .column = 3},
{.kind = TokenKind::FileEnd, .line = 4, .column = 3},
}));
}
@@ -333,35 +333,35 @@ TEST_F(LexerTest, Symbols) {
auto buffer = Lex("<<<");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{TokenKind::LessLess},
{TokenKind::Less},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
buffer = Lex("<<=>>");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{TokenKind::LessLessEqual},
{TokenKind::GreaterGreater},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
buffer = Lex("< <=> >");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{TokenKind::Less},
{TokenKind::LessEqualGreater},
{TokenKind::Greater},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
buffer = Lex("\\/?@&^!");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{TokenKind::Backslash},
{TokenKind::Slash},
{TokenKind::Question},
@@ -369,7 +369,7 @@ TEST_F(LexerTest, Symbols) {
{TokenKind::Amp},
{TokenKind::Caret},
{TokenKind::Exclaim},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
}
@@ -377,16 +377,16 @@ TEST_F(LexerTest, Parens) {
auto buffer = Lex("()");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{TokenKind::OpenParen},
{TokenKind::CloseParen},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
buffer = Lex("((()()))");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{TokenKind::OpenParen},
{TokenKind::OpenParen},
{TokenKind::OpenParen},
@@ -395,7 +395,7 @@ TEST_F(LexerTest, Parens) {
{TokenKind::CloseParen},
{TokenKind::CloseParen},
{TokenKind::CloseParen},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
}
@@ -403,16 +403,16 @@ TEST_F(LexerTest, CurlyBraces) {
auto buffer = Lex("{}");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{TokenKind::OpenCurlyBrace},
{TokenKind::CloseCurlyBrace},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
buffer = Lex("{{{}{}}}");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{TokenKind::OpenCurlyBrace},
{TokenKind::OpenCurlyBrace},
{TokenKind::OpenCurlyBrace},
@@ -421,7 +421,7 @@ TEST_F(LexerTest, CurlyBraces) {
{TokenKind::CloseCurlyBrace},
{TokenKind::CloseCurlyBrace},
{TokenKind::CloseCurlyBrace},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
}
@@ -443,7 +443,7 @@ TEST_F(LexerTest, MatchingGroups) {
EXPECT_EQ(open_curly_token,
buffer.GetMatchedOpeningToken(close_curly_token));
auto eof_token = *it++;
EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::EndOfFile);
EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::FileEnd);
EXPECT_EQ(buffer.tokens().end(), it);
}
@@ -510,7 +510,7 @@ TEST_F(LexerTest, MatchingGroups) {
buffer.GetMatchedOpeningToken(inner_close_paren_token));
auto eof_token = *it++;
EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::EndOfFile);
EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::FileEnd);
EXPECT_EQ(buffer.tokens().end(), it);
}
}
@@ -520,18 +520,18 @@ TEST_F(LexerTest, MismatchedGroups) {
EXPECT_TRUE(buffer.has_errors());
EXPECT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{TokenKind::OpenCurlyBrace},
{.kind = TokenKind::CloseCurlyBrace, .recovery = true},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
buffer = Lex("}");
EXPECT_TRUE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{.kind = TokenKind::Error, .text = "}"},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
buffer = Lex("{(}");
@@ -539,12 +539,12 @@ TEST_F(LexerTest, MismatchedGroups) {
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{.kind = TokenKind::OpenCurlyBrace, .column = 1},
{.kind = TokenKind::OpenParen, .column = 2},
{.kind = TokenKind::CloseParen, .column = 3, .recovery = true},
{.kind = TokenKind::CloseCurlyBrace, .column = 3},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
buffer = Lex(")({)");
@@ -552,13 +552,13 @@ TEST_F(LexerTest, MismatchedGroups) {
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{.kind = TokenKind::Error, .column = 1, .text = ")"},
{.kind = TokenKind::OpenParen, .column = 2},
{.kind = TokenKind::OpenCurlyBrace, .column = 3},
{.kind = TokenKind::CloseCurlyBrace, .column = 4, .recovery = true},
{.kind = TokenKind::CloseParen, .column = 4},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
}
@@ -606,15 +606,15 @@ TEST_F(LexerTest, Keywords) {
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{.kind = TokenKind::Fn, .column = 4, .indent_column = 4},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
buffer = Lex("and or not if else for return var break continue _");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{TokenKind::And},
{TokenKind::Or},
{TokenKind::Not},
@@ -626,7 +626,7 @@ TEST_F(LexerTest, Keywords) {
{TokenKind::Break},
{TokenKind::Continue},
{TokenKind::Underscore},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
}
@@ -636,28 +636,28 @@ TEST_F(LexerTest, Comments) {
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::StartOfFile, .line = 1, .column = 1},
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::Semi, .line = 1, .column = 2, .indent_column = 2},
{.kind = TokenKind::Semi, .line = 3, .column = 3, .indent_column = 3},
{.kind = TokenKind::EndOfFile, .line = 3, .column = 4},
{.kind = TokenKind::FileEnd, .line = 3, .column = 4},
}));
buffer = Lex("// foo\n//\n// bar");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile}, {TokenKind::EndOfFile}}));
{TokenKind::FileStart}, {TokenKind::FileEnd}}));
// Make sure weird characters aren't a problem.
buffer = Lex(" // foo#$!^?@-_💩🍫⃠ [̲̅$̲̅(̲̅ ͡° ͜ʖ ͡°̲̅)̲̅$̲̅]");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile}, {TokenKind::EndOfFile}}));
{TokenKind::FileStart}, {TokenKind::FileEnd}}));
// Make sure we can lex a comment at the end of the input.
buffer = Lex("//");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile}, {TokenKind::EndOfFile}}));
{TokenKind::FileStart}, {TokenKind::FileEnd}}));
}
TEST_F(LexerTest, InvalidComments) {
@@ -677,38 +677,38 @@ TEST_F(LexerTest, Identifiers) {
auto buffer = Lex(" foobar");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{.kind = TokenKind::Identifier,
.column = 4,
.indent_column = 4,
.text = "foobar"},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
// Check different kinds of identifier character sequences.
buffer = Lex("_foo_bar");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{.kind = TokenKind::Identifier, .text = "_foo_bar"},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
buffer = Lex("foo2bar00");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{.kind = TokenKind::Identifier, .text = "foo2bar00"},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
// Check that we can parse identifiers that start with a keyword.
buffer = Lex("fnord");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::StartOfFile},
{TokenKind::FileStart},
{.kind = TokenKind::Identifier, .text = "fnord"},
{TokenKind::EndOfFile},
{TokenKind::FileEnd},
}));
// Check multiple identifiers with indent and interning.
@@ -716,7 +716,7 @@ TEST_F(LexerTest, Identifiers) {
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::StartOfFile, .line = 1, .column = 1},
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::Identifier,
.line = 1,
.column = 4,
@@ -743,7 +743,7 @@ TEST_F(LexerTest, Identifiers) {
.column = 7,
.indent_column = 3,
.text = "foo"},
{.kind = TokenKind::EndOfFile, .line = 3, .column = 10},
{.kind = TokenKind::FileEnd, .line = 3, .column = 10},
}));
}
@@ -769,7 +769,7 @@ TEST_F(LexerTest, StringLiterals) {
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::StartOfFile, .line = 1, .column = 1},
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::StringLiteral,
.line = 2,
.column = 5,
@@ -826,7 +826,7 @@ TEST_F(LexerTest, StringLiterals) {
.indent_column = 5,
.value_stores = &value_stores_,
.string_contents = {""}},
{.kind = TokenKind::EndOfFile, .line = 16, .column = 3},
{.kind = TokenKind::FileEnd, .line = 16, .column = 3},
}));
}
@@ -878,7 +878,7 @@ TEST_F(LexerTest, TypeLiterals) {
EXPECT_FALSE(buffer.has_errors());
ASSERT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::StartOfFile, .line = 1, .column = 1},
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::Identifier,
.line = 2,
@@ -954,7 +954,7 @@ TEST_F(LexerTest, TypeLiterals) {
.indent_column = 5,
.text = {"s1"}},
{.kind = TokenKind::EndOfFile, .line = 6, .column = 3},
{.kind = TokenKind::FileEnd, .line = 6, .column = 3},
}));
auto token_i1 = buffer.tokens().begin() + 2;
@@ -987,17 +987,16 @@ TEST_F(LexerTest, TypeLiteralTooManyDigits) {
HasSubstr(llvm::formatv(" {0} ", Count)))));
auto buffer = Lex(code, consumer);
EXPECT_TRUE(buffer.has_errors());
ASSERT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::StartOfFile, .line = 1, .column = 1},
{.kind = TokenKind::Error,
.line = 1,
.column = 1,
.indent_column = 1,
.text = {code}},
{.kind = TokenKind::EndOfFile, .line = 1, .column = Count + 2},
}));
ASSERT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::Error,
.line = 1,
.column = 1,
.indent_column = 1,
.text = {code}},
{.kind = TokenKind::FileEnd, .line = 1, .column = Count + 2},
}));
}
TEST_F(LexerTest, DiagnosticTrailingComment) {
@@ -1075,7 +1074,7 @@ TEST_F(LexerTest, PrintingOutputYaml) {
Pair("tokens",
Yaml::Sequence(ElementsAre(
Yaml::Mapping(ElementsAre(
Pair("index", "0"), Pair("kind", "StartOfFile"),
Pair("index", "0"), Pair("kind", "FileStart"),
Pair("line", "1"), Pair("column", "1"),
Pair("indent", "1"), Pair("spelling", ""),
Pair("has_trailing_space", "true"))),
@@ -1095,7 +1094,7 @@ TEST_F(LexerTest, PrintingOutputYaml) {
Pair("indent", "1"), Pair("spelling", ";"),
Pair("has_trailing_space", "true"))),
Yaml::Mapping(ElementsAre(
Pair("index", "4"), Pair("kind", "EndOfFile"),
Pair("index", "4"), Pair("kind", "FileEnd"),
Pair("line", "15"), Pair("column", "1"),
Pair("indent", "1"), Pair("spelling", "")))))))))))));
}
toolchain/parse/context.cpp
+3 -3
View File
@@ -51,8 +51,8 @@ Context::Context(Tree& tree, Lex::TokenizedBuffer& tokens,
end_(tokens_->tokens().end()) {
CARBON_CHECK(position_ != end_) << "Empty TokenizedBuffer";
--end_;
CARBON_CHECK(tokens_->GetKind(*end_) == Lex::TokenKind::EndOfFile)
<< "TokenizedBuffer should end with EndOfFile, ended with "
CARBON_CHECK(tokens_->GetKind(*end_) == Lex::TokenKind::FileEnd)
<< "TokenizedBuffer should end with FileEnd, ended with "
<< tokens_->GetKind(*end_);
}
@@ -160,7 +160,7 @@ auto Context::FindNextOf(std::initializer_list<Lex::TokenKind> desired_kinds)
}
// Step to the next token at the current bracketing level.
if (kind.is_closing_symbol() || kind == Lex::TokenKind::EndOfFile) {
if (kind.is_closing_symbol() || kind == Lex::TokenKind::FileEnd) {
// There are no more tokens at this level.
return std::nullopt;
} else if (kind.is_opening_symbol()) {
toolchain/parse/context.h
+3 -3
View File
@@ -50,7 +50,7 @@ class Context {
// Used for restricting ordering of `package` and `import` directives.
enum class PackagingState : int8_t {
StartOfFile,
FileStart,
InImports,
AfterNonPackagingDecl,
// A warning about `import` placement has been issued so we don't keep
@@ -359,13 +359,13 @@ class Context {
// The current position within the token buffer.
Lex::TokenIterator position_;
// The EndOfFile token.
// The FileEnd token.
Lex::TokenIterator end_;
llvm::SmallVector<StateStackEntry> state_stack_;
// The current packaging state, whether `import`/`package` are allowed.
PackagingState packaging_state_ = PackagingState::StartOfFile;
PackagingState packaging_state_ = PackagingState::FileStart;
// The first non-packaging token, starting as invalid. Used for packaging
// state warnings.
Lex::Token first_non_packaging_token_ = Lex::Token::Invalid;
toolchain/parse/handle_decl_name_and_params.cpp
+1 -1
View File
@@ -28,7 +28,7 @@ static auto HandleDeclNameAndParams(Context& context, State after_name)
"`{0}` introducer should be followed by a name.",
Lex::TokenKind);
Lex::Token location = *context.position();
if (context.tokens().GetKind(location) == Lex::TokenKind::EndOfFile) {
if (context.tokens().GetKind(location) == Lex::TokenKind::FileEnd) {
// The end of file is often an especially unhelpful location. If that's
// the best we can do here, back up the location to the introducer itself.
location = state.token;
toolchain/parse/handle_decl_scope_loop.cpp
+1 -1
View File
@@ -25,7 +25,7 @@ auto HandleDeclScopeLoop(Context& context) -> void {
auto position_kind = context.PositionKind();
switch (position_kind) {
case Lex::TokenKind::CloseCurlyBrace:
case Lex::TokenKind::EndOfFile: {
case Lex::TokenKind::FileEnd: {
// This is the end of the scope, so the loop state ends.
context.PopAndDiscardState();
return;
toolchain/parse/handle_import_and_package.cpp
+1 -1
View File
@@ -167,7 +167,7 @@ auto HandleImport(Context& context) -> void {
context.AddLeafNode(NodeKind::ImportIntroducer, intro_token);
switch (context.packaging_state()) {
case Context::PackagingState::StartOfFile:
case Context::PackagingState::FileStart:
// `package` is no longer allowed, but `import` may repeat.
context.set_packaging_state(Context::PackagingState::InImports);
[[clang::fallthrough]];
toolchain/parse/node_kind.def
+2 -2
View File
@@ -61,10 +61,10 @@
#endif
// The start of the file.
CARBON_PARSE_NODE_KIND_CHILD_COUNT(FileStart, 0, CARBON_TOKEN(StartOfFile))
CARBON_PARSE_NODE_KIND_CHILD_COUNT(FileStart, 0, CARBON_TOKEN(FileStart))
// The end of the file.
CARBON_PARSE_NODE_KIND_CHILD_COUNT(FileEnd, 0, CARBON_TOKEN(EndOfFile))
CARBON_PARSE_NODE_KIND_CHILD_COUNT(FileEnd, 0, CARBON_TOKEN(FileEnd))
// An invalid parse. Used to balance the parse tree. Always has an error.
CARBON_PARSE_NODE_KIND_CHILD_COUNT(InvalidParse, 0,
toolchain/parse/tree.cpp
+1 -1
View File
@@ -27,7 +27,7 @@ auto Tree::Parse(Lex::TokenizedBuffer& tokens, DiagnosticConsumer& consumer,
[&](llvm::raw_ostream& output) { context.PrintForStackDump(output); });
context.AddLeafNode(NodeKind::FileStart,
context.ConsumeChecked(Lex::TokenKind::StartOfFile));
context.ConsumeChecked(Lex::TokenKind::FileStart));
context.PushState(State::DeclScopeLoop);
toolchain/testing/file_test.cpp
+4 -4
View File
@@ -76,12 +76,12 @@ class ToolchainFileTest : public FileTestBase {
check_line = "// CHECK:STDOUT: {{.*}}";
}
} else if (component_ == "lex") {
// Both StartOfFile and EndOfFile regularly have locations on CHECK
// Both FileStart and FileEnd regularly have locations on CHECK
// comment lines that don't work correctly. The line happens to be correct
// for the EndOfFile, but we need to avoid checking the column.
// The column happens to be right for StartOfFile, but the line is wrong.
// for the FileEnd, but we need to avoid checking the column.
// The column happens to be right for FileStart, but the line is wrong.
static RE2 file_token_re(
R"((EndOfFile.*column: |StartOfFile.*line: )( *\d+))");
R"((FileEnd.*column: |FileStart.*line: )( *\d+))");
RE2::Replace(&check_line, file_token_re, R"(\1{{ *\\d+}})");
} else {
return FileTestBase::DoExtraCheckReplacements(check_line);