diff --git a/toolchain/lex/lex.cpp b/toolchain/lex/lex.cpp index abdf03103360..00ab17af51e4 100644 --- a/toolchain/lex/lex.cpp +++ b/toolchain/lex/lex.cpp @@ -157,9 +157,9 @@ class [[clang::internal_linkage]] Lexer { auto LexError(llvm::StringRef source_text, ssize_t& position) -> LexResult; - auto LexStartOfFile(llvm::StringRef source_text, ssize_t& position) -> void; + auto LexFileStart(llvm::StringRef source_text, ssize_t& position) -> void; - auto LexEndOfFile(llvm::StringRef source_text, ssize_t position) -> void; + auto LexFileEnd(llvm::StringRef source_text, ssize_t position) -> void; // The main entry point for dispatching through the lexer's table. This method // should always fully consume the source text. @@ -647,7 +647,7 @@ static auto DispatchNext(Lexer& lexer, llvm::StringRef source_text, // When we finish the source text, stop recursing. We also hint this so that // the tail-dispatch is optimized as that's essentially the loop back-edge // and this is the loop exit. - lexer.LexEndOfFile(source_text, position); + lexer.LexFileEnd(source_text, position); } auto Lexer::Lex() && -> TokenizedBuffer { @@ -657,7 +657,7 @@ auto Lexer::Lex() && -> TokenizedBuffer { CreateLines(source_text); ssize_t position = 0; - LexStartOfFile(source_text, position); + LexFileStart(source_text, position); // Manually enter the dispatch loop. This call will tail-recurse through the // dispatch table until everything from source_text is consumed. @@ -1263,12 +1263,12 @@ auto Lexer::LexError(llvm::StringRef source_text, ssize_t& position) return token; } -auto Lexer::LexStartOfFile(llvm::StringRef source_text, ssize_t& position) +auto Lexer::LexFileStart(llvm::StringRef source_text, ssize_t& position) -> void { // Before lexing any source text, add the start-of-file token so that code // can assume a non-empty token buffer for the rest of lexing. Note that the // start-of-file always has trailing space because it *is* whitespace. - buffer_.AddToken({.kind = TokenKind::StartOfFile, + buffer_.AddToken({.kind = TokenKind::FileStart, .has_trailing_space = true, .token_line = current_line(), .column = 0}); @@ -1281,8 +1281,7 @@ auto Lexer::LexStartOfFile(llvm::StringRef source_text, ssize_t& position) line_info->indent = position; } -auto Lexer::LexEndOfFile(llvm::StringRef source_text, ssize_t position) - -> void { +auto Lexer::LexFileEnd(llvm::StringRef source_text, ssize_t position) -> void { CARBON_CHECK(position == static_cast(source_text.size())); // Check if the last line is empty and not the first line (and only). If so, // re-pin the last line to be the prior one so that diagnostics and editors @@ -1307,7 +1306,7 @@ auto Lexer::LexEndOfFile(llvm::StringRef source_text, ssize_t position) CloseInvalidOpenGroups(TokenKind::Error, position); } - buffer_.AddToken({.kind = TokenKind::EndOfFile, + buffer_.AddToken({.kind = TokenKind::FileEnd, .token_line = current_line(), .column = ComputeColumn(position)}); } diff --git a/toolchain/lex/testdata/basic_syntax.carbon b/toolchain/lex/testdata/basic_syntax.carbon index d77e5d5bbf07..c208c454a3d9 100644 --- a/toolchain/lex/testdata/basic_syntax.carbon +++ b/toolchain/lex/testdata/basic_syntax.carbon @@ -5,7 +5,7 @@ // AUTOUPDATE // CHECK:STDOUT: - filename: basic_syntax.carbon // CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, +// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, fn run(String program) { // CHECK:STDOUT: { index: 1, kind: 'Fn', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'fn', has_trailing_space: true }, @@ -22,5 +22,5 @@ fn run(String program) { } // CHECK:STDOUT: { index: 11, kind: 'CloseCurlyBrace', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '}', opening_token: 7, has_trailing_space: true }, -// CHECK:STDOUT: { index: 12, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, +// CHECK:STDOUT: { index: 12, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, // CHECK:STDOUT: ] diff --git a/toolchain/lex/testdata/fail_bad_comment_introducers.carbon b/toolchain/lex/testdata/fail_bad_comment_introducers.carbon index 775205f244e0..d7828fa05367 100644 --- a/toolchain/lex/testdata/fail_bad_comment_introducers.carbon +++ b/toolchain/lex/testdata/fail_bad_comment_introducers.carbon @@ -5,7 +5,7 @@ // AUTOUPDATE // CHECK:STDOUT: - filename: fail_bad_comment_introducers.carbon // CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, +// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, // // Comments have to have whitespace after `//` currently. @@ -50,5 +50,5 @@ // An extra un-indented comment line to anchor the end of the file checks. -// CHECK:STDOUT: { index: 1, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, +// CHECK:STDOUT: { index: 1, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, // CHECK:STDOUT: ] diff --git a/toolchain/lex/testdata/fail_bad_comment_introducers_mid_block_indent_change.carbon b/toolchain/lex/testdata/fail_bad_comment_introducers_mid_block_indent_change.carbon index 8a462dbfd38b..1bff0092c34b 100644 --- a/toolchain/lex/testdata/fail_bad_comment_introducers_mid_block_indent_change.carbon +++ b/toolchain/lex/testdata/fail_bad_comment_introducers_mid_block_indent_change.carbon @@ -30,7 +30,7 @@ // CHECK:STDERR: ^ // CHECK:STDOUT: - filename: fail_bad_comment_introducers_mid_block_indent_change.carbon // CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, +// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, -// CHECK:STDOUT: { index: 1, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, +// CHECK:STDOUT: { index: 1, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, // CHECK:STDOUT: ] diff --git a/toolchain/lex/testdata/fail_bad_raw_identifier.carbon b/toolchain/lex/testdata/fail_bad_raw_identifier.carbon index 2d6693ce25d9..f9030e1759ee 100644 --- a/toolchain/lex/testdata/fail_bad_raw_identifier.carbon +++ b/toolchain/lex/testdata/fail_bad_raw_identifier.carbon @@ -5,7 +5,7 @@ // AUTOUPDATE // CHECK:STDOUT: - filename: fail_bad_raw_identifier.carbon // CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, +// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, // Missing the character after `#`. // CHECK:STDERR: fail_bad_raw_identifier.carbon:[[@LINE+3]]:2: ERROR: Encountered unrecognized characters while parsing. @@ -32,5 +32,5 @@ r#á // CHECK:STDOUT: { index: 6, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'r', identifier: 0 }, // CHECK:STDOUT: { index: 7, kind: 'Error', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: '#á', has_trailing_space: true }, -// CHECK:STDOUT: { index: 8, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, +// CHECK:STDOUT: { index: 8, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, // CHECK:STDOUT: ] diff --git a/toolchain/lex/testdata/fail_block_string_second_line.carbon b/toolchain/lex/testdata/fail_block_string_second_line.carbon index 3f401b133aa7..6ac9af740470 100644 --- a/toolchain/lex/testdata/fail_block_string_second_line.carbon +++ b/toolchain/lex/testdata/fail_block_string_second_line.carbon @@ -12,7 +12,7 @@ // CHECK:STDERR: ^ var s: String = ''' error here: ''' -// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: 1, column: 1, indent: 1, spelling: '', has_trailing_space: true }, +// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: 1, column: 1, indent: 1, spelling: '', has_trailing_space: true }, // CHECK:STDOUT: { index: 1, kind: 'Var', line: {{ *}}[[@LINE-3]], column: 1, indent: 1, spelling: 'var', has_trailing_space: true }, // CHECK:STDOUT: { index: 2, kind: 'Identifier', line: {{ *}}[[@LINE-4]], column: 5, indent: 1, spelling: 's', identifier: 0 }, // CHECK:STDOUT: { index: 3, kind: 'Colon', line: {{ *}}[[@LINE-5]], column: 6, indent: 1, spelling: ':', has_trailing_space: true }, @@ -21,5 +21,5 @@ var s: String = ''' // CHECK:STDOUT: { index: 6, kind: 'StringLiteral', line: {{ *}}[[@LINE-8]], column: 17, indent: 1, spelling: '''' // CHECK:STDOUT: error here: '''', value: `error here: `, has_trailing_space: true }, -// CHECK:STDOUT: { index: 7, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, +// CHECK:STDOUT: { index: 7, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, // CHECK:STDOUT: ] diff --git a/toolchain/lex/testdata/fail_multifile.carbon b/toolchain/lex/testdata/fail_multifile.carbon index c8ef886ee5a2..4120d9998df1 100644 --- a/toolchain/lex/testdata/fail_multifile.carbon +++ b/toolchain/lex/testdata/fail_multifile.carbon @@ -7,24 +7,24 @@ // --- a.carbon // CHECK:STDOUT: - filename: a.carbon // CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, +// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, // CHECK:STDERR: a.carbon:[[@LINE+3]]:3: ERROR: Empty digit sequence in numeric literal. // CHECK:STDERR: 1.a // CHECK:STDERR: ^ 1.a -// CHECK:STDOUT: { index: 1, kind: 'Error', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '1.a', has_trailing_space: true }, +// CHECK:STDOUT: { index: 1, kind: 'Error', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '1.a', has_trailing_space: true }, -// CHECK:STDOUT: { index: 2, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, +// CHECK:STDOUT: { index: 2, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, // CHECK:STDOUT: ] // --- b.carbon // CHECK:STDOUT: - filename: b.carbon // CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, +// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, // CHECK:STDERR: b.carbon:[[@LINE+3]]:3: ERROR: Empty digit sequence in numeric literal. // CHECK:STDERR: 2.b // CHECK:STDERR: ^ 2.b -// CHECK:STDOUT: { index: 1, kind: 'Error', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '2.b', has_trailing_space: true }, +// CHECK:STDOUT: { index: 1, kind: 'Error', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '2.b', has_trailing_space: true }, -// CHECK:STDOUT: { index: 2, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, +// CHECK:STDOUT: { index: 2, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, // CHECK:STDOUT: ] diff --git a/toolchain/lex/testdata/fail_trailing_comments.carbon b/toolchain/lex/testdata/fail_trailing_comments.carbon index 825d393f5dc8..8df4fb73e0c9 100644 --- a/toolchain/lex/testdata/fail_trailing_comments.carbon +++ b/toolchain/lex/testdata/fail_trailing_comments.carbon @@ -23,7 +23,7 @@ var c: i32 = 0.4; // still more trailing comment // CHECK:STDERR: ^ // CHECK:STDOUT: - filename: fail_trailing_comments.carbon // CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, +// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, // CHECK:STDOUT: { index: 1, kind: 'Var', line: {{ *}}[[@LINE-19]], column: 1, indent: 1, spelling: 'var', has_trailing_space: true }, // CHECK:STDOUT: { index: 2, kind: 'Identifier', line: {{ *}}[[@LINE-20]], column: 5, indent: 1, spelling: 'a', identifier: 0 }, // CHECK:STDOUT: { index: 3, kind: 'Colon', line: {{ *}}[[@LINE-21]], column: 6, indent: 1, spelling: ':', has_trailing_space: true }, @@ -46,5 +46,5 @@ var c: i32 = 0.4; // still more trailing comment // CHECK:STDOUT: { index: 20, kind: 'RealLiteral', line: {{ *}}[[@LINE-36]], column: 14, indent: 1, spelling: '0.4', value: `4*10^-1` }, // CHECK:STDOUT: { index: 21, kind: 'Semi', line: {{ *}}[[@LINE-37]], column: 17, indent: 1, spelling: ';', has_trailing_space: true }, -// CHECK:STDOUT: { index: 22, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, +// CHECK:STDOUT: { index: 22, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, // CHECK:STDOUT: ] diff --git a/toolchain/lex/testdata/keywords.carbon b/toolchain/lex/testdata/keywords.carbon index b7be0e9f5e84..d96762cf674e 100644 --- a/toolchain/lex/testdata/keywords.carbon +++ b/toolchain/lex/testdata/keywords.carbon @@ -5,7 +5,7 @@ // AUTOUPDATE // CHECK:STDOUT: - filename: keywords.carbon // CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, +// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, abstract // CHECK:STDOUT: { index: 1, kind: 'Abstract', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'abstract', has_trailing_space: true }, @@ -133,5 +133,5 @@ while notakeyword // CHECK:STDOUT: { index: 62, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'notakeyword', identifier: 0, has_trailing_space: true }, -// CHECK:STDOUT: { index: 63, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, +// CHECK:STDOUT: { index: 63, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, // CHECK:STDOUT: ] diff --git a/toolchain/lex/testdata/multifile.carbon b/toolchain/lex/testdata/multifile.carbon index a6a0eb53b882..42745b56b789 100644 --- a/toolchain/lex/testdata/multifile.carbon +++ b/toolchain/lex/testdata/multifile.carbon @@ -7,23 +7,23 @@ // --- a.carbon // CHECK:STDOUT: - filename: a.carbon // CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, +// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, a; -// CHECK:STDOUT: { index: 1, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'a', identifier: 0 }, -// CHECK:STDOUT: { index: 2, kind: 'Semi', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: ';', has_trailing_space: true }, +// CHECK:STDOUT: { index: 1, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'a', identifier: 0 }, +// CHECK:STDOUT: { index: 2, kind: 'Semi', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: ';', has_trailing_space: true }, -// CHECK:STDOUT: { index: 3, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, +// CHECK:STDOUT: { index: 3, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, // CHECK:STDOUT: ] // --- b.carbon // CHECK:STDOUT: - filename: b.carbon // CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, +// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, b; -// CHECK:STDOUT: { index: 1, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'b', identifier: 0 }, -// CHECK:STDOUT: { index: 2, kind: 'Semi', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: ';', has_trailing_space: true }, +// CHECK:STDOUT: { index: 1, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'b', identifier: 0 }, +// CHECK:STDOUT: { index: 2, kind: 'Semi', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: ';', has_trailing_space: true }, a; -// CHECK:STDOUT: { index: 3, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'a', identifier: 1 }, -// CHECK:STDOUT: { index: 4, kind: 'Semi', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: ';', has_trailing_space: true }, +// CHECK:STDOUT: { index: 3, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'a', identifier: 1 }, +// CHECK:STDOUT: { index: 4, kind: 'Semi', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: ';', has_trailing_space: true }, -// CHECK:STDOUT: { index: 5, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, +// CHECK:STDOUT: { index: 5, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, // CHECK:STDOUT: ] diff --git a/toolchain/lex/testdata/numeric_literals.carbon b/toolchain/lex/testdata/numeric_literals.carbon index e49e28d275c0..d121c2e60b68 100644 --- a/toolchain/lex/testdata/numeric_literals.carbon +++ b/toolchain/lex/testdata/numeric_literals.carbon @@ -5,7 +5,7 @@ // AUTOUPDATE // CHECK:STDOUT: - filename: numeric_literals.carbon // CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, +// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, fn F() { // CHECK:STDOUT: { index: 1, kind: 'Fn', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'fn', has_trailing_space: true }, @@ -82,5 +82,5 @@ fn F() { } // CHECK:STDOUT: { index: 54, kind: 'CloseCurlyBrace', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '}', opening_token: 5, has_trailing_space: true }, -// CHECK:STDOUT: { index: 55, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, +// CHECK:STDOUT: { index: 55, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, // CHECK:STDOUT: ] diff --git a/toolchain/lex/testdata/printing_digit_padding.carbon b/toolchain/lex/testdata/printing_digit_padding.carbon index 8594ff9c7c6e..a7fec13d08f0 100644 --- a/toolchain/lex/testdata/printing_digit_padding.carbon +++ b/toolchain/lex/testdata/printing_digit_padding.carbon @@ -5,16 +5,16 @@ // AUTOUPDATE // CHECK:STDOUT: - filename: printing_digit_padding.carbon // CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, +// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, ; -// CHECK:STDOUT: { index: 1, kind: 'Semi', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: ';', has_trailing_space: true }, +// CHECK:STDOUT: { index: 1, kind: 'Semi', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: ';', has_trailing_space: true }, // Test digit padding with values of 9, 10, and 11. ;;; - // CHECK:STDOUT: { index: 2, kind: 'Semi', line: {{ *}}[[@LINE-1]], column: 9, indent: 9, spelling: ';' }, - // CHECK:STDOUT: { index: 3, kind: 'Semi', line: {{ *}}[[@LINE-2]], column: 10, indent: 9, spelling: ';' }, - // CHECK:STDOUT: { index: 4, kind: 'Semi', line: {{ *}}[[@LINE-3]], column: 11, indent: 9, spelling: ';', has_trailing_space: true }, + // CHECK:STDOUT: { index: 2, kind: 'Semi', line: {{ *}}[[@LINE-1]], column: 9, indent: 9, spelling: ';' }, + // CHECK:STDOUT: { index: 3, kind: 'Semi', line: {{ *}}[[@LINE-2]], column: 10, indent: 9, spelling: ';' }, + // CHECK:STDOUT: { index: 4, kind: 'Semi', line: {{ *}}[[@LINE-3]], column: 11, indent: 9, spelling: ';', has_trailing_space: true }, -// CHECK:STDOUT: { index: 5, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, +// CHECK:STDOUT: { index: 5, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, // CHECK:STDOUT: ] diff --git a/toolchain/lex/testdata/printing_integer_literal.carbon b/toolchain/lex/testdata/printing_integer_literal.carbon index 3d9b1cfb7228..9068f06487a1 100644 --- a/toolchain/lex/testdata/printing_integer_literal.carbon +++ b/toolchain/lex/testdata/printing_integer_literal.carbon @@ -5,10 +5,10 @@ // AUTOUPDATE // CHECK:STDOUT: - filename: printing_integer_literal.carbon // CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, +// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, 123 // CHECK:STDOUT: { index: 1, kind: 'IntegerLiteral', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '123', value: `123`, has_trailing_space: true }, -// CHECK:STDOUT: { index: 2, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, +// CHECK:STDOUT: { index: 2, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, // CHECK:STDOUT: ] diff --git a/toolchain/lex/testdata/printing_real_literal.carbon b/toolchain/lex/testdata/printing_real_literal.carbon index a59761771121..60aebaec149b 100644 --- a/toolchain/lex/testdata/printing_real_literal.carbon +++ b/toolchain/lex/testdata/printing_real_literal.carbon @@ -5,10 +5,10 @@ // AUTOUPDATE // CHECK:STDOUT: - filename: printing_real_literal.carbon // CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, +// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, 2.5 // CHECK:STDOUT: { index: 1, kind: 'RealLiteral', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '2.5', value: `25*10^-1`, has_trailing_space: true }, -// CHECK:STDOUT: { index: 2, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, +// CHECK:STDOUT: { index: 2, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, // CHECK:STDOUT: ] diff --git a/toolchain/lex/testdata/printing_token.carbon b/toolchain/lex/testdata/printing_token.carbon index d28e931c9c80..0b2e7a19e3de 100644 --- a/toolchain/lex/testdata/printing_token.carbon +++ b/toolchain/lex/testdata/printing_token.carbon @@ -5,10 +5,10 @@ // AUTOUPDATE // CHECK:STDOUT: - filename: printing_token.carbon // CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, +// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, 0x9 // CHECK:STDOUT: { index: 1, kind: 'IntegerLiteral', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '0x9', value: `9`, has_trailing_space: true }, -// CHECK:STDOUT: { index: 2, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, +// CHECK:STDOUT: { index: 2, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, // CHECK:STDOUT: ] diff --git a/toolchain/lex/testdata/raw_identifier.carbon b/toolchain/lex/testdata/raw_identifier.carbon index ce4fb8a59e00..a8e0597cabcd 100644 --- a/toolchain/lex/testdata/raw_identifier.carbon +++ b/toolchain/lex/testdata/raw_identifier.carbon @@ -5,7 +5,7 @@ // AUTOUPDATE // CHECK:STDOUT: - filename: raw_identifier.carbon // CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'StartOfFile', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, +// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '', has_trailing_space: true }, // A non-keyword identifier. r#foo @@ -31,5 +31,5 @@ r#i32 i32 // CHECK:STDOUT: { index: 6, kind: 'IntegerTypeLiteral', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'i32', has_trailing_space: true }, -// CHECK:STDOUT: { index: 7, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, +// CHECK:STDOUT: { index: 7, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '' }, // CHECK:STDOUT: ] diff --git a/toolchain/lex/token_kind.def b/toolchain/lex/token_kind.def index baf9faa0a142..e6c575c2b669 100644 --- a/toolchain/lex/token_kind.def +++ b/toolchain/lex/token_kind.def @@ -213,8 +213,8 @@ CARBON_TOKEN(StringLiteral) CARBON_TOKEN(IntegerTypeLiteral) CARBON_TOKEN(UnsignedIntegerTypeLiteral) CARBON_TOKEN(FloatingPointTypeLiteral) -CARBON_TOKEN(StartOfFile) -CARBON_TOKEN(EndOfFile) +CARBON_TOKEN(FileStart) +CARBON_TOKEN(FileEnd) #undef CARBON_TOKEN #undef CARBON_TOKEN_WITH_VIRTUAL_NODE diff --git a/toolchain/lex/tokenized_buffer.cpp b/toolchain/lex/tokenized_buffer.cpp index 55040b7a2e8e..d4af7a8acb0f 100644 --- a/toolchain/lex/tokenized_buffer.cpp +++ b/toolchain/lex/tokenized_buffer.cpp @@ -81,8 +81,8 @@ auto TokenizedBuffer::GetTokenText(Token token) const -> llvm::StringRef { return llvm::StringRef(suffix.data() - 1, suffix.size() + 1); } - if (token_info.kind == TokenKind::StartOfFile || - token_info.kind == TokenKind::EndOfFile) { + if (token_info.kind == TokenKind::FileStart || + token_info.kind == TokenKind::FileEnd) { return llvm::StringRef(); } diff --git a/toolchain/lex/tokenized_buffer.h b/toolchain/lex/tokenized_buffer.h index 128d8f3a0e15..019488d90cec 100644 --- a/toolchain/lex/tokenized_buffer.h +++ b/toolchain/lex/tokenized_buffer.h @@ -41,7 +41,7 @@ class TokenizedBuffer; // All other APIs to query a `Token` are on the `TokenizedBuffer`. struct Token : public ComparableIndexBase { static const Token Invalid; - // Comments aren't tokenized, so this is the first token after StartOfFile. + // Comments aren't tokenized, so this is the first token after FileStart. static const Token FirstNonCommentToken; using ComparableIndexBase::ComparableIndexBase; }; diff --git a/toolchain/lex/tokenized_buffer_test.cpp b/toolchain/lex/tokenized_buffer_test.cpp index 4aef1e457cbc..229816c4788f 100644 --- a/toolchain/lex/tokenized_buffer_test.cpp +++ b/toolchain/lex/tokenized_buffer_test.cpp @@ -60,7 +60,7 @@ TEST_F(LexerTest, HandlesEmptyBuffer) { auto buffer = Lex(""); EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::StartOfFile}, {TokenKind::EndOfFile}})); + {TokenKind::FileStart}, {TokenKind::FileEnd}})); } TEST_F(LexerTest, TracksLinesAndColumns) { @@ -69,7 +69,7 @@ TEST_F(LexerTest, TracksLinesAndColumns) { EXPECT_THAT( buffer, HasTokens(llvm::ArrayRef{ - {.kind = TokenKind::StartOfFile, + {.kind = TokenKind::FileStart, .line = 1, .column = 1, .indent_column = 1}, @@ -96,7 +96,7 @@ TEST_F(LexerTest, TracksLinesAndColumns) { .column = 6, .indent_column = 11, .text = "y"}, - {.kind = TokenKind::EndOfFile, .line = 6, .column = 7}, + {.kind = TokenKind::FileEnd, .line = 6, .column = 7}, })); } @@ -105,7 +105,7 @@ TEST_F(LexerTest, HandlesNumericLiteral) { EXPECT_FALSE(buffer.has_errors()); ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {.kind = TokenKind::StartOfFile, .line = 1, .column = 1}, + {.kind = TokenKind::FileStart, .line = 1, .column = 1}, {.kind = TokenKind::IntegerLiteral, .line = 1, .column = 1, @@ -150,7 +150,7 @@ TEST_F(LexerTest, HandlesNumericLiteral) { .column = 1, .indent_column = 1, .text = "1.5e9"}, - {.kind = TokenKind::EndOfFile, .line = 6, .column = 6}, + {.kind = TokenKind::FileEnd, .line = 6, .column = 6}, })); auto token_start = buffer.tokens().begin(); auto token_12 = token_start + 1; @@ -190,7 +190,7 @@ TEST_F(LexerTest, HandlesInvalidNumericLiterals) { EXPECT_TRUE(buffer.has_errors()); ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {.kind = TokenKind::StartOfFile, .line = 1, .column = 1}, + {.kind = TokenKind::FileStart, .line = 1, .column = 1}, {.kind = TokenKind::Error, .line = 1, .column = 1, @@ -216,7 +216,7 @@ TEST_F(LexerTest, HandlesInvalidNumericLiterals) { .column = 28, .indent_column = 1, .text = "0ops"}, - {.kind = TokenKind::EndOfFile, .line = 1, .column = 32}, + {.kind = TokenKind::FileEnd, .line = 1, .column = 32}, })); } @@ -239,7 +239,7 @@ TEST_F(LexerTest, SplitsNumericLiteralsProperly) { auto buffer = Lex(source_text); EXPECT_TRUE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {.kind = TokenKind::StartOfFile}, + {.kind = TokenKind::FileStart}, {.kind = TokenKind::IntegerLiteral, .text = "1"}, {.kind = TokenKind::Period}, // newline @@ -288,7 +288,7 @@ TEST_F(LexerTest, SplitsNumericLiteralsProperly) { {.kind = TokenKind::Period}, {.kind = TokenKind::Underscore}, // newline - {.kind = TokenKind::EndOfFile}, + {.kind = TokenKind::FileEnd}, })); } @@ -299,7 +299,7 @@ TEST_F(LexerTest, HandlesGarbageCharacters) { EXPECT_THAT( buffer, HasTokens(llvm::ArrayRef{ - {.kind = TokenKind::StartOfFile, .line = 1, .column = 1}, + {.kind = TokenKind::FileStart, .line = 1, .column = 1}, {.kind = TokenKind::Error, .line = 1, .column = 1, @@ -322,7 +322,7 @@ TEST_F(LexerTest, HandlesGarbageCharacters) { {.kind = TokenKind::Error, .line = 3, .column = 2, .text = "\"\\"}, // newline {.kind = TokenKind::Error, .line = 4, .column = 1, .text = "\"x"}, - {.kind = TokenKind::EndOfFile, .line = 4, .column = 3}, + {.kind = TokenKind::FileEnd, .line = 4, .column = 3}, })); } @@ -333,35 +333,35 @@ TEST_F(LexerTest, Symbols) { auto buffer = Lex("<<<"); EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::StartOfFile}, + {TokenKind::FileStart}, {TokenKind::LessLess}, {TokenKind::Less}, - {TokenKind::EndOfFile}, + {TokenKind::FileEnd}, })); buffer = Lex("<<=>>"); EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::StartOfFile}, + {TokenKind::FileStart}, {TokenKind::LessLessEqual}, {TokenKind::GreaterGreater}, - {TokenKind::EndOfFile}, + {TokenKind::FileEnd}, })); buffer = Lex("< <=> >"); EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::StartOfFile}, + {TokenKind::FileStart}, {TokenKind::Less}, {TokenKind::LessEqualGreater}, {TokenKind::Greater}, - {TokenKind::EndOfFile}, + {TokenKind::FileEnd}, })); buffer = Lex("\\/?@&^!"); EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::StartOfFile}, + {TokenKind::FileStart}, {TokenKind::Backslash}, {TokenKind::Slash}, {TokenKind::Question}, @@ -369,7 +369,7 @@ TEST_F(LexerTest, Symbols) { {TokenKind::Amp}, {TokenKind::Caret}, {TokenKind::Exclaim}, - {TokenKind::EndOfFile}, + {TokenKind::FileEnd}, })); } @@ -377,16 +377,16 @@ TEST_F(LexerTest, Parens) { auto buffer = Lex("()"); EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::StartOfFile}, + {TokenKind::FileStart}, {TokenKind::OpenParen}, {TokenKind::CloseParen}, - {TokenKind::EndOfFile}, + {TokenKind::FileEnd}, })); buffer = Lex("((()()))"); EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::StartOfFile}, + {TokenKind::FileStart}, {TokenKind::OpenParen}, {TokenKind::OpenParen}, {TokenKind::OpenParen}, @@ -395,7 +395,7 @@ TEST_F(LexerTest, Parens) { {TokenKind::CloseParen}, {TokenKind::CloseParen}, {TokenKind::CloseParen}, - {TokenKind::EndOfFile}, + {TokenKind::FileEnd}, })); } @@ -403,16 +403,16 @@ TEST_F(LexerTest, CurlyBraces) { auto buffer = Lex("{}"); EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::StartOfFile}, + {TokenKind::FileStart}, {TokenKind::OpenCurlyBrace}, {TokenKind::CloseCurlyBrace}, - {TokenKind::EndOfFile}, + {TokenKind::FileEnd}, })); buffer = Lex("{{{}{}}}"); EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::StartOfFile}, + {TokenKind::FileStart}, {TokenKind::OpenCurlyBrace}, {TokenKind::OpenCurlyBrace}, {TokenKind::OpenCurlyBrace}, @@ -421,7 +421,7 @@ TEST_F(LexerTest, CurlyBraces) { {TokenKind::CloseCurlyBrace}, {TokenKind::CloseCurlyBrace}, {TokenKind::CloseCurlyBrace}, - {TokenKind::EndOfFile}, + {TokenKind::FileEnd}, })); } @@ -443,7 +443,7 @@ TEST_F(LexerTest, MatchingGroups) { EXPECT_EQ(open_curly_token, buffer.GetMatchedOpeningToken(close_curly_token)); auto eof_token = *it++; - EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::EndOfFile); + EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::FileEnd); EXPECT_EQ(buffer.tokens().end(), it); } @@ -510,7 +510,7 @@ TEST_F(LexerTest, MatchingGroups) { buffer.GetMatchedOpeningToken(inner_close_paren_token)); auto eof_token = *it++; - EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::EndOfFile); + EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::FileEnd); EXPECT_EQ(buffer.tokens().end(), it); } } @@ -520,18 +520,18 @@ TEST_F(LexerTest, MismatchedGroups) { EXPECT_TRUE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::StartOfFile}, + {TokenKind::FileStart}, {TokenKind::OpenCurlyBrace}, {.kind = TokenKind::CloseCurlyBrace, .recovery = true}, - {TokenKind::EndOfFile}, + {TokenKind::FileEnd}, })); buffer = Lex("}"); EXPECT_TRUE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::StartOfFile}, + {TokenKind::FileStart}, {.kind = TokenKind::Error, .text = "}"}, - {TokenKind::EndOfFile}, + {TokenKind::FileEnd}, })); buffer = Lex("{(}"); @@ -539,12 +539,12 @@ TEST_F(LexerTest, MismatchedGroups) { EXPECT_THAT( buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::StartOfFile}, + {TokenKind::FileStart}, {.kind = TokenKind::OpenCurlyBrace, .column = 1}, {.kind = TokenKind::OpenParen, .column = 2}, {.kind = TokenKind::CloseParen, .column = 3, .recovery = true}, {.kind = TokenKind::CloseCurlyBrace, .column = 3}, - {TokenKind::EndOfFile}, + {TokenKind::FileEnd}, })); buffer = Lex(")({)"); @@ -552,13 +552,13 @@ TEST_F(LexerTest, MismatchedGroups) { EXPECT_THAT( buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::StartOfFile}, + {TokenKind::FileStart}, {.kind = TokenKind::Error, .column = 1, .text = ")"}, {.kind = TokenKind::OpenParen, .column = 2}, {.kind = TokenKind::OpenCurlyBrace, .column = 3}, {.kind = TokenKind::CloseCurlyBrace, .column = 4, .recovery = true}, {.kind = TokenKind::CloseParen, .column = 4}, - {TokenKind::EndOfFile}, + {TokenKind::FileEnd}, })); } @@ -606,15 +606,15 @@ TEST_F(LexerTest, Keywords) { EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::StartOfFile}, + {TokenKind::FileStart}, {.kind = TokenKind::Fn, .column = 4, .indent_column = 4}, - {TokenKind::EndOfFile}, + {TokenKind::FileEnd}, })); buffer = Lex("and or not if else for return var break continue _"); EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::StartOfFile}, + {TokenKind::FileStart}, {TokenKind::And}, {TokenKind::Or}, {TokenKind::Not}, @@ -626,7 +626,7 @@ TEST_F(LexerTest, Keywords) { {TokenKind::Break}, {TokenKind::Continue}, {TokenKind::Underscore}, - {TokenKind::EndOfFile}, + {TokenKind::FileEnd}, })); } @@ -636,28 +636,28 @@ TEST_F(LexerTest, Comments) { EXPECT_THAT( buffer, HasTokens(llvm::ArrayRef{ - {.kind = TokenKind::StartOfFile, .line = 1, .column = 1}, + {.kind = TokenKind::FileStart, .line = 1, .column = 1}, {.kind = TokenKind::Semi, .line = 1, .column = 2, .indent_column = 2}, {.kind = TokenKind::Semi, .line = 3, .column = 3, .indent_column = 3}, - {.kind = TokenKind::EndOfFile, .line = 3, .column = 4}, + {.kind = TokenKind::FileEnd, .line = 3, .column = 4}, })); buffer = Lex("// foo\n//\n// bar"); EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::StartOfFile}, {TokenKind::EndOfFile}})); + {TokenKind::FileStart}, {TokenKind::FileEnd}})); // Make sure weird characters aren't a problem. buffer = Lex(" // foo#$!^?@-_💩🍫⃠ [̲̅$̲̅(̲̅ ͡° ͜ʖ ͡°̲̅)̲̅$̲̅]"); EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::StartOfFile}, {TokenKind::EndOfFile}})); + {TokenKind::FileStart}, {TokenKind::FileEnd}})); // Make sure we can lex a comment at the end of the input. buffer = Lex("//"); EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::StartOfFile}, {TokenKind::EndOfFile}})); + {TokenKind::FileStart}, {TokenKind::FileEnd}})); } TEST_F(LexerTest, InvalidComments) { @@ -677,38 +677,38 @@ TEST_F(LexerTest, Identifiers) { auto buffer = Lex(" foobar"); EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::StartOfFile}, + {TokenKind::FileStart}, {.kind = TokenKind::Identifier, .column = 4, .indent_column = 4, .text = "foobar"}, - {TokenKind::EndOfFile}, + {TokenKind::FileEnd}, })); // Check different kinds of identifier character sequences. buffer = Lex("_foo_bar"); EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::StartOfFile}, + {TokenKind::FileStart}, {.kind = TokenKind::Identifier, .text = "_foo_bar"}, - {TokenKind::EndOfFile}, + {TokenKind::FileEnd}, })); buffer = Lex("foo2bar00"); EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::StartOfFile}, + {TokenKind::FileStart}, {.kind = TokenKind::Identifier, .text = "foo2bar00"}, - {TokenKind::EndOfFile}, + {TokenKind::FileEnd}, })); // Check that we can parse identifiers that start with a keyword. buffer = Lex("fnord"); EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::StartOfFile}, + {TokenKind::FileStart}, {.kind = TokenKind::Identifier, .text = "fnord"}, - {TokenKind::EndOfFile}, + {TokenKind::FileEnd}, })); // Check multiple identifiers with indent and interning. @@ -716,7 +716,7 @@ TEST_F(LexerTest, Identifiers) { EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {.kind = TokenKind::StartOfFile, .line = 1, .column = 1}, + {.kind = TokenKind::FileStart, .line = 1, .column = 1}, {.kind = TokenKind::Identifier, .line = 1, .column = 4, @@ -743,7 +743,7 @@ TEST_F(LexerTest, Identifiers) { .column = 7, .indent_column = 3, .text = "foo"}, - {.kind = TokenKind::EndOfFile, .line = 3, .column = 10}, + {.kind = TokenKind::FileEnd, .line = 3, .column = 10}, })); } @@ -769,7 +769,7 @@ TEST_F(LexerTest, StringLiterals) { EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {.kind = TokenKind::StartOfFile, .line = 1, .column = 1}, + {.kind = TokenKind::FileStart, .line = 1, .column = 1}, {.kind = TokenKind::StringLiteral, .line = 2, .column = 5, @@ -826,7 +826,7 @@ TEST_F(LexerTest, StringLiterals) { .indent_column = 5, .value_stores = &value_stores_, .string_contents = {""}}, - {.kind = TokenKind::EndOfFile, .line = 16, .column = 3}, + {.kind = TokenKind::FileEnd, .line = 16, .column = 3}, })); } @@ -878,7 +878,7 @@ TEST_F(LexerTest, TypeLiterals) { EXPECT_FALSE(buffer.has_errors()); ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {.kind = TokenKind::StartOfFile, .line = 1, .column = 1}, + {.kind = TokenKind::FileStart, .line = 1, .column = 1}, {.kind = TokenKind::Identifier, .line = 2, @@ -954,7 +954,7 @@ TEST_F(LexerTest, TypeLiterals) { .indent_column = 5, .text = {"s1"}}, - {.kind = TokenKind::EndOfFile, .line = 6, .column = 3}, + {.kind = TokenKind::FileEnd, .line = 6, .column = 3}, })); auto token_i1 = buffer.tokens().begin() + 2; @@ -987,17 +987,16 @@ TEST_F(LexerTest, TypeLiteralTooManyDigits) { HasSubstr(llvm::formatv(" {0} ", Count))))); auto buffer = Lex(code, consumer); EXPECT_TRUE(buffer.has_errors()); - ASSERT_THAT( - buffer, - HasTokens(llvm::ArrayRef{ - {.kind = TokenKind::StartOfFile, .line = 1, .column = 1}, - {.kind = TokenKind::Error, - .line = 1, - .column = 1, - .indent_column = 1, - .text = {code}}, - {.kind = TokenKind::EndOfFile, .line = 1, .column = Count + 2}, - })); + ASSERT_THAT(buffer, + HasTokens(llvm::ArrayRef{ + {.kind = TokenKind::FileStart, .line = 1, .column = 1}, + {.kind = TokenKind::Error, + .line = 1, + .column = 1, + .indent_column = 1, + .text = {code}}, + {.kind = TokenKind::FileEnd, .line = 1, .column = Count + 2}, + })); } TEST_F(LexerTest, DiagnosticTrailingComment) { @@ -1075,7 +1074,7 @@ TEST_F(LexerTest, PrintingOutputYaml) { Pair("tokens", Yaml::Sequence(ElementsAre( Yaml::Mapping(ElementsAre( - Pair("index", "0"), Pair("kind", "StartOfFile"), + Pair("index", "0"), Pair("kind", "FileStart"), Pair("line", "1"), Pair("column", "1"), Pair("indent", "1"), Pair("spelling", ""), Pair("has_trailing_space", "true"))), @@ -1095,7 +1094,7 @@ TEST_F(LexerTest, PrintingOutputYaml) { Pair("indent", "1"), Pair("spelling", ";"), Pair("has_trailing_space", "true"))), Yaml::Mapping(ElementsAre( - Pair("index", "4"), Pair("kind", "EndOfFile"), + Pair("index", "4"), Pair("kind", "FileEnd"), Pair("line", "15"), Pair("column", "1"), Pair("indent", "1"), Pair("spelling", ""))))))))))))); } diff --git a/toolchain/parse/context.cpp b/toolchain/parse/context.cpp index b1012310afec..65202fc4a9e1 100644 --- a/toolchain/parse/context.cpp +++ b/toolchain/parse/context.cpp @@ -51,8 +51,8 @@ Context::Context(Tree& tree, Lex::TokenizedBuffer& tokens, end_(tokens_->tokens().end()) { CARBON_CHECK(position_ != end_) << "Empty TokenizedBuffer"; --end_; - CARBON_CHECK(tokens_->GetKind(*end_) == Lex::TokenKind::EndOfFile) - << "TokenizedBuffer should end with EndOfFile, ended with " + CARBON_CHECK(tokens_->GetKind(*end_) == Lex::TokenKind::FileEnd) + << "TokenizedBuffer should end with FileEnd, ended with " << tokens_->GetKind(*end_); } @@ -160,7 +160,7 @@ auto Context::FindNextOf(std::initializer_list desired_kinds) } // Step to the next token at the current bracketing level. - if (kind.is_closing_symbol() || kind == Lex::TokenKind::EndOfFile) { + if (kind.is_closing_symbol() || kind == Lex::TokenKind::FileEnd) { // There are no more tokens at this level. return std::nullopt; } else if (kind.is_opening_symbol()) { diff --git a/toolchain/parse/context.h b/toolchain/parse/context.h index 6a1a8f4b8937..c6937be4511e 100644 --- a/toolchain/parse/context.h +++ b/toolchain/parse/context.h @@ -50,7 +50,7 @@ class Context { // Used for restricting ordering of `package` and `import` directives. enum class PackagingState : int8_t { - StartOfFile, + FileStart, InImports, AfterNonPackagingDecl, // A warning about `import` placement has been issued so we don't keep @@ -359,13 +359,13 @@ class Context { // The current position within the token buffer. Lex::TokenIterator position_; - // The EndOfFile token. + // The FileEnd token. Lex::TokenIterator end_; llvm::SmallVector state_stack_; // The current packaging state, whether `import`/`package` are allowed. - PackagingState packaging_state_ = PackagingState::StartOfFile; + PackagingState packaging_state_ = PackagingState::FileStart; // The first non-packaging token, starting as invalid. Used for packaging // state warnings. Lex::Token first_non_packaging_token_ = Lex::Token::Invalid; diff --git a/toolchain/parse/handle_decl_name_and_params.cpp b/toolchain/parse/handle_decl_name_and_params.cpp index 59045d6b333e..75adae066cd2 100644 --- a/toolchain/parse/handle_decl_name_and_params.cpp +++ b/toolchain/parse/handle_decl_name_and_params.cpp @@ -28,7 +28,7 @@ static auto HandleDeclNameAndParams(Context& context, State after_name) "`{0}` introducer should be followed by a name.", Lex::TokenKind); Lex::Token location = *context.position(); - if (context.tokens().GetKind(location) == Lex::TokenKind::EndOfFile) { + if (context.tokens().GetKind(location) == Lex::TokenKind::FileEnd) { // The end of file is often an especially unhelpful location. If that's // the best we can do here, back up the location to the introducer itself. location = state.token; diff --git a/toolchain/parse/handle_decl_scope_loop.cpp b/toolchain/parse/handle_decl_scope_loop.cpp index 5bcf02087395..a98dff9552b2 100644 --- a/toolchain/parse/handle_decl_scope_loop.cpp +++ b/toolchain/parse/handle_decl_scope_loop.cpp @@ -25,7 +25,7 @@ auto HandleDeclScopeLoop(Context& context) -> void { auto position_kind = context.PositionKind(); switch (position_kind) { case Lex::TokenKind::CloseCurlyBrace: - case Lex::TokenKind::EndOfFile: { + case Lex::TokenKind::FileEnd: { // This is the end of the scope, so the loop state ends. context.PopAndDiscardState(); return; diff --git a/toolchain/parse/handle_import_and_package.cpp b/toolchain/parse/handle_import_and_package.cpp index 506739d22dc3..583d07caaaca 100644 --- a/toolchain/parse/handle_import_and_package.cpp +++ b/toolchain/parse/handle_import_and_package.cpp @@ -167,7 +167,7 @@ auto HandleImport(Context& context) -> void { context.AddLeafNode(NodeKind::ImportIntroducer, intro_token); switch (context.packaging_state()) { - case Context::PackagingState::StartOfFile: + case Context::PackagingState::FileStart: // `package` is no longer allowed, but `import` may repeat. context.set_packaging_state(Context::PackagingState::InImports); [[clang::fallthrough]]; diff --git a/toolchain/parse/node_kind.def b/toolchain/parse/node_kind.def index 847a145490ca..157ca6c07a07 100644 --- a/toolchain/parse/node_kind.def +++ b/toolchain/parse/node_kind.def @@ -61,10 +61,10 @@ #endif // The start of the file. -CARBON_PARSE_NODE_KIND_CHILD_COUNT(FileStart, 0, CARBON_TOKEN(StartOfFile)) +CARBON_PARSE_NODE_KIND_CHILD_COUNT(FileStart, 0, CARBON_TOKEN(FileStart)) // The end of the file. -CARBON_PARSE_NODE_KIND_CHILD_COUNT(FileEnd, 0, CARBON_TOKEN(EndOfFile)) +CARBON_PARSE_NODE_KIND_CHILD_COUNT(FileEnd, 0, CARBON_TOKEN(FileEnd)) // An invalid parse. Used to balance the parse tree. Always has an error. CARBON_PARSE_NODE_KIND_CHILD_COUNT(InvalidParse, 0, diff --git a/toolchain/parse/tree.cpp b/toolchain/parse/tree.cpp index 99b8fa6b5be0..15dc5cde90d2 100644 --- a/toolchain/parse/tree.cpp +++ b/toolchain/parse/tree.cpp @@ -27,7 +27,7 @@ auto Tree::Parse(Lex::TokenizedBuffer& tokens, DiagnosticConsumer& consumer, [&](llvm::raw_ostream& output) { context.PrintForStackDump(output); }); context.AddLeafNode(NodeKind::FileStart, - context.ConsumeChecked(Lex::TokenKind::StartOfFile)); + context.ConsumeChecked(Lex::TokenKind::FileStart)); context.PushState(State::DeclScopeLoop); diff --git a/toolchain/testing/file_test.cpp b/toolchain/testing/file_test.cpp index ade3983db8eb..093c591a0ebc 100644 --- a/toolchain/testing/file_test.cpp +++ b/toolchain/testing/file_test.cpp @@ -76,12 +76,12 @@ class ToolchainFileTest : public FileTestBase { check_line = "// CHECK:STDOUT: {{.*}}"; } } else if (component_ == "lex") { - // Both StartOfFile and EndOfFile regularly have locations on CHECK + // Both FileStart and FileEnd regularly have locations on CHECK // comment lines that don't work correctly. The line happens to be correct - // for the EndOfFile, but we need to avoid checking the column. - // The column happens to be right for StartOfFile, but the line is wrong. + // for the FileEnd, but we need to avoid checking the column. + // The column happens to be right for FileStart, but the line is wrong. static RE2 file_token_re( - R"((EndOfFile.*column: |StartOfFile.*line: )( *\d+))"); + R"((FileEnd.*column: |FileStart.*line: )( *\d+))"); RE2::Replace(&check_line, file_token_re, R"(\1{{ *\\d+}})"); } else { return FileTestBase::DoExtraCheckReplacements(check_line);