carbon-language/carbon-lang
1
0 Fork 0
mirror of https://github.com/carbon-language/carbon-lang.git synced 2026-11-14 16:58:29 +00:00
Code Issues Packages Projects Releases 10 Wiki Activity
Files
0421bf3b4386901c99da67c41cfea21830d79c05
carbon-lang/toolchain/lex/tokenized_buffer_test.cpp
T
Chandler Carruth 0421bf3b43 Optimize token kinds and whitespace flags (#8044) 
Currently, these lookup the token in its `ValueStore`, and then access
the kind and flag from that. This is near-optimal in the lexer while
_writing_ the tokens, but ends up being considerably slower in parsing
when the overhead of digging the token out of the value store dominates
the actual use case.

This PR does an struct-of-arrays reorganization of the kinds and
whitespace flags, but only after lexing. This lets lexing write a linear
series of info structs, and only afterward re-organize into a
query-optimized representation that is a struct of arrays.

However, to make this a net performance gain, we have to create the
arrays _very_ efficiently. We can avoid hand-written assembly here by
very carefully and precisely writing the inner loop to build these
arrays so that LLVM will vectorize it into high-quality SIMD code for us
across both x86 and Arm. We also need the arrays to have a particular
shape (for example, using a `SmallVector<bool>` rather than a
`BitVector`) in order to preserve this vectorized creation. It burns a
tiny amount of memory, but is key to the whole thing being a net win of
5% to 10% across parsing.

Benchmark results (./scripts/bench_runner.py --runs=12,
-march=x86-64-v3):

- BM_CompileApiFileDenseDecls<Lang::Carbon, Phase::Parse>/16384: CPU
Time: -9.045%
- BM_CompileApiFileDenseDecls<Lang::Carbon, Phase::Parse>/65536: CPU
Time: -6.526%
- BM_CompileImplFile<Lang::Carbon, Phase::Parse>/16384: CPU Time:
-9.275%
- BM_CompileImplFile<Lang::Carbon, Phase::Parse>/65536: CPU Time:
-8.382%

Assisted-by: Antigravity with both Gemini and Claude
2026-10-28 05:36:57 +00:00

1620 lines
63 KiB
C++
Raw Blame History

// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
// Exceptions. See /LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#include "toolchain/lex/tokenized_buffer.h"
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <algorithm>
#include <cmath>
#include <forward_list>
#include <iterator>
#include <string>
#include "common/raw_string_ostream.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/Support/FormatVariadic.h"
#include "toolchain/base/shared_value_stores.h"
#include "toolchain/diagnostics/emitter.h"
#include "toolchain/diagnostics/mocks.h"
#include "toolchain/lex/lex.h"
#include "toolchain/lex/tokenized_buffer_test_helpers.h"
#include "toolchain/testing/compile_helper.h"
#include "toolchain/testing/yaml_test_helpers.h"
namespace Carbon::Lex {
namespace {
using ::Carbon::Testing::ExpectedToken;
using ::Carbon::Testing::IsSingleDiagnostic;
using ::testing::_;
using ::testing::ElementsAre;
using ::testing::Eq;
using ::testing::HasSubstr;
using ::testing::Pair;
namespace Yaml = ::Carbon::Testing::Yaml;
class LexerTest : public ::testing::Test {
public:
Testing::CompileHelper compile_helper_;
};
TEST_F(LexerTest, HandlesEmptyBuffer) {
auto& buffer = compile_helper_.GetTokenizedBuffer("");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::FileEnd}}));
}
TEST_F(LexerTest, NullStringRef) {
auto& buffer = compile_helper_.GetTokenizedBuffer(llvm::StringRef());
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::FileEnd}}));
}
TEST_F(LexerTest, TracksLinesAndColumns) {
auto& buffer = compile_helper_.GetTokenizedBuffer(
"\n ;;\n ;;;\n x\"foo\" '''baz\n a\n ''' y");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart,
.line = 1,
.column = 1,
.indent_column = 1},
{.kind = TokenKind::Semi, .line = 2, .column = 3, .indent_column = 3},
{.kind = TokenKind::Semi, .line = 2, .column = 4, .indent_column = 3},
{.kind = TokenKind::Semi, .line = 3, .column = 4, .indent_column = 4},
{.kind = TokenKind::Semi, .line = 3, .column = 5, .indent_column = 4},
{.kind = TokenKind::Semi, .line = 3, .column = 6, .indent_column = 4},
{.kind = TokenKind::Identifier,
.line = 4,
.column = 4,
.indent_column = 4,
.text = "x"},
{.kind = TokenKind::StringLiteral,
.line = 4,
.column = 5,
.indent_column = 4},
{.kind = TokenKind::StringLiteral,
.line = 4,
.column = 11,
.indent_column = 4},
{.kind = TokenKind::Identifier,
.line = 6,
.column = 6,
.indent_column = 2,
.text = "y"},
{.kind = TokenKind::FileEnd, .line = 6, .column = 7},
}));
}
TEST_F(LexerTest, TracksLinesAndColumnsCrLf) {
auto& buffer = compile_helper_.GetTokenizedBuffer(
"\r\n ;;\r\n ;;;\r\n x\"foo\" '''baz\r\n a\r\n ''' y");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart,
.line = 1,
.column = 1,
.indent_column = 1},
{.kind = TokenKind::Semi, .line = 2, .column = 3, .indent_column = 3},
{.kind = TokenKind::Semi, .line = 2, .column = 4, .indent_column = 3},
{.kind = TokenKind::Semi, .line = 3, .column = 4, .indent_column = 4},
{.kind = TokenKind::Semi, .line = 3, .column = 5, .indent_column = 4},
{.kind = TokenKind::Semi, .line = 3, .column = 6, .indent_column = 4},
{.kind = TokenKind::Identifier,
.line = 4,
.column = 4,
.indent_column = 4,
.text = "x"},
{.kind = TokenKind::StringLiteral,
.line = 4,
.column = 5,
.indent_column = 4},
{.kind = TokenKind::StringLiteral,
.line = 4,
.column = 11,
.indent_column = 4},
{.kind = TokenKind::Identifier,
.line = 6,
.column = 6,
.indent_column = 2,
.text = "y"},
{.kind = TokenKind::FileEnd, .line = 6, .column = 7},
}));
}
TEST_F(LexerTest, InvalidCR) {
auto& buffer = compile_helper_.GetTokenizedBuffer("\n ;;\r ;\n x");
EXPECT_TRUE(buffer.has_errors());
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart,
.line = 1,
.column = 1,
.indent_column = 1},
{.kind = TokenKind::Semi, .line = 2, .column = 2, .indent_column = 2},
{.kind = TokenKind::Semi, .line = 2, .column = 3, .indent_column = 2},
{.kind = TokenKind::Semi, .line = 2, .column = 6, .indent_column = 2},
{.kind = TokenKind::Identifier,
.line = 3,
.column = 4,
.indent_column = 4,
.text = "x"},
{.kind = TokenKind::FileEnd, .line = 3, .column = 5},
}));
}
TEST_F(LexerTest, InvalidLfCr) {
auto& buffer = compile_helper_.GetTokenizedBuffer("\n ;;\n\r ;\n x");
EXPECT_TRUE(buffer.has_errors());
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart,
.line = 1,
.column = 1,
.indent_column = 1},
{.kind = TokenKind::Semi, .line = 2, .column = 2, .indent_column = 2},
{.kind = TokenKind::Semi, .line = 2, .column = 3, .indent_column = 2},
{.kind = TokenKind::Semi, .line = 3, .column = 3, .indent_column = 1},
{.kind = TokenKind::Identifier,
.line = 4,
.column = 4,
.indent_column = 4,
.text = "x"},
{.kind = TokenKind::FileEnd, .line = 4, .column = 5},
}));
}
TEST_F(LexerTest, HandlesNumericLiteral) {
auto [buffer, value_stores] =
compile_helper_.GetTokenizedBufferWithSharedValueStore(
"12-578\n 1 2\n0x12_3ABC\n0b10_10_11\n1_234_567\n1.5e9");
EXPECT_FALSE(buffer.has_errors());
ASSERT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::IntLiteral,
.line = 1,
.column = 1,
.indent_column = 1,
.text = "12"},
{.kind = TokenKind::Minus,
.line = 1,
.column = 3,
.indent_column = 1},
{.kind = TokenKind::IntLiteral,
.line = 1,
.column = 4,
.indent_column = 1,
.text = "578"},
{.kind = TokenKind::IntLiteral,
.line = 2,
.column = 3,
.indent_column = 3,
.text = "1"},
{.kind = TokenKind::IntLiteral,
.line = 2,
.column = 6,
.indent_column = 3,
.text = "2"},
{.kind = TokenKind::IntLiteral,
.line = 3,
.column = 1,
.indent_column = 1,
.text = "0x12_3ABC"},
{.kind = TokenKind::IntLiteral,
.line = 4,
.column = 1,
.indent_column = 1,
.text = "0b10_10_11"},
{.kind = TokenKind::IntLiteral,
.line = 5,
.column = 1,
.indent_column = 1,
.text = "1_234_567"},
{.kind = TokenKind::RealLiteral,
.line = 6,
.column = 1,
.indent_column = 1,
.text = "1.5e9"},
{.kind = TokenKind::FileEnd, .line = 6, .column = 6},
}));
auto token_start = buffer.tokens().begin();
auto token_12 = token_start + 1;
EXPECT_EQ(value_stores.ints().Get(buffer.GetIntLiteral(*token_12)), 12);
auto token_578 = token_12 + 2;
EXPECT_EQ(value_stores.ints().Get(buffer.GetIntLiteral(*token_578)), 578);
auto token_1 = token_578 + 1;
EXPECT_EQ(value_stores.ints().Get(buffer.GetIntLiteral(*token_1)), 1);
auto token_2 = token_1 + 1;
EXPECT_EQ(value_stores.ints().Get(buffer.GetIntLiteral(*token_2)), 2);
auto token_0x12_3abc = token_2 + 1;
EXPECT_EQ(value_stores.ints().Get(buffer.GetIntLiteral(*token_0x12_3abc)),
0x12'3abc);
auto token_0b10_10_11 = token_0x12_3abc + 1;
EXPECT_EQ(value_stores.ints().Get(buffer.GetIntLiteral(*token_0b10_10_11)),
0b10'10'11);
auto token_1_234_567 = token_0b10_10_11 + 1;
EXPECT_EQ(value_stores.ints().Get(buffer.GetIntLiteral(*token_1_234_567)),
1'234'567);
auto token_1_5e9 = token_1_234_567 + 1;
auto value_1_5e9 =
value_stores.reals().Get(buffer.GetRealLiteral(*token_1_5e9));
EXPECT_EQ(value_1_5e9.mantissa.getSExtValue(), 15);
EXPECT_EQ(value_1_5e9.exponent.getSExtValue(), 8);
EXPECT_EQ(value_1_5e9.is_decimal, true);
}
TEST_F(LexerTest, HandlesInvalidNumericLiterals) {
auto& buffer =
compile_helper_.GetTokenizedBuffer("14x 15_49 0x3.5q 0x3_4.5_6 0ops");
EXPECT_TRUE(buffer.has_errors());
ASSERT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::Error,
.line = 1,
.column = 1,
.indent_column = 1,
.text = "14x"},
{.kind = TokenKind::IntLiteral,
.line = 1,
.column = 5,
.indent_column = 1,
.text = "15_49"},
{.kind = TokenKind::Error,
.line = 1,
.column = 11,
.indent_column = 1,
.text = "0x3.5q"},
{.kind = TokenKind::RealLiteral,
.line = 1,
.column = 18,
.indent_column = 1,
.text = "0x3_4.5_6"},
{.kind = TokenKind::Error,
.line = 1,
.column = 28,
.indent_column = 1,
.text = "0ops"},
{.kind = TokenKind::FileEnd, .line = 1, .column = 32},
}));
}
TEST_F(LexerTest, SplitsNumericLiteralsProperly) {
llvm::StringLiteral source_text = R"(
1.
.2
3.+foo
4.0-bar
5.0e+123+456
6.0e+1e+2
1e7
8..10
9.0.9.5
10.foo
11.0.foo
12e+1
13._
)";
auto& buffer = compile_helper_.GetTokenizedBuffer(source_text);
EXPECT_TRUE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::IntLiteral, .text = "1"},
{.kind = TokenKind::Period},
// newline
{.kind = TokenKind::Period},
{.kind = TokenKind::IntLiteral, .text = "2"},
// newline
{.kind = TokenKind::IntLiteral, .text = "3"},
{.kind = TokenKind::Period},
{.kind = TokenKind::Plus},
{.kind = TokenKind::Identifier, .text = "foo"},
// newline
{.kind = TokenKind::RealLiteral, .text = "4.0"},
{.kind = TokenKind::Minus},
{.kind = TokenKind::Identifier, .text = "bar"},
// newline
{.kind = TokenKind::RealLiteral, .text = "5.0e+123"},
{.kind = TokenKind::Plus},
{.kind = TokenKind::IntLiteral, .text = "456"},
// newline
{.kind = TokenKind::Error, .text = "6.0e+1e"},
{.kind = TokenKind::Plus},
{.kind = TokenKind::IntLiteral, .text = "2"},
// newline
{.kind = TokenKind::Error, .text = "1e7"},
// newline
{.kind = TokenKind::IntLiteral, .text = "8"},
{.kind = TokenKind::Period},
{.kind = TokenKind::Period},
{.kind = TokenKind::IntLiteral, .text = "10"},
// newline
{.kind = TokenKind::RealLiteral, .text = "9.0"},
{.kind = TokenKind::Period},
{.kind = TokenKind::IntLiteral, .text = "9"},
{.kind = TokenKind::Period},
{.kind = TokenKind::IntLiteral, .text = "5"},
// newline
{.kind = TokenKind::Error, .text = "10.foo"},
// newline
{.kind = TokenKind::RealLiteral, .text = "11.0"},
{.kind = TokenKind::Period},
{.kind = TokenKind::Identifier, .text = "foo"},
// newline
{.kind = TokenKind::Error, .text = "12e"},
{.kind = TokenKind::Plus},
{.kind = TokenKind::IntLiteral, .text = "1"},
// newline
{.kind = TokenKind::IntLiteral, .text = "13"},
{.kind = TokenKind::Period},
{.kind = TokenKind::Underscore},
// newline
{.kind = TokenKind::FileEnd},
}));
}
TEST_F(LexerTest, HandlesGarbageCharacters) {
constexpr char GarbageText[] = "##💩-#\n#\0#12#\n\\\"\\\n\"x";
auto& buffer = compile_helper_.GetTokenizedBuffer(
llvm::StringRef(GarbageText, sizeof(GarbageText) - 1));
EXPECT_TRUE(buffer.has_errors());
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::Error,
.line = 1,
.column = 1,
// 💩 takes 4 bytes, and we count column as bytes offset.
.text = llvm::StringRef("##💩", 6)},
{.kind = TokenKind::Minus, .line = 1, .column = 7},
{.kind = TokenKind::Error, .line = 1, .column = 8, .text = "#"},
// newline
{.kind = TokenKind::Error,
.line = 2,
.column = 1,
.text = llvm::StringRef("#\0#", 3)},
{.kind = TokenKind::IntLiteral, .line = 2, .column = 4, .text = "12"},
{.kind = TokenKind::Error, .line = 2, .column = 6, .text = "#"},
// newline
{.kind = TokenKind::Backslash, .line = 3, .column = 1, .text = "\\"},
{.kind = TokenKind::Error, .line = 3, .column = 2, .text = "\"\\"},
// newline
{.kind = TokenKind::Error, .line = 4, .column = 1, .text = "\"x"},
{.kind = TokenKind::FileEnd, .line = 4, .column = 3},
}));
}
TEST_F(LexerTest, Symbols) {
// We don't need to exhaustively test symbols here as they're handled with
// common code, but we want to check specific patterns to verify things like
// max-munch rule and handling of interesting symbols.
auto& buffer1 = compile_helper_.GetTokenizedBuffer("<<<");
EXPECT_FALSE(buffer1.has_errors());
EXPECT_THAT(buffer1, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::LessLess},
{.kind = TokenKind::Less},
{.kind = TokenKind::FileEnd},
}));
auto& buffer2 = compile_helper_.GetTokenizedBuffer("<<=>>");
EXPECT_FALSE(buffer2.has_errors());
EXPECT_THAT(buffer2, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::LessLessEqual},
{.kind = TokenKind::GreaterGreater},
{.kind = TokenKind::FileEnd},
}));
auto& buffer3 = compile_helper_.GetTokenizedBuffer("< <=> >");
EXPECT_FALSE(buffer3.has_errors());
EXPECT_THAT(buffer3, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::Less},
{.kind = TokenKind::LessEqualGreater},
{.kind = TokenKind::Greater},
{.kind = TokenKind::FileEnd},
}));
auto& buffer4 = compile_helper_.GetTokenizedBuffer("\\/?@&^!");
EXPECT_FALSE(buffer4.has_errors());
EXPECT_THAT(buffer4, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::Backslash},
{.kind = TokenKind::Slash},
{.kind = TokenKind::Question},
{.kind = TokenKind::At},
{.kind = TokenKind::Amp},
{.kind = TokenKind::Caret},
{.kind = TokenKind::Exclaim},
{.kind = TokenKind::FileEnd},
}));
}
TEST_F(LexerTest, Parens) {
auto& buffer1 = compile_helper_.GetTokenizedBuffer("()");
EXPECT_FALSE(buffer1.has_errors());
EXPECT_THAT(buffer1, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::OpenParen},
{.kind = TokenKind::CloseParen},
{.kind = TokenKind::FileEnd},
}));
auto& buffer2 = compile_helper_.GetTokenizedBuffer("((()()))");
EXPECT_FALSE(buffer2.has_errors());
EXPECT_THAT(buffer2, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::OpenParen},
{.kind = TokenKind::OpenParen},
{.kind = TokenKind::OpenParen},
{.kind = TokenKind::CloseParen},
{.kind = TokenKind::OpenParen},
{.kind = TokenKind::CloseParen},
{.kind = TokenKind::CloseParen},
{.kind = TokenKind::CloseParen},
{.kind = TokenKind::FileEnd},
}));
}
TEST_F(LexerTest, CurlyBraces) {
auto& buffer1 = compile_helper_.GetTokenizedBuffer("{}");
EXPECT_FALSE(buffer1.has_errors());
EXPECT_THAT(buffer1, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::OpenCurlyBrace},
{.kind = TokenKind::CloseCurlyBrace},
{.kind = TokenKind::FileEnd},
}));
auto& buffer2 = compile_helper_.GetTokenizedBuffer("{{{}{}}}");
EXPECT_FALSE(buffer2.has_errors());
EXPECT_THAT(buffer2, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::OpenCurlyBrace},
{.kind = TokenKind::OpenCurlyBrace},
{.kind = TokenKind::OpenCurlyBrace},
{.kind = TokenKind::CloseCurlyBrace},
{.kind = TokenKind::OpenCurlyBrace},
{.kind = TokenKind::CloseCurlyBrace},
{.kind = TokenKind::CloseCurlyBrace},
{.kind = TokenKind::CloseCurlyBrace},
{.kind = TokenKind::FileEnd},
}));
}
TEST_F(LexerTest, MatchingGroups) {
{
auto& buffer = compile_helper_.GetTokenizedBuffer("(){}");
ASSERT_FALSE(buffer.has_errors());
auto it = ++buffer.tokens().begin();
auto open_paren_token = *it++;
auto close_paren_token = *it++;
EXPECT_EQ(close_paren_token,
buffer.GetMatchedClosingToken(open_paren_token));
EXPECT_EQ(open_paren_token,
buffer.GetMatchedOpeningToken(close_paren_token));
auto open_curly_token = *it++;
auto close_curly_token = *it++;
EXPECT_EQ(close_curly_token,
buffer.GetMatchedClosingToken(open_curly_token));
EXPECT_EQ(open_curly_token,
buffer.GetMatchedOpeningToken(close_curly_token));
auto eof_token = *it++;
EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::FileEnd);
EXPECT_EQ(buffer.tokens().end(), it);
}
{
auto [buffer, value_stores] =
compile_helper_.GetTokenizedBufferWithSharedValueStore(
"({x}){(y)} {{((z))}}");
ASSERT_FALSE(buffer.has_errors());
auto it = ++buffer.tokens().begin();
auto open_paren_token = *it++;
auto open_curly_token = *it++;
ASSERT_EQ("x", value_stores.identifiers().Get(buffer.GetIdentifier(*it++)));
auto close_curly_token = *it++;
auto close_paren_token = *it++;
EXPECT_EQ(close_paren_token,
buffer.GetMatchedClosingToken(open_paren_token));
EXPECT_EQ(open_paren_token,
buffer.GetMatchedOpeningToken(close_paren_token));
EXPECT_EQ(close_curly_token,
buffer.GetMatchedClosingToken(open_curly_token));
EXPECT_EQ(open_curly_token,
buffer.GetMatchedOpeningToken(close_curly_token));
open_curly_token = *it++;
open_paren_token = *it++;
ASSERT_EQ("y", value_stores.identifiers().Get(buffer.GetIdentifier(*it++)));
close_paren_token = *it++;
close_curly_token = *it++;
EXPECT_EQ(close_curly_token,
buffer.GetMatchedClosingToken(open_curly_token));
EXPECT_EQ(open_curly_token,
buffer.GetMatchedOpeningToken(close_curly_token));
EXPECT_EQ(close_paren_token,
buffer.GetMatchedClosingToken(open_paren_token));
EXPECT_EQ(open_paren_token,
buffer.GetMatchedOpeningToken(close_paren_token));
open_curly_token = *it++;
auto inner_open_curly_token = *it++;
open_paren_token = *it++;
auto inner_open_paren_token = *it++;
ASSERT_EQ("z", value_stores.identifiers().Get(buffer.GetIdentifier(*it++)));
auto inner_close_paren_token = *it++;
close_paren_token = *it++;
auto inner_close_curly_token = *it++;
close_curly_token = *it++;
EXPECT_EQ(close_curly_token,
buffer.GetMatchedClosingToken(open_curly_token));
EXPECT_EQ(open_curly_token,
buffer.GetMatchedOpeningToken(close_curly_token));
EXPECT_EQ(inner_close_curly_token,
buffer.GetMatchedClosingToken(inner_open_curly_token));
EXPECT_EQ(inner_open_curly_token,
buffer.GetMatchedOpeningToken(inner_close_curly_token));
EXPECT_EQ(close_paren_token,
buffer.GetMatchedClosingToken(open_paren_token));
EXPECT_EQ(open_paren_token,
buffer.GetMatchedOpeningToken(close_paren_token));
EXPECT_EQ(inner_close_paren_token,
buffer.GetMatchedClosingToken(inner_open_paren_token));
EXPECT_EQ(inner_open_paren_token,
buffer.GetMatchedOpeningToken(inner_close_paren_token));
auto eof_token = *it++;
EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::FileEnd);
EXPECT_EQ(buffer.tokens().end(), it);
}
}
TEST_F(LexerTest, MismatchedGroups) {
auto& buffer1 = compile_helper_.GetTokenizedBuffer("{");
EXPECT_TRUE(buffer1.has_errors());
EXPECT_THAT(
buffer1,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::OpenCurlyBrace, .column = 1},
{.kind = TokenKind::CloseCurlyBrace, .column = 2, .recovery = true},
{.kind = TokenKind::FileEnd},
}));
auto& buffer2 = compile_helper_.GetTokenizedBuffer("}");
EXPECT_TRUE(buffer2.has_errors());
EXPECT_THAT(buffer2, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::Error, .text = "}"},
{.kind = TokenKind::FileEnd},
}));
auto& buffer3 = compile_helper_.GetTokenizedBuffer("{(}");
EXPECT_TRUE(buffer3.has_errors());
EXPECT_THAT(
buffer3,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::OpenCurlyBrace, .column = 1},
{.kind = TokenKind::OpenParen, .column = 2},
{.kind = TokenKind::CloseParen, .column = 3, .recovery = true},
{.kind = TokenKind::CloseCurlyBrace, .column = 3},
{.kind = TokenKind::FileEnd},
}));
// `{((}` is recovered by closing both parens before the `}`, i.e. `{(())}`.
auto& buffer3b = compile_helper_.GetTokenizedBuffer("{((}");
EXPECT_TRUE(buffer3b.has_errors());
EXPECT_THAT(
buffer3b,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::OpenCurlyBrace, .column = 1},
{.kind = TokenKind::OpenParen, .column = 2},
{.kind = TokenKind::OpenParen, .column = 3},
{.kind = TokenKind::CloseParen, .column = 4, .recovery = true},
{.kind = TokenKind::CloseParen, .column = 4, .recovery = true},
{.kind = TokenKind::CloseCurlyBrace, .column = 4},
{.kind = TokenKind::FileEnd},
}));
// Recovery insertions at two separate points: the second one's merged index
// is shifted by the first, and a real token must not be flagged in its
// place.
auto& buffer3c = compile_helper_.GetTokenizedBuffer("{(} {(}");
EXPECT_TRUE(buffer3c.has_errors());
EXPECT_THAT(
buffer3c,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::OpenCurlyBrace, .column = 1},
{.kind = TokenKind::OpenParen, .column = 2},
{.kind = TokenKind::CloseParen, .column = 3, .recovery = true},
{.kind = TokenKind::CloseCurlyBrace, .column = 3},
{.kind = TokenKind::OpenCurlyBrace, .column = 5},
{.kind = TokenKind::OpenParen, .column = 6},
{.kind = TokenKind::CloseParen, .column = 7, .recovery = true},
{.kind = TokenKind::CloseCurlyBrace, .column = 7},
{.kind = TokenKind::FileEnd},
}));
auto& buffer4 = compile_helper_.GetTokenizedBuffer(")({)");
EXPECT_TRUE(buffer4.has_errors());
EXPECT_THAT(
buffer4,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::Error, .column = 1, .text = ")"},
{.kind = TokenKind::OpenParen, .column = 2},
{.kind = TokenKind::OpenCurlyBrace, .column = 3},
{.kind = TokenKind::CloseCurlyBrace, .column = 4, .recovery = true},
{.kind = TokenKind::CloseParen, .column = 4},
{.kind = TokenKind::FileEnd},
}));
}
TEST_F(LexerTest, Whitespace) {
// The trailing `{(` is recovered by inserting `)` and `}` at end of file,
// giving `{()} {()}`.
auto& buffer = compile_helper_.GetTokenizedBuffer("{( } {(");
// Whether there should be whitespace at each boundary, from before the
// first token to after the last.
bool space[] = {false,
// start-of-file
true,
// {
false,
// (
true,
// inserted )
false,
// }
true,
// {
false,
// (
true,
// inserted )
false,
// inserted }
false,
// EOF
false};
int pos = 0;
for (TokenIndex token : buffer.tokens()) {
SCOPED_TRACE(
llvm::formatv("Token #{0}: '{1}'", token, buffer.GetTokenText(token)));
ASSERT_LT(pos, std::size(space));
EXPECT_THAT(buffer.HasLeadingWhitespace(token), Eq(space[pos]));
++pos;
ASSERT_LT(pos, std::size(space));
EXPECT_THAT(buffer.HasTrailingWhitespace(token), Eq(space[pos]));
}
ASSERT_EQ(pos + 1, std::size(space));
}
TEST_F(LexerTest, KindDataAndWhitespaceForLongInput) {
// The token kinds and whitespace flags are built in bulk once lexing
// finishes, so check them on an input long enough to exercise the bulk path
// as well as its tail, with a mix of kinds and whitespace.
constexpr llvm::StringLiteral Line =
"fn x(a: i32) -> bool { return a+1 == 2; }\n";
constexpr int Repeats = 37;
std::string source;
for (int i = 0; i < Repeats; ++i) {
source += Line;
}
auto& buffer = compile_helper_.GetTokenizedBuffer(source);
EXPECT_FALSE(buffer.has_errors());
struct ExpectedLineToken {
TokenKind kind;
bool has_leading_space;
};
const ExpectedLineToken expected_line[] = {
{TokenKind::Fn, true}, //
{TokenKind::Identifier, true}, //
{TokenKind::OpenParen, false}, //
{TokenKind::Identifier, false}, //
{TokenKind::Colon, false}, //
{TokenKind::IntTypeLiteral, true}, //
{TokenKind::CloseParen, false}, //
{TokenKind::MinusGreater, true}, //
{TokenKind::Bool, true}, //
{TokenKind::OpenCurlyBrace, true}, //
{TokenKind::Return, true}, //
{TokenKind::Identifier, true}, //
{TokenKind::Plus, false}, //
{TokenKind::IntLiteral, false}, //
{TokenKind::EqualEqual, true}, //
{TokenKind::IntLiteral, true}, //
{TokenKind::Semi, false}, //
{TokenKind::CloseCurlyBrace, true},
};
constexpr int TokensPerLine = std::size(expected_line);
ASSERT_EQ(buffer.size(), 2 + Repeats * TokensPerLine);
const TokenKind* kind_data = buffer.kind_data();
for (TokenIndex token : buffer.tokens()) {
SCOPED_TRACE(
llvm::formatv("Token #{0}: '{1}'", token, buffer.GetTokenText(token)));
ExpectedLineToken expected;
if (token.index == 0) {
expected = {TokenKind::FileStart, false};
} else if (token.index == buffer.size() - 1) {
expected = {TokenKind::FileEnd, true};
} else {
// The start of the file counts as whitespace, so the first `fn` matches
// the pattern too.
expected = expected_line[(token.index - 1) % TokensPerLine];
}
EXPECT_EQ(buffer.GetKind(token), expected.kind);
EXPECT_EQ(kind_data[token.index], expected.kind);
EXPECT_EQ(buffer.HasLeadingWhitespace(token), expected.has_leading_space);
if (token.index + 1 < buffer.size()) {
EXPECT_EQ(buffer.HasTrailingWhitespace(token),
buffer.HasLeadingWhitespace(TokenIndex(token.index + 1)));
} else {
EXPECT_FALSE(buffer.HasTrailingWhitespace(token));
}
}
for (int i = 0; i < TokenizedBuffer::KindDataPadding; ++i) {
EXPECT_EQ(kind_data[buffer.size() + i], TokenKind::FileEnd) << i;
}
// Adding recovery tokens after lexing extends the arrays and keeps the
// padding intact.
int lexed_size = buffer.size();
for (int i = 0; i < 3; ++i) {
TokenIndex fn_token(1 + i * TokensPerLine);
TokenIndex recovered =
buffer.AddPostLexingRecoveryTokenAsIdentifier(fn_token);
EXPECT_EQ(recovered.index, lexed_size + i);
EXPECT_EQ(buffer.GetKind(recovered), TokenKind::Identifier);
EXPECT_EQ(buffer.GetTokenText(recovered), "fn");
EXPECT_EQ(buffer.HasLeadingWhitespace(recovered),
buffer.HasLeadingWhitespace(fn_token));
}
EXPECT_EQ(buffer.size(), lexed_size + 3);
EXPECT_EQ(buffer.tokens().end() - buffer.tokens().begin(), lexed_size);
kind_data = buffer.kind_data();
for (TokenIndex token : buffer.tokens()) {
EXPECT_EQ(kind_data[token.index], buffer.GetKind(token));
}
for (int i = 0; i < TokenizedBuffer::KindDataPadding; ++i) {
EXPECT_EQ(kind_data[buffer.size() + i], TokenKind::FileEnd) << i;
}
}
TEST_F(LexerTest, Keywords) {
TokenKind keywords[] = {
#define CARBON_TOKEN(TokenName)
#define CARBON_KEYWORD_TOKEN(TokenName, ...) TokenKind::TokenName,
#include "toolchain/lex/token_kind.def"
};
for (const auto& keyword : keywords) {
auto& buffer = compile_helper_.GetTokenizedBuffer(keyword.fixed_spelling());
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = keyword, .column = 1, .indent_column = 1},
{.kind = TokenKind::FileEnd},
}));
}
}
TEST_F(LexerTest, Comments) {
auto& buffer1 = compile_helper_.GetTokenizedBuffer(" ;\n // foo\n ;\n");
EXPECT_FALSE(buffer1.has_errors());
EXPECT_THAT(
buffer1,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::Semi, .line = 1, .column = 2, .indent_column = 2},
{.kind = TokenKind::Semi, .line = 3, .column = 3, .indent_column = 3},
{.kind = TokenKind::FileEnd, .line = 3, .column = 4},
}));
auto& buffer2 = compile_helper_.GetTokenizedBuffer("// foo\n//\n// bar");
EXPECT_FALSE(buffer2.has_errors());
EXPECT_THAT(buffer2, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::FileEnd}}));
// Make sure weird characters aren't a problem.
auto& buffer3 =
compile_helper_.GetTokenizedBuffer(" // foo#$!^?@-_💩🍫⃠ [̲̅$̲̅(̲̅ ͡° ͜ʖ ͡°̲̅)̲̅$̲̅]");
EXPECT_FALSE(buffer3.has_errors());
EXPECT_THAT(buffer3, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::FileEnd}}));
// Make sure we can lex a comment at the end of the input.
auto& buffer4 = compile_helper_.GetTokenizedBuffer("//");
EXPECT_FALSE(buffer4.has_errors());
EXPECT_THAT(buffer4, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::FileEnd}}));
}
TEST_F(LexerTest, InvalidComments) {
llvm::StringLiteral testcases[] = {
" /// foo\n",
"//! hello",
" //world",
};
for (llvm::StringLiteral testcase : testcases) {
auto& buffer = compile_helper_.GetTokenizedBuffer(testcase);
EXPECT_TRUE(buffer.has_errors());
}
}
TEST_F(LexerTest, Identifiers) {
auto& buffer1 = compile_helper_.GetTokenizedBuffer(" foobar");
EXPECT_FALSE(buffer1.has_errors());
EXPECT_THAT(buffer1, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::Identifier,
.column = 4,
.indent_column = 4,
.text = "foobar"},
{.kind = TokenKind::FileEnd},
}));
// Check different kinds of identifier character sequences.
auto& buffer2 = compile_helper_.GetTokenizedBuffer("_foo_bar");
EXPECT_FALSE(buffer2.has_errors());
EXPECT_THAT(buffer2, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::Identifier, .text = "_foo_bar"},
{.kind = TokenKind::FileEnd},
}));
auto& buffer3 = compile_helper_.GetTokenizedBuffer("foo2bar00");
EXPECT_FALSE(buffer3.has_errors());
EXPECT_THAT(buffer3, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::Identifier, .text = "foo2bar00"},
{.kind = TokenKind::FileEnd},
}));
// Check that we can parse identifiers that start with a keyword.
auto& buffer4 = compile_helper_.GetTokenizedBuffer("fnord");
EXPECT_FALSE(buffer4.has_errors());
EXPECT_THAT(buffer4, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::Identifier, .text = "fnord"},
{.kind = TokenKind::FileEnd},
}));
// Check multiple identifiers with indent and interning.
auto& buffer5 =
compile_helper_.GetTokenizedBuffer(" foo;bar\nbar \n foo\tfoo");
EXPECT_FALSE(buffer5.has_errors());
EXPECT_THAT(buffer5,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::Identifier,
.line = 1,
.column = 4,
.indent_column = 4,
.text = "foo"},
{.kind = TokenKind::Semi},
{.kind = TokenKind::Identifier,
.line = 1,
.column = 8,
.indent_column = 4,
.text = "bar"},
{.kind = TokenKind::Identifier,
.line = 2,
.column = 1,
.indent_column = 1,
.text = "bar"},
{.kind = TokenKind::Identifier,
.line = 3,
.column = 3,
.indent_column = 3,
.text = "foo"},
{.kind = TokenKind::Identifier,
.line = 3,
.column = 7,
.indent_column = 3,
.text = "foo"},
{.kind = TokenKind::FileEnd, .line = 3, .column = 10},
}));
}
TEST_F(LexerTest, RawIdentifierIntroducerAtEndOfFile) {
// `r#` at the very end of the source is not a raw identifier -- there is no
// identifier after the `#` -- but the leading `r` is still an identifier
// whose text begins with `r#`. Computing that text checks for the raw form
// and must not read past the end of the buffer.
auto& buffer = compile_helper_.GetTokenizedBuffer("r#");
for (TokenIndex token : buffer.tokens()) {
if (buffer.GetKind(token) == TokenKind::Identifier) {
EXPECT_EQ(buffer.GetTokenText(token), "r");
}
}
}
TEST_F(LexerTest, StringLiterals) {
llvm::StringLiteral testcase = R"(
"hello world\n"
'''foo
test \
\xAB
''' trailing
#"""#
"\0"
#"\0"foo"\1"#
"""x"""
)";
auto [buffer, value_stores] =
compile_helper_.GetTokenizedBufferWithSharedValueStore(testcase);
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::StringLiteral,
.line = 2,
.column = 5,
.indent_column = 5,
.value_stores = &value_stores,
.string_contents = {"hello world\n"}},
{.kind = TokenKind::StringLiteral,
.line = 4,
.column = 5,
.indent_column = 5,
.value_stores = &value_stores,
.string_contents = {" test \xAB\n"}},
{.kind = TokenKind::Identifier,
.line = 7,
.column = 10,
.indent_column = 6,
.text = "trailing"},
{.kind = TokenKind::StringLiteral,
.line = 9,
.column = 7,
.indent_column = 7,
.value_stores = &value_stores,
.string_contents = {"\""}},
{.kind = TokenKind::StringLiteral,
.line = 11,
.column = 5,
.indent_column = 5,
.value_stores = &value_stores,
.string_contents = llvm::StringLiteral::withInnerNUL("\0")},
{.kind = TokenKind::StringLiteral,
.line = 13,
.column = 5,
.indent_column = 5,
.value_stores = &value_stores,
.string_contents = {"\\0\"foo\"\\1"}},
// """x""" is three string literals, not one invalid
// attempt at a block string literal.
{.kind = TokenKind::StringLiteral,
.line = 15,
.column = 5,
.indent_column = 5,
.value_stores = &value_stores,
.string_contents = {""}},
{.kind = TokenKind::StringLiteral,
.line = 15,
.column = 7,
.indent_column = 5,
.value_stores = &value_stores,
.string_contents = {"x"}},
{.kind = TokenKind::StringLiteral,
.line = 15,
.column = 10,
.indent_column = 5,
.value_stores = &value_stores,
.string_contents = {""}},
{.kind = TokenKind::FileEnd, .line = 16, .column = 3},
}));
}
TEST_F(LexerTest, InvalidStringLiterals) {
llvm::StringLiteral invalid[] = {
// clang-format off
R"(")",
R"('''
'')",
R"("\)",
R"("\")",
R"("\\)",
R"("\\\")",
R"(''')",
R"('''
)",
R"('''\)",
R"(#'''
''')",
// clang-format on
};
for (llvm::StringLiteral test : invalid) {
SCOPED_TRACE(test);
auto& buffer = compile_helper_.GetTokenizedBuffer(test);
EXPECT_TRUE(buffer.has_errors());
// We should have formed at least one error token.
bool found_error = false;
for (TokenIndex token : buffer.tokens()) {
if (buffer.GetKind(token) == TokenKind::Error) {
found_error = true;
break;
}
}
EXPECT_TRUE(found_error);
}
}
TEST_F(LexerTest, TypeLiterals) {
llvm::StringLiteral testcase = R"(
i0 i1 i20 i999999999999 i0x1
u0 u1 u64 u64b
f32 f80 f1 fi
s1
)";
auto [buffer, value_stores] =
compile_helper_.GetTokenizedBufferWithSharedValueStore(testcase);
EXPECT_FALSE(buffer.has_errors());
ASSERT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::Identifier,
.line = 2,
.column = 5,
.indent_column = 5,
.text = {"i0"}},
{.kind = TokenKind::IntTypeLiteral,
.line = 2,
.column = 8,
.indent_column = 5,
.text = {"i1"}},
{.kind = TokenKind::IntTypeLiteral,
.line = 2,
.column = 11,
.indent_column = 5,
.text = {"i20"}},
{.kind = TokenKind::IntTypeLiteral,
.line = 2,
.column = 15,
.indent_column = 5,
.text = {"i999999999999"}},
{.kind = TokenKind::Identifier,
.line = 2,
.column = 29,
.indent_column = 5,
.text = {"i0x1"}},
{.kind = TokenKind::Identifier,
.line = 3,
.column = 5,
.indent_column = 5,
.text = {"u0"}},
{.kind = TokenKind::UnsignedIntTypeLiteral,
.line = 3,
.column = 8,
.indent_column = 5,
.text = {"u1"}},
{.kind = TokenKind::UnsignedIntTypeLiteral,
.line = 3,
.column = 11,
.indent_column = 5,
.text = {"u64"}},
{.kind = TokenKind::Identifier,
.line = 3,
.column = 15,
.indent_column = 5,
.text = {"u64b"}},
{.kind = TokenKind::FloatTypeLiteral,
.line = 4,
.column = 5,
.indent_column = 5,
.text = {"f32"}},
{.kind = TokenKind::FloatTypeLiteral,
.line = 4,
.column = 9,
.indent_column = 5,
.text = {"f80"}},
{.kind = TokenKind::FloatTypeLiteral,
.line = 4,
.column = 13,
.indent_column = 5,
.text = {"f1"}},
{.kind = TokenKind::Identifier,
.line = 4,
.column = 16,
.indent_column = 5,
.text = {"fi"}},
{.kind = TokenKind::Identifier,
.line = 5,
.column = 5,
.indent_column = 5,
.text = {"s1"}},
{.kind = TokenKind::FileEnd, .line = 6, .column = 3},
}));
auto type_size = [&](int token_index) {
auto token = buffer.tokens().begin()[token_index];
return value_stores.ints().Get(buffer.GetTypeLiteralSize(token));
};
EXPECT_EQ(type_size(2), 1);
EXPECT_EQ(type_size(3), 20);
EXPECT_EQ(type_size(4), 999999999999ULL);
EXPECT_EQ(type_size(7), 1);
EXPECT_EQ(type_size(8), 64);
EXPECT_EQ(type_size(10), 32);
EXPECT_EQ(type_size(11), 80);
EXPECT_EQ(type_size(12), 1);
}
TEST_F(LexerTest, TypeLiteralTooManyDigits) {
// We increase the number of digits until the first one that is to large.
Testing::MockDiagnosticConsumer consumer;
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
Diagnostics::Kind::TooManyTypeBitWidthDigits,
Diagnostics::Level::Error, 1, 2, _)));
std::string code = "i";
// A 128-bit APInt should be plenty large, but if needed in the future it can
// be widened without issue.
llvm::APInt bits = llvm::APInt::getZero(128);
for ([[maybe_unused]] auto _ : llvm::seq(1, 30)) {
code.append("9");
bits = bits * 10 + 9;
auto [buffer, value_stores] =
compile_helper_.GetTokenizedBufferWithSharedValueStore(code, &consumer);
if (buffer.has_errors()) {
ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::Error, .text = code},
{.kind = TokenKind::FileEnd},
}));
break;
}
ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::IntTypeLiteral, .text = code},
{.kind = TokenKind::FileEnd},
}));
auto token = buffer.tokens().begin()[1];
EXPECT_TRUE(llvm::APInt::isSameValue(
value_stores.ints().Get(buffer.GetTypeLiteralSize(token)), bits));
}
// Make sure we can also gracefully reject very large number of digits without
// crashing or hanging, and show the correct number.
constexpr int Count = 10000;
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
Diagnostics::Kind::TooManyTypeBitWidthDigits,
Diagnostics::Level::Error, 1, 2,
HasSubstr(llvm::formatv(" {0} ", Count)))));
code = "i";
code.append(Count, '9');
auto& buffer = compile_helper_.GetTokenizedBuffer(code, &consumer);
ASSERT_TRUE(buffer.has_errors());
ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::Error, .text = code},
{.kind = TokenKind::FileEnd},
}));
}
TEST_F(LexerTest, TrailingComment) {
// A comment that follows other content on a line is a valid trailing comment.
auto& buffer = compile_helper_.GetTokenizedBuffer(
"// leading\nvar x: i32 = 0; // trailing\n// trailing's neighbor\n");
EXPECT_FALSE(buffer.has_errors());
// The trailing comment is recorded but never coalesced with an adjacent
// full-line comment, so the leading comment, the trailing comment, and the
// following full-line comment remain three separate comments.
EXPECT_THAT(buffer.comments_size(), Eq(3));
}
TEST_F(LexerTest, TrailingCommentAfterMultiLineString) {
// A multi-line string literal records the real indentation of each line it
// spans. Here the trailing `//` is deliberately aligned to the column where
// the literal opened (both at column 17); if the literal instead recorded
// that opening column as the final line's indent (as it once did), the
// trailing-comment check in `Lexer::LexComment` would misclassify this as a
// full-line comment.
auto& buffer = compile_helper_.GetTokenizedBuffer(
"var x: String = '''\n"
" text\n"
" '''; // trailing\n");
EXPECT_FALSE(buffer.has_errors());
ASSERT_THAT(buffer.comments_size(), Eq(1));
EXPECT_TRUE(buffer.IsTrailingComment(CommentIndex(0)));
}
TEST_F(LexerTest, DirectiveComments) {
// A `//@...` directive line is consumed for its tooling side effects and is
// also recorded as a comment: the tokens and comments together reconstruct
// the source, so tooling such as the formatter preserves the directive.
auto& buffer = compile_helper_.GetTokenizedBuffer(
"//@include-in-dumps\n"
"\n"
"//@dump-sem-ir-begin\n"
"var x: i32 = 0;\n"
"//@dump-sem-ir-end\n");
EXPECT_FALSE(buffer.has_errors());
EXPECT_TRUE(buffer.has_include_in_dumps());
EXPECT_TRUE(buffer.has_dump_sem_ir_ranges());
ASSERT_THAT(buffer.comments_size(), Eq(3));
EXPECT_THAT(buffer.GetCommentText(CommentIndex(0)),
Eq("//@include-in-dumps\n"));
EXPECT_THAT(buffer.GetCommentText(CommentIndex(1)),
Eq("//@dump-sem-ir-begin\n"));
EXPECT_THAT(buffer.GetCommentText(CommentIndex(2)),
Eq("//@dump-sem-ir-end\n"));
EXPECT_FALSE(buffer.IsTrailingComment(CommentIndex(0)));
// Adjacent full-line comments coalesce only within a category: a directive
// next to an ordinary comment is its own record, while adjacent directives
// share one.
auto& adjacent = compile_helper_.GetTokenizedBuffer(
"// Dump this file.\n"
"//@include-in-dumps\n"
"//@dump-sem-ir-begin\n"
"var x: i32 = 0;\n"
"//@dump-sem-ir-end\n");
EXPECT_FALSE(adjacent.has_errors());
EXPECT_TRUE(adjacent.has_include_in_dumps());
ASSERT_THAT(adjacent.comments_size(), Eq(3));
EXPECT_THAT(adjacent.GetCommentText(CommentIndex(0)),
Eq("// Dump this file.\n"));
EXPECT_THAT(adjacent.GetCommentText(CommentIndex(1)),
Eq("//@include-in-dumps\n//@dump-sem-ir-begin\n"));
EXPECT_THAT(adjacent.GetCommentText(CommentIndex(2)),
Eq("//@dump-sem-ir-end\n"));
}
TEST_F(LexerTest, DirectiveAfterInvalidComment) {
// An invalid comment line does not absorb a following directive: the
// directive ends the invalid run, so its side effects are still recognized
// and it is recorded separately.
auto& buffer = compile_helper_.GetTokenizedBuffer(
"//!bad\n"
"//@dump-sem-ir-begin\n"
"var x: i32 = 0;\n"
"//@dump-sem-ir-end\n");
EXPECT_TRUE(buffer.has_errors());
EXPECT_TRUE(buffer.has_dump_sem_ir_ranges());
ASSERT_THAT(buffer.comments_size(), Eq(3));
EXPECT_THAT(buffer.GetCommentText(CommentIndex(0)), Eq("//!bad\n"));
EXPECT_THAT(buffer.GetCommentText(CommentIndex(1)),
Eq("//@dump-sem-ir-begin\n"));
}
TEST_F(LexerTest, InvalidCommentRunsLumpTogether) {
// A run of invalid comment lines lumps into one comment and one diagnostic,
// no matter which invalid byte follows each `//`; a valid comment ends the
// run and starts its own record.
Testing::MockDiagnosticConsumer consumer;
EXPECT_CALL(consumer,
HandleDiagnostic(IsSingleDiagnostic(
Diagnostics::Kind::NoWhitespaceAfterCommentIntroducer,
Diagnostics::Level::Error, 1, 3, _)));
auto& buffer = compile_helper_.GetTokenizedBuffer(
"//!one\n"
"//?two\n"
"// valid\n",
&consumer);
ASSERT_THAT(buffer.comments_size(), Eq(2));
EXPECT_THAT(buffer.GetCommentText(CommentIndex(0)), Eq("//!one\n//?two\n"));
EXPECT_THAT(buffer.GetCommentText(CommentIndex(1)), Eq("// valid\n"));
}
TEST_F(LexerTest, InvalidCommentRunAtEof) {
// An invalid run's line-by-line skip reads the byte after each line's `//`;
// these cases end the source at that read's boundary, with no trailing
// newline, to pin its bounds.
// The final line's introducer byte is the last byte of the source.
auto& tight = compile_helper_.GetTokenizedBuffer(
"//!a\n"
"//!");
EXPECT_TRUE(tight.has_errors());
ASSERT_THAT(tight.comments_size(), Eq(1));
EXPECT_THAT(tight.GetCommentText(CommentIndex(0)), Eq("//!a\n//!"));
// A bare `//` ending the source has no byte after the introducer at all:
// the run must stop before it rather than read past the end, and it lexes
// as its own valid, empty comment.
auto& bare = compile_helper_.GetTokenizedBuffer(
"//!a\n"
"//");
EXPECT_TRUE(bare.has_errors());
ASSERT_THAT(bare.comments_size(), Eq(2));
EXPECT_THAT(bare.GetCommentText(CommentIndex(0)), Eq("//!a\n"));
EXPECT_THAT(bare.GetCommentText(CommentIndex(1)), Eq("//"));
}
TEST_F(LexerTest, DiagnosticWhitespace) {
Testing::MockDiagnosticConsumer consumer;
EXPECT_CALL(consumer,
HandleDiagnostic(IsSingleDiagnostic(
Diagnostics::Kind::NoWhitespaceAfterCommentIntroducer,
Diagnostics::Level::Error, 1, 3, _)));
compile_helper_.GetTokenizedBuffer("//no space after comment", &consumer);
}
TEST_F(LexerTest, DiagnosticUnrecognizedEscape) {
Testing::MockDiagnosticConsumer consumer;
EXPECT_CALL(consumer,
HandleDiagnostic(IsSingleDiagnostic(
Diagnostics::Kind::UnknownEscapeSequence,
Diagnostics::Level::Error, 1, 8, HasSubstr("`b`"))));
compile_helper_.GetTokenizedBuffer(R"("hello\bworld")", &consumer);
}
TEST_F(LexerTest, DiagnosticBadHex) {
Testing::MockDiagnosticConsumer consumer;
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
Diagnostics::Kind::HexadecimalEscapeMissingDigits,
Diagnostics::Level::Error, 1, 9, _)));
compile_helper_.GetTokenizedBuffer(R"("hello\xabworld")", &consumer);
}
TEST_F(LexerTest, DiagnosticInvalidDigit) {
Testing::MockDiagnosticConsumer consumer;
EXPECT_CALL(consumer,
HandleDiagnostic(IsSingleDiagnostic(
Diagnostics::Kind::InvalidDigit, Diagnostics::Level::Error, 1,
6, HasSubstr("'a'"))));
compile_helper_.GetTokenizedBuffer("0x123abc", &consumer);
}
TEST_F(LexerTest, DiagnosticCR) {
Testing::MockDiagnosticConsumer consumer;
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
Diagnostics::Kind::UnsupportedCrLineEnding,
Diagnostics::Level::Error, 1, 1, _)));
compile_helper_.GetTokenizedBuffer("\r", &consumer);
}
TEST_F(LexerTest, DiagnosticLfCr) {
Testing::MockDiagnosticConsumer consumer;
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
Diagnostics::Kind::UnsupportedLfCrLineEnding,
Diagnostics::Level::Error, 2, 1, _)));
compile_helper_.GetTokenizedBuffer("\n\r", &consumer);
}
TEST_F(LexerTest, DiagnosticMissingTerminator) {
Testing::MockDiagnosticConsumer consumer;
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
Diagnostics::Kind::UnterminatedString,
Diagnostics::Level::Error, 1, 1, _)));
compile_helper_.GetTokenizedBuffer(R"(#" ")", &consumer);
}
TEST_F(LexerTest, DiagnosticUnrecognizedChar) {
Testing::MockDiagnosticConsumer consumer;
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
Diagnostics::Kind::UnrecognizedCharacters,
Diagnostics::Level::Error, 1, 1, _)));
compile_helper_.GetTokenizedBuffer("\b", &consumer);
}
TEST_F(LexerTest, DiagnosticFileTooLarge) {
Testing::MockDiagnosticConsumer consumer;
static constexpr size_t NumLines = 10'000'000;
std::string input;
input.reserve(NumLines * 3);
for ([[maybe_unused]] auto _ : llvm::seq(NumLines)) {
input += "{}\n";
}
EXPECT_CALL(consumer,
HandleDiagnostic(IsSingleDiagnostic(
Diagnostics::Kind::TooManyTokens, Diagnostics::Level::Error,
TokenIndex::Max / 2, 1, _)));
compile_helper_.GetTokenizedBuffer(input, &consumer);
}
// Outputs comments to the stream to create a comment block.
static auto AppendCommentLines(RawStringOstream& out, int count,
llvm::StringRef tag) -> void {
for (int i : llvm::seq(count)) {
out << "// " << tag << i << "\n";
}
}
TEST_F(LexerTest, CommentBlock) {
for (int comments_before = 0; comments_before < 5; ++comments_before) {
RawStringOstream prefix;
AppendCommentLines(prefix, comments_before, "B");
std::string prefix_out = prefix.TakeStr();
for (int comments_after = 1; comments_after < 5; ++comments_after) {
RawStringOstream source;
source << prefix_out;
if (comments_before > 0) {
source << "//\n";
}
AppendCommentLines(source, comments_after, "C");
SCOPED_TRACE(llvm::formatv(
"{0} comment lines before the empty comment line, {1} after",
comments_before, comments_after));
auto& buffer = compile_helper_.GetTokenizedBuffer(source.TakeStr());
ASSERT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer.comments_size(), Eq(1));
}
}
}
TEST_F(LexerTest, IndentedComments) {
for (int indent = 0; indent < 40; ++indent) {
SCOPED_TRACE(llvm::formatv("Indent: {0}", indent));
RawStringOstream source;
source.indent(indent);
source << "// Comment\n";
std::string source_str = source.TakeStr();
auto& buffer = compile_helper_.GetTokenizedBuffer(source_str);
ASSERT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer.comments_size(), Eq(1));
std::string simd_source =
source_str +
"\"Add a bunch of padding so that SIMD logic shouldn't hit EOF\"";
auto& simd_buffer = compile_helper_.GetTokenizedBuffer(simd_source);
ASSERT_FALSE(simd_buffer.has_errors());
EXPECT_THAT(simd_buffer.comments_size(), Eq(1));
}
}
TEST_F(LexerTest, MultipleComments) {
// TODO: Switch format to `llvm::StringLiteral` if
// `llvm::StringLiteral::c_str` is added.
constexpr char Format[] = R"(
{0}
{1}
{2}
{3}
'''This is a string, not a comment. The next comment will stop SIMD due to being
too close to the EOF.
'''
{4}
x
)";
constexpr llvm::StringLiteral Groups[] = {
// NOLINTNEXTLINE(bugprone-suspicious-missing-comma)
"// This comment should be possible to parse with SIMD.\n"
"// This one too.\n",
"// This one as well, though it's a different indent.\n"
" // And mixes indent.\n"
" // And mixes indent more.\n",
"// This is several comments:\n"
"//Invalid\n"
"// Valid\n"
"//Invalid\n"
"//\n"
"// Valid\n"
"//\n"
"// Valid\n",
"// This uses a high indent, which stops SIMD.\n", "//\n"};
std::string source = llvm::formatv(Format, Groups[0], Groups[1], Groups[2],
Groups[3], Groups[4])
.str();
auto& buffer = compile_helper_.GetTokenizedBuffer(source);
EXPECT_TRUE(buffer.has_errors());
// The third group splits at each transition between ordinary and invalid
// comment introducers; the other groups each form one comment.
constexpr llvm::StringLiteral ExpectedComments[] = {
// NOLINTNEXTLINE(bugprone-suspicious-missing-comma)
"// This comment should be possible to parse with SIMD.\n"
"// This one too.\n",
"// This one as well, though it's a different indent.\n"
" // And mixes indent.\n"
" // And mixes indent more.\n",
"// This is several comments:\n", "//Invalid\n", "// Valid\n",
"//Invalid\n",
// NOLINTNEXTLINE(bugprone-suspicious-missing-comma)
"//\n"
"// Valid\n"
"//\n"
"// Valid\n",
"// This uses a high indent, which stops SIMD.\n", "//\n"};
EXPECT_THAT(buffer.comments_size(), Eq(std::size(ExpectedComments)));
for (int i : llvm::seq(std::min<int>(buffer.comments_size(),
std::size(ExpectedComments)))) {
EXPECT_THAT(buffer.GetCommentText(CommentIndex(i)).str(),
testing::StrEq(ExpectedComments[i]));
}
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::StringLiteral},
{.kind = TokenKind::Identifier},
{.kind = TokenKind::FileEnd},
}));
}
TEST_F(LexerTest, PrintingOutputYaml) {
// Test that we can parse this into YAML and verify line and indent data.
auto& buffer =
compile_helper_.GetTokenizedBuffer("\n ;\n\n\n; ;\n\n\n\n\n\n\n\n\n\n\n");
ASSERT_FALSE(buffer.has_errors());
RawStringOstream print_stream;
buffer.Print(print_stream);
EXPECT_THAT(
Yaml::Value::FromText(print_stream.TakeStr()),
IsYaml(ElementsAre(Yaml::Sequence(ElementsAre(Yaml::Mapping(ElementsAre(
Pair("filename", buffer.source().filename().str()),
Pair("tokens", Yaml::Sequence(ElementsAre(
Yaml::Mapping(ElementsAre(
Pair("index", "0"), Pair("kind", "FileStart"),
Pair("line", "1"), Pair("column", "1"),
Pair("indent", "1"), Pair("spelling", ""))),
Yaml::Mapping(ElementsAre(
Pair("index", "1"), Pair("kind", "Semi"),
Pair("line", "2"), Pair("column", "2"),
Pair("indent", "2"), Pair("spelling", ";"),
Pair("has_leading_space", "true"))),
Yaml::Mapping(ElementsAre(
Pair("index", "2"), Pair("kind", "Semi"),
Pair("line", "5"), Pair("column", "1"),
Pair("indent", "1"), Pair("spelling", ";"),
Pair("has_leading_space", "true"))),
Yaml::Mapping(ElementsAre(
Pair("index", "3"), Pair("kind", "Semi"),
Pair("line", "5"), Pair("column", "3"),
Pair("indent", "1"), Pair("spelling", ";"),
Pair("has_leading_space", "true"))),
Yaml::Mapping(ElementsAre(
Pair("index", "4"), Pair("kind", "FileEnd"),
Pair("line", "15"), Pair("column", "1"),
Pair("indent", "1"), Pair("spelling", ""),
Pair("has_leading_space", "true")))))))))))));
}
} // namespace
} // namespace Carbon::Lex
Reference in new issue View Git Blame Copy Permalink