carbon-language/carbon-lang
1
0
Fork 0
mirror of https://github.com/carbon-language/carbon-lang.git synced 2026-10-05 12:51:04 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Initial lexing support for integer literals following #143. (#269)

Browse Source
This commit is contained in:
Richard Smith
2021-02-19 14:01:55 -08:00
committed by GitHub
parent d16e042950
commit 7670b6377c
2 changed files with 364 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lexer/tokenized_buffer_test.cpp
+127 -1
View File
@@ -77,7 +77,7 @@ TEST_F(LexerTest, TracksLinesAndColumns) {
}
TEST_F(LexerTest, HandlesIntegerLiteral) {
auto buffer = Lex("12-578\n 1 2");
auto buffer = Lex("12-578\n 1 2\n0x12_3ABC\n0b10_10_11\n1_234_567");
EXPECT_FALSE(buffer.HasErrors());
ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::IntegerLiteral(),
@@ -104,6 +104,21 @@ TEST_F(LexerTest, HandlesIntegerLiteral) {
.column = 6,
.indent_column = 3,
.text = "2"},
{.kind = TokenKind::IntegerLiteral(),
.line = 3,
.column = 1,
.indent_column = 1,
.text = "0x12_3ABC"},
{.kind = TokenKind::IntegerLiteral(),
.line = 4,
.column = 1,
.indent_column = 1,
.text = "0b10_10_11"},
{.kind = TokenKind::IntegerLiteral(),
.line = 5,
.column = 1,
.indent_column = 1,
.text = "1_234_567"},
}));
auto token_12 = buffer.Tokens().begin();
EXPECT_EQ(buffer.GetIntegerLiteral(*token_12), 12);
@@ -113,6 +128,117 @@ TEST_F(LexerTest, HandlesIntegerLiteral) {
EXPECT_EQ(buffer.GetIntegerLiteral(*token_1), 1);
auto token_2 = buffer.Tokens().begin() + 4;
EXPECT_EQ(buffer.GetIntegerLiteral(*token_2), 2);
auto token_0x12_3abc = buffer.Tokens().begin() + 5;
EXPECT_EQ(buffer.GetIntegerLiteral(*token_0x12_3abc), 0x12'3abc);
auto token_0b10_10_11 = buffer.Tokens().begin() + 6;
EXPECT_EQ(buffer.GetIntegerLiteral(*token_0b10_10_11), 0b10'10'11);
auto token_1_234_567 = buffer.Tokens().begin() + 7;
EXPECT_EQ(buffer.GetIntegerLiteral(*token_1_234_567), 1'234'567);
}
TEST_F(LexerTest, ValidatesBaseSpecifier) {
llvm::StringLiteral valid[] = {
// Decimal integer literals.
"0",
"1",
"123456789000000000000000000000000000000000000",
// Hexadecimal integer literals.
"0x0123456789ABCDEF",
"0x0000000000000000000000000000000",
// Binary integer literals.
"0b10110100101001010",
"0b0000000",
};
for (llvm::StringLiteral literal : valid) {
auto buffer = Lex(literal);
EXPECT_FALSE(buffer.HasErrors()) << literal;
ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::IntegerLiteral(),
.line = 1,
.column = 1,
.indent_column = 1,
.text = literal}}));
}
llvm::StringLiteral invalid[] = {
"00", "0X123", "0o123", "0B1",
"007", "123L", "123456789A", "0x",
"0b", "0x123abc", "0b011101201001", "0b10A",
};
for (llvm::StringLiteral literal : invalid) {
auto buffer = Lex(literal);
EXPECT_TRUE(buffer.HasErrors()) << literal;
ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::Error(),
.line = 1,
.column = 1,
.indent_column = 1,
.text = literal}}));
}
}
TEST_F(LexerTest, ValidatesIntegerDigitSeparators) {
llvm::StringLiteral valid[] = {
// Decimal literals optionally have digit separators every 3 places.
"1_234",
"123_456",
"1_234_567",
// Hexadecimal literals optionally have digit separators every 4 places.
"0x1_0000",
"0x1000_0000",
"0x1_0000_0000",
// Binary integer literals can have digit separators anywhere..
"0b1_0_1_0_1_0",
"0b111_0000",
};
for (llvm::StringLiteral literal : valid) {
auto buffer = Lex(literal);
EXPECT_FALSE(buffer.HasErrors()) << literal;
ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::IntegerLiteral(),
.line = 1,
.column = 1,
.indent_column = 1,
.text = literal}}));
}
llvm::StringLiteral invalid[] = {
// Decimal literals.
"12_34",
"123_4_6_789",
"12_3456_789",
"12__345",
"1_",
// Hexadecimal literals.
"0x_1234",
"0x123_",
"0x12_3",
"0x_234_5678",
"0x1234_567",
// Binary literals.
"0b_10101",
"0b1__01",
"0b1011_",
"0b1_01_01_",
};
for (llvm::StringLiteral literal : invalid) {
auto buffer = Lex(literal);
EXPECT_TRUE(buffer.HasErrors()) << literal;
// We expect to produce a token even for a literal containing invalid digit
// separators, for better error recovery.
ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::IntegerLiteral(),
.line = 1,
.column = 1,
.indent_column = 1,
.text = literal}}));
}
}
TEST_F(LexerTest, HandlesGarbageCharacters) {