mirror of
https://github.com/carbon-language/carbon-lang.git
synced 2026-11-15 22:38:10 +00:00
Move numeric literal parsing out into its own files. (#326)
This commit is contained in:
Richard Smith authored and 
GitHub committed
1 parent
5884aa1a5c
commit
9a873b46de
6 files changed
+855
-717
No files matched your search
+37
-261
@@ -77,8 +77,8 @@ TEST_F(LexerTest, TracksLinesAndColumns) {
|
||||
}));
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, HandlesIntegerLiteral) {
|
||||
auto buffer = Lex("12-578\n 1 2\n0x12_3ABC\n0b10_10_11\n1_234_567");
|
||||
TEST_F(LexerTest, HandlesNumericLiteral) {
|
||||
auto buffer = Lex("12-578\n 1 2\n0x12_3ABC\n0b10_10_11\n1_234_567\n1.5e9");
|
||||
EXPECT_FALSE(buffer.HasErrors());
|
||||
ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::IntegerLiteral(),
|
||||
@@ -120,6 +120,11 @@ TEST_F(LexerTest, HandlesIntegerLiteral) {
|
||||
.column = 1,
|
||||
.indent_column = 1,
|
||||
.text = "1_234_567"},
|
||||
{.kind = TokenKind::RealLiteral(),
|
||||
.line = 6,
|
||||
.column = 1,
|
||||
.indent_column = 1,
|
||||
.text = "1.5e9"},
|
||||
}));
|
||||
auto token_12 = buffer.Tokens().begin();
|
||||
EXPECT_EQ(buffer.GetIntegerLiteral(*token_12), 12);
|
||||
@@ -135,272 +140,43 @@ TEST_F(LexerTest, HandlesIntegerLiteral) {
|
||||
EXPECT_EQ(buffer.GetIntegerLiteral(*token_0b10_10_11), 0b10'10'11);
|
||||
auto token_1_234_567 = buffer.Tokens().begin() + 7;
|
||||
EXPECT_EQ(buffer.GetIntegerLiteral(*token_1_234_567), 1'234'567);
|
||||
auto token_1_5e9 = buffer.Tokens().begin() + 8;
|
||||
auto value_1_5e9 = buffer.GetRealLiteral(*token_1_5e9);
|
||||
EXPECT_EQ(value_1_5e9.Mantissa().getZExtValue(), 15);
|
||||
EXPECT_EQ(value_1_5e9.Exponent().getSExtValue(), 8);
|
||||
EXPECT_EQ(value_1_5e9.IsDecimal(), true);
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, ValidatesBaseSpecifier) {
|
||||
llvm::StringLiteral valid[] = {
|
||||
// Decimal integer literals.
|
||||
"0",
|
||||
"1",
|
||||
"123456789000000000000000000000000000000000000",
|
||||
|
||||
// Hexadecimal integer literals.
|
||||
"0x0123456789ABCDEF",
|
||||
"0x0000000000000000000000000000000",
|
||||
|
||||
// Binary integer literals.
|
||||
"0b10110100101001010",
|
||||
"0b0000000",
|
||||
};
|
||||
for (llvm::StringLiteral literal : valid) {
|
||||
auto buffer = Lex(literal);
|
||||
EXPECT_FALSE(buffer.HasErrors()) << literal;
|
||||
ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::IntegerLiteral(),
|
||||
.line = 1,
|
||||
.column = 1,
|
||||
.indent_column = 1,
|
||||
.text = literal}}));
|
||||
}
|
||||
|
||||
llvm::StringLiteral invalid[] = {
|
||||
"00", "0X123", "0o123", "0B1",
|
||||
"007", "123L", "123456789A", "0x",
|
||||
"0b", "0x123abc", "0b011101201001", "0b10A",
|
||||
"0x_", "0b_",
|
||||
};
|
||||
for (llvm::StringLiteral literal : invalid) {
|
||||
auto buffer = Lex(literal);
|
||||
EXPECT_TRUE(buffer.HasErrors()) << literal;
|
||||
ASSERT_THAT(
|
||||
buffer,
|
||||
HasTokens(llvm::ArrayRef<ExpectedToken>{{.kind = TokenKind::Error(),
|
||||
.line = 1,
|
||||
.column = 1,
|
||||
.indent_column = 1,
|
||||
.text = literal}}));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, ValidatesIntegerDigitSeparators) {
|
||||
llvm::StringLiteral valid[] = {
|
||||
// Decimal literals optionally have digit separators every 3 places.
|
||||
"1_234",
|
||||
"123_456",
|
||||
"1_234_567",
|
||||
|
||||
// Hexadecimal literals optionally have digit separators every 4 places.
|
||||
"0x1_0000",
|
||||
"0x1000_0000",
|
||||
"0x1_0000_0000",
|
||||
|
||||
// Binary integer literals can have digit separators anywhere..
|
||||
"0b1_0_1_0_1_0",
|
||||
"0b111_0000",
|
||||
};
|
||||
for (llvm::StringLiteral literal : valid) {
|
||||
auto buffer = Lex(literal);
|
||||
EXPECT_FALSE(buffer.HasErrors()) << literal;
|
||||
ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::IntegerLiteral(),
|
||||
.line = 1,
|
||||
.column = 1,
|
||||
.indent_column = 1,
|
||||
.text = literal}}));
|
||||
}
|
||||
|
||||
llvm::StringLiteral invalid[] = {
|
||||
// Decimal literals.
|
||||
"12_34",
|
||||
"123_4_6_789",
|
||||
"12_3456_789",
|
||||
"12__345",
|
||||
"1_",
|
||||
|
||||
// Hexadecimal literals.
|
||||
"0x_1234",
|
||||
"0x123_",
|
||||
"0x12_3",
|
||||
"0x_234_5678",
|
||||
"0x1234_567",
|
||||
|
||||
// Binary literals.
|
||||
"0b_10101",
|
||||
"0b1__01",
|
||||
"0b1011_",
|
||||
"0b1_01_01_",
|
||||
};
|
||||
for (llvm::StringLiteral literal : invalid) {
|
||||
auto buffer = Lex(literal);
|
||||
EXPECT_TRUE(buffer.HasErrors()) << literal;
|
||||
// We expect to produce a token even for a literal containing invalid digit
|
||||
// separators, for better error recovery.
|
||||
ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::IntegerLiteral(),
|
||||
.line = 1,
|
||||
.column = 1,
|
||||
.indent_column = 1,
|
||||
.text = literal}}));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, HandlesRealLiteral) {
|
||||
struct Testcase {
|
||||
llvm::StringLiteral token;
|
||||
uint64_t mantissa;
|
||||
int64_t exponent;
|
||||
unsigned radix;
|
||||
};
|
||||
Testcase testcases[] = {
|
||||
// Decimal real literals.
|
||||
{.token = "0.0", .mantissa = 0, .exponent = -1, .radix = 10},
|
||||
{.token = "12.345", .mantissa = 12345, .exponent = -3, .radix = 10},
|
||||
{.token = "12.345e6", .mantissa = 12345, .exponent = 3, .radix = 10},
|
||||
{.token = "12.345e+6", .mantissa = 12345, .exponent = 3, .radix = 10},
|
||||
{.token = "1_234.5e-2", .mantissa = 12345, .exponent = -3, .radix = 10},
|
||||
{.token = "1.0e-2_000_000",
|
||||
.mantissa = 10,
|
||||
.exponent = -2'000'001,
|
||||
.radix = 10},
|
||||
|
||||
// Hexadecimal real literals.
|
||||
{.token = "0x1_2345_6789.CDEF",
|
||||
.mantissa = 0x1'2345'6789'CDEF,
|
||||
.exponent = -16,
|
||||
.radix = 16},
|
||||
{.token = "0x0.0001p4", .mantissa = 1, .exponent = -12, .radix = 16},
|
||||
{.token = "0x0.0001p+4", .mantissa = 1, .exponent = -12, .radix = 16},
|
||||
{.token = "0x0.0001p-4", .mantissa = 1, .exponent = -20, .radix = 16},
|
||||
// The exponent here works out as exactly INT64_MIN.
|
||||
{.token = "0x1.01p-9223372036854775800",
|
||||
.mantissa = 0x101,
|
||||
.exponent = -9223372036854775807L - 1L,
|
||||
.radix = 16},
|
||||
// The exponent here doesn't fit in a signed 64-bit integer until we
|
||||
// adjust for the radix point.
|
||||
{.token = "0x1.01p9223372036854775809",
|
||||
.mantissa = 0x101,
|
||||
.exponent = 9223372036854775801L,
|
||||
.radix = 16},
|
||||
|
||||
// Binary real literals. These are invalid, but we accept them for error
|
||||
// recovery.
|
||||
{.token = "0b10_11_01.01",
|
||||
.mantissa = 0b10110101,
|
||||
.exponent = -2,
|
||||
.radix = 2},
|
||||
};
|
||||
for (Testcase testcase : testcases) {
|
||||
auto buffer = Lex(testcase.token);
|
||||
EXPECT_EQ(buffer.HasErrors(), testcase.radix == 2);
|
||||
ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::RealLiteral(),
|
||||
.line = 1,
|
||||
.column = 1,
|
||||
.indent_column = 1,
|
||||
.text = testcase.token},
|
||||
}));
|
||||
auto token = buffer.Tokens().begin();
|
||||
TokenizedBuffer::RealLiteralValue value = buffer.GetRealLiteral(*token);
|
||||
EXPECT_EQ(value.Mantissa().getZExtValue(), testcase.mantissa);
|
||||
EXPECT_EQ(value.Exponent().getSExtValue(), testcase.exponent);
|
||||
EXPECT_EQ(value.IsDecimal(), testcase.radix == 10);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, HandlesRealLiteralOverflow) {
|
||||
llvm::StringLiteral input = "0x1.000001p-9223372036854775800";
|
||||
auto buffer = Lex(input);
|
||||
EXPECT_FALSE(buffer.HasErrors());
|
||||
TEST_F(LexerTest, HandlesInvalidNumericLiterals) {
|
||||
auto buffer = Lex("14x 15_49 0x3.5q 0x3_4.5_6 0ops");
|
||||
EXPECT_TRUE(buffer.HasErrors());
|
||||
ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::RealLiteral(),
|
||||
{.kind = TokenKind::Error(),
|
||||
.line = 1,
|
||||
.column = 1,
|
||||
.indent_column = 1,
|
||||
.text = input},
|
||||
.text = "14x"},
|
||||
{.kind = TokenKind::IntegerLiteral(),
|
||||
.line = 1,
|
||||
.column = 5,
|
||||
.indent_column = 1,
|
||||
.text = "15_49"},
|
||||
{.kind = TokenKind::Error(),
|
||||
.line = 1,
|
||||
.column = 11,
|
||||
.indent_column = 1,
|
||||
.text = "0x3.5q"},
|
||||
{.kind = TokenKind::RealLiteral(),
|
||||
.line = 1,
|
||||
.column = 18,
|
||||
.indent_column = 1,
|
||||
.text = "0x3_4.5_6"},
|
||||
{.kind = TokenKind::Error(),
|
||||
.line = 1,
|
||||
.column = 28,
|
||||
.indent_column = 1,
|
||||
.text = "0ops"},
|
||||
}));
|
||||
auto token = buffer.Tokens().begin();
|
||||
TokenizedBuffer::RealLiteralValue value = buffer.GetRealLiteral(*token);
|
||||
EXPECT_EQ(value.Mantissa(), 0x1000001);
|
||||
EXPECT_EQ((value.Exponent() + 9223372036854775800).getSExtValue(), -24);
|
||||
EXPECT_EQ(value.IsDecimal(), false);
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, ValidatesRealLiterals) {
|
||||
llvm::StringLiteral invalid_digit_separators[] = {
|
||||
// Invalid digit separators.
|
||||
"12_34.5", "123.4_567", "123.456_7", "1_2_3.4",
|
||||
"123.4e56_78", "0x12_34.5", "0x12.3_4", "0x12.34p5_6",
|
||||
};
|
||||
for (llvm::StringLiteral literal : invalid_digit_separators) {
|
||||
auto buffer = Lex(literal);
|
||||
EXPECT_TRUE(buffer.HasErrors()) << literal;
|
||||
// We expect to produce a token even for a literal containing invalid digit
|
||||
// separators, for better error recovery.
|
||||
ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
||||
{.kind = TokenKind::RealLiteral(),
|
||||
.line = 1,
|
||||
.column = 1,
|
||||
.indent_column = 1,
|
||||
.text = literal}}));
|
||||
}
|
||||
|
||||
llvm::StringLiteral invalid[] = {
|
||||
// No digits in integer part.
|
||||
"0x.0",
|
||||
"0b.0",
|
||||
"0x_.0",
|
||||
"0b_.0",
|
||||
|
||||
// No digits in fractional part.
|
||||
"0.e",
|
||||
"0.e0",
|
||||
"0.e+0",
|
||||
"0x0.p",
|
||||
"0x0.p-0",
|
||||
|
||||
// Invalid digits in mantissa.
|
||||
"123A.4",
|
||||
"123.4A",
|
||||
"123A.4e0",
|
||||
"123.4Ae0",
|
||||
"0x123ABCDEFG.0",
|
||||
"0x123.ABCDEFG",
|
||||
"0x123ABCDEFG.0p0",
|
||||
"0x123.ABCDEFGp0",
|
||||
|
||||
// Invalid exponent letter.
|
||||
"0.0f0",
|
||||
"0.0p0",
|
||||
"0.0z+0",
|
||||
"0x0.0e0",
|
||||
"0x0.0f0",
|
||||
"0x0.0z-0",
|
||||
|
||||
// No digits in exponent part.
|
||||
"0.0e",
|
||||
"0x0.0p",
|
||||
"0.0e_",
|
||||
"0x0.0p_",
|
||||
|
||||
// Invalid digits in exponent part.
|
||||
"0.0eHELLO",
|
||||
"0.0eA",
|
||||
"0.0e+A",
|
||||
"0x0.0pA",
|
||||
"0x0.0p-A",
|
||||
};
|
||||
for (llvm::StringLiteral literal : invalid) {
|
||||
auto buffer = Lex(literal);
|
||||
EXPECT_TRUE(buffer.HasErrors()) << literal;
|
||||
ASSERT_THAT(
|
||||
buffer,
|
||||
HasTokens(llvm::ArrayRef<ExpectedToken>{{.kind = TokenKind::Error(),
|
||||
.line = 1,
|
||||
.column = 1,
|
||||
.indent_column = 1,
|
||||
.text = literal}}));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, SplitsNumericLiteralsProperly) {
|
||||
|
||||
Reference in new issue
Block a user