Initial lexing support for real literals following #143. (#273)

2026-10-04 22:02:52 +01:00 · 2021-02-26 17:34:01 -08:00
parent a09693b38d
commit a8e4a69328
4 changed files with 751 additions and 178 deletions
@@ -39,7 +39,8 @@ struct LexerTest : ::testing::Test {

  auto Lex(llvm::Twine text) -> TokenizedBuffer {
    // TODO: build a full mock for this.
-    return TokenizedBuffer::Lex(GetSourceBuffer(text), NullDiagnosticEmitter());
+    return TokenizedBuffer::Lex(GetSourceBuffer(text),
+                                ConsoleDiagnosticEmitter());
  }
 };

@@ -166,6 +167,7 @@ TEST_F(LexerTest, ValidatesBaseSpecifier) {
      "00",  "0X123",    "0o123",          "0B1",
      "007", "123L",     "123456789A",     "0x",
      "0b",  "0x123abc", "0b011101201001", "0b10A",
+      "0x_", "0b_",
  };
  for (llvm::StringLiteral literal : invalid) {
    auto buffer = Lex(literal);
@@ -242,6 +244,235 @@ TEST_F(LexerTest, ValidatesIntegerDigitSeparators) {
  }
 }

+TEST_F(LexerTest, HandlesRealLiteral) {
+  struct Testcase {
+    llvm::StringLiteral token;
+    uint64_t mantissa;
+    int64_t exponent;
+    unsigned radix;
+  };
+  Testcase testcases[] = {
+      // Decimal real literals.
+      {.token = "0.0", .mantissa = 0, .exponent = -1, .radix = 10},
+      {.token = "12.345", .mantissa = 12345, .exponent = -3, .radix = 10},
+      {.token = "12.345e6", .mantissa = 12345, .exponent = 3, .radix = 10},
+      {.token = "12.345e+6", .mantissa = 12345, .exponent = 3, .radix = 10},
+      {.token = "1_234.5e-2", .mantissa = 12345, .exponent = -3, .radix = 10},
+      {.token = "1.0e-2_000_000",
+       .mantissa = 10,
+       .exponent = -2'000'001,
+       .radix = 10},
+
+      // Hexadecimal real literals.
+      {.token = "0x1_2345_6789.CDEF",
+       .mantissa = 0x1'2345'6789'CDEF,
+       .exponent = -16,
+       .radix = 16},
+      {.token = "0x0.0001p4", .mantissa = 1, .exponent = -12, .radix = 16},
+      {.token = "0x0.0001p+4", .mantissa = 1, .exponent = -12, .radix = 16},
+      {.token = "0x0.0001p-4", .mantissa = 1, .exponent = -20, .radix = 16},
+      // The exponent here works out as exactly INT64_MIN.
+      {.token = "0x1.01p-9223372036854775800",
+       .mantissa = 0x101,
+       .exponent = -9223372036854775807L - 1L,
+       .radix = 16},
+      // The exponent here doesn't fit in a signed 64-bit integer until we
+      // adjust for the radix point.
+      {.token = "0x1.01p9223372036854775809",
+       .mantissa = 0x101,
+       .exponent = 9223372036854775801L,
+       .radix = 16},
+
+      // Binary real literals. These are invalid, but we accept them for error
+      // recovery.
+      {.token = "0b10_11_01.01",
+       .mantissa = 0b10110101,
+       .exponent = -2,
+       .radix = 2},
+  };
+  for (Testcase testcase : testcases) {
+    auto buffer = Lex(testcase.token);
+    EXPECT_EQ(buffer.HasErrors(), testcase.radix == 2);
+    ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
+                            {.kind = TokenKind::RealLiteral(),
+                             .line = 1,
+                             .column = 1,
+                             .indent_column = 1,
+                             .text = testcase.token},
+                        }));
+    auto token = buffer.Tokens().begin();
+    TokenizedBuffer::RealLiteralValue value = buffer.GetRealLiteral(*token);
+    EXPECT_EQ(value.Mantissa().getZExtValue(), testcase.mantissa);
+    EXPECT_EQ(value.Exponent().getSExtValue(), testcase.exponent);
+    EXPECT_EQ(value.IsDecimal(), testcase.radix == 10);
+  }
+}
+
+TEST_F(LexerTest, HandlesRealLiteralOverflow) {
+  llvm::StringLiteral input = "0x1.000001p-9223372036854775800";
+  auto buffer = Lex(input);
+  EXPECT_FALSE(buffer.HasErrors());
+  ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
+                          {.kind = TokenKind::RealLiteral(),
+                           .line = 1,
+                           .column = 1,
+                           .indent_column = 1,
+                           .text = input},
+                      }));
+  auto token = buffer.Tokens().begin();
+  TokenizedBuffer::RealLiteralValue value = buffer.GetRealLiteral(*token);
+  EXPECT_EQ(value.Mantissa(), 0x1000001);
+  EXPECT_EQ((value.Exponent() + 9223372036854775800).getSExtValue(), -24);
+  EXPECT_EQ(value.IsDecimal(), false);
+}
+
+TEST_F(LexerTest, ValidatesRealLiterals) {
+  llvm::StringLiteral invalid_digit_separators[] = {
+      // Invalid digit separators.
+      "12_34.5",     "123.4_567", "123.456_7", "1_2_3.4",
+      "123.4e56_78", "0x12_34.5", "0x12.3_4",  "0x12.34p5_6",
+  };
+  for (llvm::StringLiteral literal : invalid_digit_separators) {
+    auto buffer = Lex(literal);
+    EXPECT_TRUE(buffer.HasErrors()) << literal;
+    // We expect to produce a token even for a literal containing invalid digit
+    // separators, for better error recovery.
+    ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
+                            {.kind = TokenKind::RealLiteral(),
+                             .line = 1,
+                             .column = 1,
+                             .indent_column = 1,
+                             .text = literal}}));
+  }
+
+  llvm::StringLiteral invalid[] = {
+      // No digits in integer part.
+      "0x.0",
+      "0b.0",
+      "0x_.0",
+      "0b_.0",
+
+      // No digits in fractional part.
+      "0.e",
+      "0.e0",
+      "0.e+0",
+      "0x0.p",
+      "0x0.p-0",
+
+      // Invalid digits in mantissa.
+      "123A.4",
+      "123.4A",
+      "123A.4e0",
+      "123.4Ae0",
+      "0x123ABCDEFG.0",
+      "0x123.ABCDEFG",
+      "0x123ABCDEFG.0p0",
+      "0x123.ABCDEFGp0",
+
+      // Invalid exponent letter.
+      "0.0f0",
+      "0.0p0",
+      "0.0z+0",
+      "0x0.0e0",
+      "0x0.0f0",
+      "0x0.0z-0",
+
+      // No digits in exponent part.
+      "0.0e",
+      "0x0.0p",
+      "0.0e_",
+      "0x0.0p_",
+
+      // Invalid digits in exponent part.
+      "0.0eHELLO",
+      "0.0eA",
+      "0.0e+A",
+      "0x0.0pA",
+      "0x0.0p-A",
+  };
+  for (llvm::StringLiteral literal : invalid) {
+    auto buffer = Lex(literal);
+    EXPECT_TRUE(buffer.HasErrors()) << literal;
+    ASSERT_THAT(
+        buffer,
+        HasTokens(llvm::ArrayRef<ExpectedToken>{{.kind = TokenKind::Error(),
+                                                 .line = 1,
+                                                 .column = 1,
+                                                 .indent_column = 1,
+                                                 .text = literal}}));
+  }
+}
+
+TEST_F(LexerTest, SplitsNumericLiteralsProperly) {
+  llvm::StringLiteral source_text = R"(
+    1.
+    .2
+    3.+foo
+    4.0-bar
+    5.0e+123+456
+    6.0e+1e+2
+    1e7
+    8..10
+    9.0.9.5
+    10.foo
+    11.0.foo
+    12e+1
+    13._
+  )";
+  auto buffer = Lex(source_text);
+  EXPECT_TRUE(buffer.HasErrors());
+  EXPECT_THAT(buffer,
+              HasTokens(llvm::ArrayRef<ExpectedToken>{
+                  {.kind = TokenKind::IntegerLiteral(), .text = "1"},
+                  {.kind = TokenKind::Period()},
+                  // newline
+                  {.kind = TokenKind::Period()},
+                  {.kind = TokenKind::IntegerLiteral(), .text = "2"},
+                  // newline
+                  {.kind = TokenKind::IntegerLiteral(), .text = "3"},
+                  {.kind = TokenKind::Period()},
+                  {.kind = TokenKind::Plus()},
+                  {.kind = TokenKind::Identifier(), .text = "foo"},
+                  // newline
+                  {.kind = TokenKind::RealLiteral(), .text = "4.0"},
+                  {.kind = TokenKind::Minus()},
+                  {.kind = TokenKind::Identifier(), .text = "bar"},
+                  // newline
+                  {.kind = TokenKind::RealLiteral(), .text = "5.0e+123"},
+                  {.kind = TokenKind::Plus()},
+                  {.kind = TokenKind::IntegerLiteral(), .text = "456"},
+                  // newline
+                  {.kind = TokenKind::Error(), .text = "6.0e+1e"},
+                  {.kind = TokenKind::Plus()},
+                  {.kind = TokenKind::IntegerLiteral(), .text = "2"},
+                  // newline
+                  {.kind = TokenKind::Error(), .text = "1e7"},
+                  // newline
+                  {.kind = TokenKind::IntegerLiteral(), .text = "8"},
+                  {.kind = TokenKind::Period()},
+                  {.kind = TokenKind::Period()},
+                  {.kind = TokenKind::IntegerLiteral(), .text = "10"},
+                  // newline
+                  {.kind = TokenKind::RealLiteral(), .text = "9.0"},
+                  {.kind = TokenKind::Period()},
+                  {.kind = TokenKind::RealLiteral(), .text = "9.5"},
+                  // newline
+                  {.kind = TokenKind::Error(), .text = "10.foo"},
+                  // newline
+                  {.kind = TokenKind::RealLiteral(), .text = "11.0"},
+                  {.kind = TokenKind::Period()},
+                  {.kind = TokenKind::Identifier(), .text = "foo"},
+                  // newline
+                  {.kind = TokenKind::Error(), .text = "12e"},
+                  {.kind = TokenKind::Plus()},
+                  {.kind = TokenKind::IntegerLiteral(), .text = "1"},
+                  // newline
+                  {.kind = TokenKind::IntegerLiteral(), .text = "13"},
+                  {.kind = TokenKind::Period()},
+                  {.kind = TokenKind::UnderscoreKeyword()},
+              }));
+}
+
 TEST_F(LexerTest, HandlesGarbageCharacters) {
  constexpr char GarbageText[] = "$$💩-$\n$\0$12$";
  auto buffer = Lex(llvm::StringRef(GarbageText, sizeof(GarbageText) - 1));