Initial lexing support for real literals following #143. (#273)

lexer/token_registry.def

@@ -87,11 +87,11 @@ CARBON_SYMBOL_TOKEN(Tilde,               "~")
 // clang-format on
 
 #ifndef CARBON_OPENING_GROUP_SYMBOL_TOKEN
-#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(Name, Spelling, ClosingName)         \
+#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(Name, Spelling, ClosingName) \
   CARBON_SYMBOL_TOKEN(Name, Spelling)
 #endif
 // clang-format on
-CARBON_OPENING_GROUP_SYMBOL_TOKEN(OpenParen,      "(", CloseParen)
+CARBON_OPENING_GROUP_SYMBOL_TOKEN(OpenParen, "(", CloseParen)
 CARBON_OPENING_GROUP_SYMBOL_TOKEN(OpenCurlyBrace, "{", CloseCurlyBrace)
 // clang-format off
 #undef CARBON_OPENING_GROUP_SYMBOL_TOKEN
@@ -101,7 +101,7 @@ CARBON_OPENING_GROUP_SYMBOL_TOKEN(OpenCurlyBrace, "{", CloseCurlyBrace)
   CARBON_SYMBOL_TOKEN(Name, Spelling)
 #endif
 // clang-format on
-CARBON_CLOSING_GROUP_SYMBOL_TOKEN(CloseParen,      ")", OpenParen)
+CARBON_CLOSING_GROUP_SYMBOL_TOKEN(CloseParen, ")", OpenParen)
 CARBON_CLOSING_GROUP_SYMBOL_TOKEN(CloseCurlyBrace, "}", OpenCurlyBrace)
 // clang-format off
 #undef CARBON_CLOSING_GROUP_SYMBOL_TOKEN
@@ -156,6 +156,7 @@ CARBON_KEYWORD_TOKEN(XorKeyword,        "xor")
 
 CARBON_TOKEN(Identifier)
 CARBON_TOKEN(IntegerLiteral)
+CARBON_TOKEN(RealLiteral)
 CARBON_TOKEN(Error)
 
 #undef CARBON_TOKEN

lexer/tokenized_buffer.cpp

@@ -21,20 +21,6 @@
 
 namespace Carbon {
 
-static auto TakeLeadingIntegerLiteral(llvm::StringRef source_text)
-    -> llvm::StringRef {
-  if (source_text.empty() || !llvm::isDigit(source_text.front()))
-    return llvm::StringRef();
-
-  // Greedily consume all following characters that might be part of an integer
-  // literal. This allows us to produce better diagnostics on invalid literals.
-  //
-  // TODO(zygoloid): Update lexical rules to specify that an integer literal
-  // cannot be immediately followed by another integer literal or a word.
-  return source_text.take_while(
-      [](char c) { return llvm::isAlnum(c) || c == '_'; });
-}
-
 struct TrailingComment {
   static constexpr llvm::StringLiteral ShortName = "syntax-comments";
   static constexpr llvm::StringLiteral Message =
@@ -95,15 +81,16 @@ struct InvalidDigit {
 
   struct Substitutions {
     char digit;
-    unsigned radix;
+    int radix;
   };
   static auto Format(const Substitutions& subst) -> std::string {
     // TODO: Switch Format to using raw_ostream so we can easily use
     // llvm::format here.
     llvm::StringRef digit_str(&subst.digit, 1);
     return (llvm::Twine("Invalid digit '") + digit_str + "' in " +
-            (subst.radix == 2 ? "binary"
-                              : subst.radix == 16 ? "hexadecimal" : "decimal") +
+            (subst.radix == 2    ? "binary"
+             : subst.radix == 16 ? "hexadecimal"
+                                 : "decimal") +
             " numeric literal.")
         .str();
   }
@@ -125,7 +112,7 @@ struct IrregularDigitSeparators {
       "syntax-irregular-digit-separators";
 
   struct Substitutions {
-    unsigned radix;
+    int radix;
   };
   static auto Format(const Substitutions& subst) -> std::string {
     assert((subst.radix == 10 || subst.radix == 16) && "unexpected radix");
@@ -148,6 +135,31 @@ struct UnknownBaseSpecifier {
   }
 };
 
+struct BinaryRealLiteral {
+  static constexpr llvm::StringLiteral ShortName = "syntax-invalid-number";
+  static constexpr llvm::StringLiteral Message =
+      "Binary real number literals are not supported.";
+
+  struct Substitutions {};
+  static auto Format(const Substitutions&) -> std::string {
+    return Message.str();
+  }
+};
+
+struct WrongRealLiteralExponent {
+  static constexpr llvm::StringLiteral ShortName = "syntax-invalid-number";
+
+  struct Substitutions {
+    char expected;
+  };
+  static auto Format(const Substitutions& subst) -> std::string {
+    char expected_str[] = {subst.expected, '\0'};
+    return (llvm::Twine("Expected '") + expected_str +
+            "' to introduce exponent.")
+        .str();
+  }
+};
+
 struct UnrecognizedCharacters {
   static constexpr llvm::StringLiteral ShortName =
       "syntax-unrecognized-characters";
@@ -166,6 +178,390 @@ static bool isSpace(char c) {
   return c == ' ' || c == '\n' || c == '\t';
 }
 
+static bool isLower(char c) { return 'a' <= c && c <= 'z'; }
+
+namespace {
+struct NumericLiteral {
+  llvm::StringRef text;
+
+  // The offset of the '.'. Set to text.size() if none is present.
+  int radix_point;
+
+  // The offset of the alphabetical character introducing the exponent. In a
+  // valid literal, this will be an 'e' or a 'p', and may be followed by a '+'
+  // or a '-', but for error recovery, this may simply be the last lowercase
+  // letter in the invalid token. Always greater than or equal to radix_point.
+  // Set to text.size() if none is present.
+  int exponent;
+};
+}  // namespace
+
+static auto TakeLeadingNumericLiteral(llvm::StringRef source_text)
+    -> NumericLiteral {
+  NumericLiteral result;
+
+  if (source_text.empty() || !llvm::isDigit(source_text.front()))
+    return result;
+
+  bool seen_plus_minus = false;
+  bool seen_radix_point = false;
+  bool seen_potential_exponent = false;
+
+  // Greedily consume all following characters that might be part of a numeric
+  // literal. This allows us to produce better diagnostics on invalid literals.
+  //
+  // TODO(zygoloid): Update lexical rules to specify that a numeric literal
+  // cannot be immediately followed by an alphanumeric character.
+  int i = 1, n = source_text.size();
+  for (; i != n; ++i) {
+    char c = source_text[i];
+    if (llvm::isAlnum(c) || c == '_') {
+      if (isLower(c) && seen_radix_point && !seen_plus_minus) {
+        result.exponent = i;
+        seen_potential_exponent = true;
+      }
+      continue;
+    }
+
+    // Exactly one `.` can be part of the literal, but only if it's followed by
+    // an alphanumeric character.
+    if (c == '.' && i + 1 != n && llvm::isAlnum(source_text[i + 1]) &&
+        !seen_radix_point) {
+      result.radix_point = i;
+      seen_radix_point = true;
+      continue;
+    }
+
+    // A `+` or `-` continues the literal only if it's preceded by a lowercase
+    // letter (which will be 'e' or 'p' or part of an invalid literal) and
+    // followed by an alphanumeric character. This '+' or '-' cannot be an
+    // operator because a literal cannot end in a lowercase letter.
+    if ((c == '+' || c == '-') && seen_potential_exponent &&
+        result.exponent == i - 1 && i + 1 != n &&
+        llvm::isAlnum(source_text[i + 1])) {
+      // This is not possible because we don't update result.exponent after we
+      // see a '+' or '-'.
+      assert(!seen_plus_minus && "should only consume one + or -");
+      seen_plus_minus = true;
+      continue;
+    }
+
+    break;
+  }
+
+  result.text = source_text.substr(0, i);
+  if (!seen_radix_point)
+    result.radix_point = i;
+  if (!seen_potential_exponent)
+    result.exponent = i;
+
+  return result;
+}
+
+namespace {
+// Parser for numeric literal tokens.
+//
+// Responsible for checking that a numeric literal is valid and meaningful and
+// either diagnosing or extracting its meaning.
+class NumericLiteralParser {
+ public:
+  NumericLiteralParser(DiagnosticEmitter& emitter, NumericLiteral literal)
+      : emitter(emitter), literal(literal) {
+    int_part = literal.text.substr(0, literal.radix_point);
+    if (int_part.consume_front("0x")) {
+      radix = 16;
+    } else if (int_part.consume_front("0b")) {
+      radix = 2;
+    }
+
+    fract_part = literal.text.substr(
+        literal.radix_point + 1, literal.exponent - literal.radix_point - 1);
+
+    exponent_part = literal.text.substr(literal.exponent + 1);
+    if (!exponent_part.consume_front("+")) {
+      exponent_is_negative = exponent_part.consume_front("-");
+    }
+  }
+
+  auto IsInteger() -> bool {
+    return literal.radix_point == static_cast<int>(literal.text.size());
+  }
+
+  enum CheckResult {
+    // The token is valid.
+    Valid,
+    // The token is invalid, but we've diagnosed and recovered from the error.
+    RecoverableError,
+    // The token is invalid, and we've diagnosed, but we can't assign meaning
+    // to it.
+    UnrecoverableError,
+  };
+
+  // Check that the numeric literal token is syntactically valid and
+  // meaningful, and diagnose if not.
+  auto Check() -> CheckResult {
+    if (!CheckLeadingZero() || !CheckIntPart() || !CheckFractionalPart() ||
+        !CheckExponentPart())
+      return UnrecoverableError;
+    return recovered_from_error ? RecoverableError : Valid;
+  }
+
+  auto GetMantissa() -> llvm::APInt {
+    const char* end = IsInteger() ? int_part.end() : fract_part.end();
+    llvm::StringRef digits(int_part.begin(), end - int_part.begin());
+    return ParseInteger(digits, radix, mantissa_needs_cleaning);
+  }
+
+  auto GetExponent() -> llvm::APInt {
+    // Compute the effective exponent from the specified exponent, if any,
+    // and the position of the radix point.
+    llvm::APInt exponent(64, 0);
+    if (!exponent_part.empty()) {
+      exponent = ParseInteger(exponent_part, 10, exponent_needs_cleaning);
+
+      // The exponent is a signed integer, and the number we just parsed is
+      // non-negative, so ensure we have a wide enough representation to
+      // include a sign bit. Also make sure the exponent isn't too narrow so
+      // the calculation below can't lose information through overflow.
+      if (exponent.isSignBitSet() || exponent.getBitWidth() < 64) {
+        exponent = exponent.zext(std::max(64u, exponent.getBitWidth() + 1));
+      }
+      if (exponent_is_negative) {
+        exponent.negate();
+      }
+    }
+
+    // Each character after the decimal point reduces the effective exponent.
+    int excess_exponent = fract_part.size();
+    if (radix == 16) {
+      excess_exponent *= 4;
+    }
+    exponent -= excess_exponent;
+    if (exponent_is_negative && !exponent.isNegative()) {
+      // We overflowed. Note that we can only overflow by a little, and only
+      // from negative to positive, because exponent is at least 64 bits wide
+      // and excess_exponent is bounded above by four times the size of the
+      // input buffer, which we assume fits into 32 bits.
+      exponent = exponent.zext(exponent.getBitWidth() + 1);
+      exponent.setSignBit();
+    }
+    return exponent;
+  }
+
+ private:
+  struct CheckDigitSequenceResult {
+    bool ok;
+    bool has_digit_separators = false;
+  };
+
+  // Check that a digit sequence is valid: that it contains one or more digits,
+  // contains only digits in the specified base, and that any digit separators
+  // are present and correctly positioned.
+  auto CheckDigitSequence(llvm::StringRef text, int radix,
+                          bool allow_digit_separators = true)
+      -> CheckDigitSequenceResult {
+    assert((radix == 2 || radix == 10 || radix == 16) && "unknown radix");
+
+    std::bitset<256> valid_digits;
+    if (radix == 2) {
+      for (char c : "01")
+        valid_digits[static_cast<unsigned char>(c)] = true;
+    } else if (radix == 10) {
+      for (char c : "0123456789")
+        valid_digits[static_cast<unsigned char>(c)] = true;
+    } else {
+      for (char c : "0123456789ABCDEF")
+        valid_digits[static_cast<unsigned char>(c)] = true;
+    }
+
+    int num_digit_separators = 0;
+
+    for (int i = 0, n = text.size(); i != n; ++i) {
+      char c = text[i];
+      if (valid_digits[static_cast<unsigned char>(c)]) {
+        continue;
+      }
+
+      if (c == '_') {
+        // A digit separator cannot appear at the start of a digit sequence,
+        // next to another digit separator, or at the end.
+        if (!allow_digit_separators || i == 0 || text[i - 1] == '_' ||
+            i + 1 == n) {
+          emitter.EmitError<InvalidDigitSeparator>(
+              [&](InvalidDigitSeparator::Substitutions&) {});
+          recovered_from_error = true;
+        }
+        ++num_digit_separators;
+        continue;
+      }
+
+      emitter.EmitError<InvalidDigit>([&](InvalidDigit::Substitutions& subst) {
+        subst.digit = c;
+        subst.radix = radix;
+      });
+      return {.ok = false};
+    }
+
+    if (num_digit_separators == static_cast<int>(text.size())) {
+      emitter.EmitError<EmptyDigitSequence>(
+          [&](EmptyDigitSequence::Substitutions&) {});
+      return {.ok = false};
+    }
+
+    // Check that digit separators occur in exactly the expected positions.
+    if (num_digit_separators && radix != 2)
+      CheckDigitSeparatorPlacement(text, radix, num_digit_separators);
+
+    return {.ok = true, .has_digit_separators = (num_digit_separators != 0)};
+  }
+
+  // Given a number with digit separators, check that the digit separators are
+  // correctly positioned.
+  auto CheckDigitSeparatorPlacement(llvm::StringRef text, int radix,
+                                    int num_digit_separators) -> void {
+    assert((radix == 10 || radix == 16) &&
+           "unexpected radix for digit separator checks");
+    assert(std::count(text.begin(), text.end(), '_') == num_digit_separators &&
+           "given wrong number of digit separators");
+
+    auto diagnose_irregular_digit_separators = [&] {
+      emitter.EmitError<IrregularDigitSeparators>(
+          [&](IrregularDigitSeparators::Substitutions& subst) {
+            subst.radix = radix;
+          });
+      recovered_from_error = true;
+    };
+
+    // For decimal and hexadecimal digit sequences, digit separators must form
+    // groups of 3 or 4 digits (4 or 5 characters), respectively.
+    int stride = (radix == 10 ? 4 : 5);
+    int remaining_digit_separators = num_digit_separators;
+    for (auto pos = text.end(); pos - text.begin() >= stride; /*in loop*/) {
+      pos -= stride;
+      if (*pos != '_')
+        return diagnose_irregular_digit_separators();
+
+      --remaining_digit_separators;
+    }
+
+    // Check there weren't any other digit separators.
+    if (remaining_digit_separators)
+      diagnose_irregular_digit_separators();
+  };
+
+  // Check that we don't have a '0' prefix on a non-zero decimal integer.
+  auto CheckLeadingZero() -> bool {
+    if (radix == 10 && int_part.startswith("0") && int_part != "0") {
+      emitter.EmitError<UnknownBaseSpecifier>(
+          [&](UnknownBaseSpecifier::Substitutions& subst) {});
+      return false;
+    }
+    return true;
+  }
+
+  // Check the integer part (before the '.', if any) is valid.
+  auto CheckIntPart() -> bool {
+    auto int_result = CheckDigitSequence(int_part, radix);
+    mantissa_needs_cleaning |= int_result.has_digit_separators;
+    return int_result.ok;
+  }
+
+  // Check the fractional part (after the '.' and before the exponent, if any)
+  // is valid.
+  auto CheckFractionalPart() -> bool {
+    if (IsInteger()) {
+      return true;
+    }
+
+    if (radix == 2) {
+      emitter.EmitError<BinaryRealLiteral>(
+          [&](BinaryRealLiteral::Substitutions& subst) {});
+      recovered_from_error = true;
+      // Carry on and parse the binary real literal anyway.
+    }
+
+    // We need to remove a '.' from the mantissa.
+    mantissa_needs_cleaning = true;
+
+    return CheckDigitSequence(fract_part, radix,
+                              /*allow_digit_separators=*/false)
+        .ok;
+  }
+
+  // Check the exponent part (if any) is valid.
+  auto CheckExponentPart() -> bool {
+    if (literal.exponent == static_cast<int>(literal.text.size())) {
+      return true;
+    }
+
+    char expected_exponent_kind = (radix == 10 ? 'e' : 'p');
+    if (literal.text[literal.exponent] != expected_exponent_kind) {
+      emitter.EmitError<WrongRealLiteralExponent>(
+          [&](WrongRealLiteralExponent::Substitutions& subst) {
+            subst.expected = expected_exponent_kind;
+          });
+      return false;
+    }
+
+    auto exponent_result = CheckDigitSequence(exponent_part, 10);
+    exponent_needs_cleaning = exponent_result.has_digit_separators;
+    return exponent_result.ok;
+  }
+
+  // Parse a string that is known to be a valid base-radix integer into an
+  // APInt.  If needs_cleaning is true, the string may additionally contain '_'
+  // and '.' characters that should be ignored.
+  //
+  // Ignoring '.' is used when parsing a real literal. For example, when
+  // parsing 123.456e7, we want to decompose it into an integer mantissa
+  // (123456) and an exponent (7 - 3 = 2), and this routine is given the
+  // "123.456" to parse as the mantissa.
+  static auto ParseInteger(llvm::StringRef digits, int radix,
+                           bool needs_cleaning) -> llvm::APInt {
+    llvm::SmallString<32> cleaned;
+    if (needs_cleaning) {
+      cleaned.reserve(digits.size());
+      std::remove_copy_if(digits.begin(), digits.end(),
+                          std::back_inserter(cleaned),
+                          [](char c) { return c == '_' || c == '.'; });
+      digits = cleaned;
+    }
+
+    llvm::APInt value;
+    if (digits.getAsInteger(radix, value)) {
+      llvm_unreachable("should never fail");
+    }
+    return value;
+  }
+
+ private:
+  DiagnosticEmitter& emitter;
+  NumericLiteral literal;
+
+  // The radix of the literal: 2, 10, or 16, for a prefix of '0b', no prefix,
+  // or '0x', respectively.
+  int radix = 10;
+
+  // The various components of a numeric literal:
+  //
+  //     [radix] int_part [. fract_part [[ep] [+-] exponent_part]]
+  llvm::StringRef int_part;
+  llvm::StringRef fract_part;
+  llvm::StringRef exponent_part;
+
+  // Do we need to remove any special characters (digit separator or radix
+  // point) before interpreting the mantissa or exponent as an integer?
+  bool mantissa_needs_cleaning = false;
+  bool exponent_needs_cleaning = false;
+
+  // True if we found a `-` before `exponent_part`.
+  bool exponent_is_negative = false;
+
+  // True if we produced an error but recovered.
+  bool recovered_from_error = false;
+};
+}  // namespace
+
 // Implementation of the lexer logic itself.
 //
 // The design is that lexing can loop over the source buffer, consuming it into
@@ -261,170 +657,58 @@ class TokenizedBuffer::Lexer {
     return false;
   }
 
-  auto CheckDigitSeparatorPlacement(llvm::StringRef text, unsigned radix,
-                                    unsigned num_digit_separators) {
-    assert((radix == 10 || radix == 16) &&
-           "unexpected radix for digit separator checks");
-    assert(std::count(text.begin(), text.end(), '_') == num_digit_separators &&
-           "given wrong number of digit separators");
-
-    auto diagnose_irregular_digit_separators = [&] {
-      emitter.EmitError<IrregularDigitSeparators>(
-          [&](IrregularDigitSeparators::Substitutions& subst) {
-            subst.radix = radix;
-          });
-      buffer.has_errors = true;
-    };
-
-    // For decimal and hexadecimal digit sequences, digit separators must form
-    // groups of 3 or 4 digits (4 or 5 characters), respectively.
-    unsigned stride = (radix == 10 ? 4 : 5);
-    unsigned remaining_digit_separators = num_digit_separators;
-    for (auto pos = text.end(); pos - text.begin() >= stride; /*in loop*/) {
-      pos -= stride;
-      if (*pos != '_')
-        return diagnose_irregular_digit_separators();
-
-      --remaining_digit_separators;
-    }
-
-    // Check there weren't any other digit separators.
-    if (remaining_digit_separators)
-      diagnose_irregular_digit_separators();
-  };
-
-  struct CheckDigitSequenceResult {
-    bool ok;
-    bool has_digit_separators = false;
-  };
-
-  auto CheckDigitSequence(llvm::StringRef text, unsigned radix)
-      -> CheckDigitSequenceResult {
-    assert((radix == 2 || radix == 10 || radix == 16) && "unknown radix");
-
-    if (text.empty()) {
-      emitter.EmitError<EmptyDigitSequence>(
-          [&](EmptyDigitSequence::Substitutions&) {});
-      return {.ok = false};
-    }
-
-    std::bitset<256> valid_digits;
-    if (radix == 2) {
-      for (char c : "01")
-        valid_digits[static_cast<unsigned char>(c)] = true;
-    } else if (radix == 10) {
-      for (char c : "0123456789")
-        valid_digits[static_cast<unsigned char>(c)] = true;
-    } else {
-      for (char c : "0123456789ABCDEF")
-        valid_digits[static_cast<unsigned char>(c)] = true;
-    }
-
-    unsigned num_digit_separators = 0;
-
-    for (std::size_t i = 0, n = text.size(); i != n; ++i) {
-      char c = text[i];
-      if (valid_digits[static_cast<unsigned char>(c)]) {
-        continue;
-      }
-
-      if (c == '_') {
-        // A digit separator cannot appear at the start of a digit sequence,
-        // next to another digit separator, or at the end.
-        if (i == 0 || text[i - 1] == '_' || i + 1 == n) {
-          emitter.EmitError<InvalidDigitSeparator>(
-              [&](InvalidDigitSeparator::Substitutions&) {});
-          buffer.has_errors = true;
-        }
-        ++num_digit_separators;
-        continue;
-      }
-
-      emitter.EmitError<InvalidDigit>([&](InvalidDigit::Substitutions& subst) {
-        subst.digit = c;
-        subst.radix = radix;
-      });
-      return {.ok = false};
-    }
-
-    // Check that digit separators occur in exactly the expected positions.
-    if (num_digit_separators && radix != 2)
-      CheckDigitSeparatorPlacement(text, radix, num_digit_separators);
-
-    return {.ok = true, .has_digit_separators = (num_digit_separators != 0)};
-  }
-
-  auto LexIntegerLiteral(llvm::StringRef& source_text) -> bool {
-    llvm::StringRef int_text = TakeLeadingIntegerLiteral(source_text);
-    if (int_text.empty()) {
+  auto LexNumericLiteral(llvm::StringRef& source_text) -> bool {
+    NumericLiteral literal = TakeLeadingNumericLiteral(source_text);
+    if (literal.text.empty()) {
       return false;
     }
 
     int int_column = current_column;
-    current_column += int_text.size();
-    source_text = source_text.drop_front(int_text.size());
+    current_column += literal.text.size();
+    source_text = source_text.drop_front(literal.text.size());
 
     if (!set_indent) {
       current_line_info->indent = int_column;
       set_indent = true;
     }
 
-    auto add_error_token_and_continue_lexing = [&] {
-      buffer.AddToken({
-          .kind = TokenKind::Error(),
-          .token_line = current_line,
-          .column = int_column,
-          .error_length = static_cast<int32_t>(int_text.size()),
-      });
-      buffer.has_errors = true;
-      // Indicate to the caller that we consumed a token.
-      return true;
-    };
+    NumericLiteralParser literal_parser(emitter, literal);
 
-    unsigned radix = 10;
-    llvm::StringRef digits = int_text;
-    if (int_text.size() >= 2 && int_text[0] == '0') {
-      if (int_text[1] == 'x') {
-        radix = 16;
-        digits = digits.drop_front(2);
-      } else if (int_text[1] == 'b') {
-        radix = 2;
-        digits = digits.drop_front(2);
-      } else {
-        emitter.EmitError<UnknownBaseSpecifier>(
-            [&](UnknownBaseSpecifier::Substitutions& subst) {});
-        return add_error_token_and_continue_lexing();
-      }
+    switch (literal_parser.Check()) {
+      case NumericLiteralParser::UnrecoverableError:
+        buffer.AddToken({
+            .kind = TokenKind::Error(),
+            .token_line = current_line,
+            .column = int_column,
+            .error_length = static_cast<int32_t>(literal.text.size()),
+        });
+        buffer.has_errors = true;
+        return true;
+
+      case NumericLiteralParser::RecoverableError:
+        buffer.has_errors = true;
+        break;
+
+      case NumericLiteralParser::Valid:
+        break;
     }
 
-    llvm::APInt int_value;
-
-    auto result = CheckDigitSequence(digits, radix);
-    if (!result.ok) {
-      return add_error_token_and_continue_lexing();
-    }
-
-    if (result.has_digit_separators) {
-      // TODO(zygoloid): Avoid the memory allocation here.
-      std::string cleaned;
-      cleaned.reserve(digits.size());
-      std::remove_copy_if(digits.begin(), digits.end(),
-                          std::back_inserter(cleaned),
-                          [](char c) { return c == '_'; });
-      if (llvm::StringRef(cleaned).getAsInteger(radix, int_value)) {
-        llvm_unreachable("should never fail");
-      }
+    if (literal_parser.IsInteger()) {
+      auto token = buffer.AddToken({.kind = TokenKind::IntegerLiteral(),
+                                    .token_line = current_line,
+                                    .column = int_column});
+      buffer.GetTokenInfo(token).literal_index =
+          buffer.literal_int_storage.size();
+      buffer.literal_int_storage.push_back(literal_parser.GetMantissa());
     } else {
-      if (digits.getAsInteger(radix, int_value)) {
-        llvm_unreachable("should never fail");
-      }
+      auto token = buffer.AddToken({.kind = TokenKind::RealLiteral(),
+                                    .token_line = current_line,
+                                    .column = int_column});
+      buffer.GetTokenInfo(token).literal_index =
+          buffer.literal_int_storage.size();
+      buffer.literal_int_storage.push_back(literal_parser.GetMantissa());
+      buffer.literal_int_storage.push_back(literal_parser.GetExponent());
     }
-
-    auto token = buffer.AddToken({.kind = TokenKind::IntegerLiteral(),
-                                  .token_line = current_line,
-                                  .column = int_column});
-    buffer.GetTokenInfo(token).literal_index = buffer.int_literals.size();
-    buffer.int_literals.push_back(std::move(int_value));
     return true;
   }
 
@@ -619,7 +903,7 @@ auto TokenizedBuffer::Lex(SourceBuffer& source, DiagnosticEmitter& emitter)
     if (lexer.LexKeywordOrIdentifier(source_text)) {
       continue;
     }
-    if (lexer.LexIntegerLiteral(source_text)) {
+    if (lexer.LexNumericLiteral(source_text)) {
       continue;
     }
     lexer.LexError(source_text);
@@ -660,10 +944,11 @@ auto TokenizedBuffer::GetTokenText(Token token) const -> llvm::StringRef {
 
   // Refer back to the source text to preserve oddities like radix or digit
   // separators the author included.
-  if (token_info.kind == TokenKind::IntegerLiteral()) {
+  if (token_info.kind == TokenKind::IntegerLiteral() ||
+      token_info.kind == TokenKind::RealLiteral()) {
     auto& line_info = GetLineInfo(token_info.token_line);
     int64_t token_start = line_info.start + token_info.column;
-    return TakeLeadingIntegerLiteral(source->Text().substr(token_start));
+    return TakeLeadingNumericLiteral(source->Text().substr(token_start)).text;
   }
 
   assert(token_info.kind == TokenKind::Identifier() &&
@@ -678,11 +963,28 @@ auto TokenizedBuffer::GetIdentifier(Token token) const -> Identifier {
   return token_info.id;
 }
 
-auto TokenizedBuffer::GetIntegerLiteral(Token token) const -> llvm::APInt {
+auto TokenizedBuffer::GetIntegerLiteral(Token token) const
+    -> const llvm::APInt& {
   auto& token_info = GetTokenInfo(token);
   assert(token_info.kind == TokenKind::IntegerLiteral() &&
          "The token must be an integer literal!");
-  return int_literals[token_info.literal_index];
+  return literal_int_storage[token_info.literal_index];
+}
+
+auto TokenizedBuffer::GetRealLiteral(Token token) const -> RealLiteralValue {
+  auto& token_info = GetTokenInfo(token);
+  assert(token_info.kind == TokenKind::RealLiteral() &&
+         "The token must be a real literal!");
+
+  // Note that every real literal is at least three characters long, so we can
+  // safely look at the second character to determine whether we have a decimal
+  // or hexadecimal literal.
+  auto& line_info = GetLineInfo(token_info.token_line);
+  int64_t token_start = line_info.start + token_info.column;
+  char second_char = source->Text()[token_start + 1];
+  bool is_decimal = second_char != 'x' && second_char != 'b';
+
+  return RealLiteralValue(this, token_info.literal_index, is_decimal);
 }
 
 auto TokenizedBuffer::GetMatchedClosingToken(Token opening_token) const

lexer/tokenized_buffer.h

@@ -186,6 +186,41 @@ class TokenizedBuffer {
     Token token;
   };
 
+  // The value of a real literal.
+  //
+  // This is either a dyadic fraction (mantissa * 2^exponent) or a decadic
+  // fraction (mantissa * 10^exponent).
+  //
+  // The `TokenizedBuffer` must outlive any `RealLiteralValue`s referring to
+  // its tokens.
+  class RealLiteralValue {
+    const TokenizedBuffer* buffer;
+    int32_t literal_index;
+    bool is_decimal;
+
+   public:
+    // The mantissa, represented as an unsigned integer.
+    const llvm::APInt& Mantissa() const {
+      return buffer->literal_int_storage[literal_index];
+    }
+    // The exponent, represented as a signed integer.
+    const llvm::APInt& Exponent() const {
+      return buffer->literal_int_storage[literal_index + 1];
+    }
+    // If false, the value is mantissa * 2^exponent.
+    // If true, the value is mantissa * 10^exponent.
+    bool IsDecimal() const { return is_decimal; }
+
+   private:
+    friend class TokenizedBuffer;
+
+    RealLiteralValue(const TokenizedBuffer* buffer, int32_t literal_index,
+                     bool is_decimal)
+        : buffer(buffer),
+          literal_index(literal_index),
+          is_decimal(is_decimal) {}
+  };
+
   // Lexes a buffer of source code into a tokenized buffer.
   //
   // The provided source buffer must outlive any returned `TokenizedBuffer`
@@ -223,7 +258,10 @@ class TokenizedBuffer {
   [[nodiscard]] auto GetIdentifier(Token token) const -> Identifier;
 
   // Returns the value of an `IntegerLiteral()` token.
-  auto GetIntegerLiteral(Token token) const -> llvm::APInt;
+  [[nodiscard]] auto GetIntegerLiteral(Token token) const -> const llvm::APInt&;
+
+  // Returns the value of an `RealLiteral()` token.
+  [[nodiscard]] auto GetRealLiteral(Token token) const -> RealLiteralValue;
 
   // Returns the closing token matched with the given opening token.
   //
@@ -361,7 +399,8 @@ class TokenizedBuffer {
 
   llvm::SmallVector<IdentifierInfo, 16> identifier_infos;
 
-  llvm::SmallVector<llvm::APInt, 16> int_literals;
+  // Storage for integers that form part of the value of a numeric literal.
+  llvm::SmallVector<llvm::APInt, 16> literal_int_storage;
 
   llvm::DenseMap<llvm::StringRef, Identifier> identifier_map;
 

lexer/tokenized_buffer_test.cpp

@@ -39,7 +39,8 @@ struct LexerTest : ::testing::Test {
 
   auto Lex(llvm::Twine text) -> TokenizedBuffer {
     // TODO: build a full mock for this.
-    return TokenizedBuffer::Lex(GetSourceBuffer(text), NullDiagnosticEmitter());
+    return TokenizedBuffer::Lex(GetSourceBuffer(text),
+                                ConsoleDiagnosticEmitter());
   }
 };
 
@@ -166,6 +167,7 @@ TEST_F(LexerTest, ValidatesBaseSpecifier) {
       "00",  "0X123",    "0o123",          "0B1",
       "007", "123L",     "123456789A",     "0x",
       "0b",  "0x123abc", "0b011101201001", "0b10A",
+      "0x_", "0b_",
   };
   for (llvm::StringLiteral literal : invalid) {
     auto buffer = Lex(literal);
@@ -242,6 +244,235 @@ TEST_F(LexerTest, ValidatesIntegerDigitSeparators) {
   }
 }
 
+TEST_F(LexerTest, HandlesRealLiteral) {
+  struct Testcase {
+    llvm::StringLiteral token;
+    uint64_t mantissa;
+    int64_t exponent;
+    unsigned radix;
+  };
+  Testcase testcases[] = {
+      // Decimal real literals.
+      {.token = "0.0", .mantissa = 0, .exponent = -1, .radix = 10},
+      {.token = "12.345", .mantissa = 12345, .exponent = -3, .radix = 10},
+      {.token = "12.345e6", .mantissa = 12345, .exponent = 3, .radix = 10},
+      {.token = "12.345e+6", .mantissa = 12345, .exponent = 3, .radix = 10},
+      {.token = "1_234.5e-2", .mantissa = 12345, .exponent = -3, .radix = 10},
+      {.token = "1.0e-2_000_000",
+       .mantissa = 10,
+       .exponent = -2'000'001,
+       .radix = 10},
+
+      // Hexadecimal real literals.
+      {.token = "0x1_2345_6789.CDEF",
+       .mantissa = 0x1'2345'6789'CDEF,
+       .exponent = -16,
+       .radix = 16},
+      {.token = "0x0.0001p4", .mantissa = 1, .exponent = -12, .radix = 16},
+      {.token = "0x0.0001p+4", .mantissa = 1, .exponent = -12, .radix = 16},
+      {.token = "0x0.0001p-4", .mantissa = 1, .exponent = -20, .radix = 16},
+      // The exponent here works out as exactly INT64_MIN.
+      {.token = "0x1.01p-9223372036854775800",
+       .mantissa = 0x101,
+       .exponent = -9223372036854775807L - 1L,
+       .radix = 16},
+      // The exponent here doesn't fit in a signed 64-bit integer until we
+      // adjust for the radix point.
+      {.token = "0x1.01p9223372036854775809",
+       .mantissa = 0x101,
+       .exponent = 9223372036854775801L,
+       .radix = 16},
+
+      // Binary real literals. These are invalid, but we accept them for error
+      // recovery.
+      {.token = "0b10_11_01.01",
+       .mantissa = 0b10110101,
+       .exponent = -2,
+       .radix = 2},
+  };
+  for (Testcase testcase : testcases) {
+    auto buffer = Lex(testcase.token);
+    EXPECT_EQ(buffer.HasErrors(), testcase.radix == 2);
+    ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
+                            {.kind = TokenKind::RealLiteral(),
+                             .line = 1,
+                             .column = 1,
+                             .indent_column = 1,
+                             .text = testcase.token},
+                        }));
+    auto token = buffer.Tokens().begin();
+    TokenizedBuffer::RealLiteralValue value = buffer.GetRealLiteral(*token);
+    EXPECT_EQ(value.Mantissa().getZExtValue(), testcase.mantissa);
+    EXPECT_EQ(value.Exponent().getSExtValue(), testcase.exponent);
+    EXPECT_EQ(value.IsDecimal(), testcase.radix == 10);
+  }
+}
+
+TEST_F(LexerTest, HandlesRealLiteralOverflow) {
+  llvm::StringLiteral input = "0x1.000001p-9223372036854775800";
+  auto buffer = Lex(input);
+  EXPECT_FALSE(buffer.HasErrors());
+  ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
+                          {.kind = TokenKind::RealLiteral(),
+                           .line = 1,
+                           .column = 1,
+                           .indent_column = 1,
+                           .text = input},
+                      }));
+  auto token = buffer.Tokens().begin();
+  TokenizedBuffer::RealLiteralValue value = buffer.GetRealLiteral(*token);
+  EXPECT_EQ(value.Mantissa(), 0x1000001);
+  EXPECT_EQ((value.Exponent() + 9223372036854775800).getSExtValue(), -24);
+  EXPECT_EQ(value.IsDecimal(), false);
+}
+
+TEST_F(LexerTest, ValidatesRealLiterals) {
+  llvm::StringLiteral invalid_digit_separators[] = {
+      // Invalid digit separators.
+      "12_34.5",     "123.4_567", "123.456_7", "1_2_3.4",
+      "123.4e56_78", "0x12_34.5", "0x12.3_4",  "0x12.34p5_6",
+  };
+  for (llvm::StringLiteral literal : invalid_digit_separators) {
+    auto buffer = Lex(literal);
+    EXPECT_TRUE(buffer.HasErrors()) << literal;
+    // We expect to produce a token even for a literal containing invalid digit
+    // separators, for better error recovery.
+    ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
+                            {.kind = TokenKind::RealLiteral(),
+                             .line = 1,
+                             .column = 1,
+                             .indent_column = 1,
+                             .text = literal}}));
+  }
+
+  llvm::StringLiteral invalid[] = {
+      // No digits in integer part.
+      "0x.0",
+      "0b.0",
+      "0x_.0",
+      "0b_.0",
+
+      // No digits in fractional part.
+      "0.e",
+      "0.e0",
+      "0.e+0",
+      "0x0.p",
+      "0x0.p-0",
+
+      // Invalid digits in mantissa.
+      "123A.4",
+      "123.4A",
+      "123A.4e0",
+      "123.4Ae0",
+      "0x123ABCDEFG.0",
+      "0x123.ABCDEFG",
+      "0x123ABCDEFG.0p0",
+      "0x123.ABCDEFGp0",
+
+      // Invalid exponent letter.
+      "0.0f0",
+      "0.0p0",
+      "0.0z+0",
+      "0x0.0e0",
+      "0x0.0f0",
+      "0x0.0z-0",
+
+      // No digits in exponent part.
+      "0.0e",
+      "0x0.0p",
+      "0.0e_",
+      "0x0.0p_",
+
+      // Invalid digits in exponent part.
+      "0.0eHELLO",
+      "0.0eA",
+      "0.0e+A",
+      "0x0.0pA",
+      "0x0.0p-A",
+  };
+  for (llvm::StringLiteral literal : invalid) {
+    auto buffer = Lex(literal);
+    EXPECT_TRUE(buffer.HasErrors()) << literal;
+    ASSERT_THAT(
+        buffer,
+        HasTokens(llvm::ArrayRef<ExpectedToken>{{.kind = TokenKind::Error(),
+                                                 .line = 1,
+                                                 .column = 1,
+                                                 .indent_column = 1,
+                                                 .text = literal}}));
+  }
+}
+
+TEST_F(LexerTest, SplitsNumericLiteralsProperly) {
+  llvm::StringLiteral source_text = R"(
+    1.
+    .2
+    3.+foo
+    4.0-bar
+    5.0e+123+456
+    6.0e+1e+2
+    1e7
+    8..10
+    9.0.9.5
+    10.foo
+    11.0.foo
+    12e+1
+    13._
+  )";
+  auto buffer = Lex(source_text);
+  EXPECT_TRUE(buffer.HasErrors());
+  EXPECT_THAT(buffer,
+              HasTokens(llvm::ArrayRef<ExpectedToken>{
+                  {.kind = TokenKind::IntegerLiteral(), .text = "1"},
+                  {.kind = TokenKind::Period()},
+                  // newline
+                  {.kind = TokenKind::Period()},
+                  {.kind = TokenKind::IntegerLiteral(), .text = "2"},
+                  // newline
+                  {.kind = TokenKind::IntegerLiteral(), .text = "3"},
+                  {.kind = TokenKind::Period()},
+                  {.kind = TokenKind::Plus()},
+                  {.kind = TokenKind::Identifier(), .text = "foo"},
+                  // newline
+                  {.kind = TokenKind::RealLiteral(), .text = "4.0"},
+                  {.kind = TokenKind::Minus()},
+                  {.kind = TokenKind::Identifier(), .text = "bar"},
+                  // newline
+                  {.kind = TokenKind::RealLiteral(), .text = "5.0e+123"},
+                  {.kind = TokenKind::Plus()},
+                  {.kind = TokenKind::IntegerLiteral(), .text = "456"},
+                  // newline
+                  {.kind = TokenKind::Error(), .text = "6.0e+1e"},
+                  {.kind = TokenKind::Plus()},
+                  {.kind = TokenKind::IntegerLiteral(), .text = "2"},
+                  // newline
+                  {.kind = TokenKind::Error(), .text = "1e7"},
+                  // newline
+                  {.kind = TokenKind::IntegerLiteral(), .text = "8"},
+                  {.kind = TokenKind::Period()},
+                  {.kind = TokenKind::Period()},
+                  {.kind = TokenKind::IntegerLiteral(), .text = "10"},
+                  // newline
+                  {.kind = TokenKind::RealLiteral(), .text = "9.0"},
+                  {.kind = TokenKind::Period()},
+                  {.kind = TokenKind::RealLiteral(), .text = "9.5"},
+                  // newline
+                  {.kind = TokenKind::Error(), .text = "10.foo"},
+                  // newline
+                  {.kind = TokenKind::RealLiteral(), .text = "11.0"},
+                  {.kind = TokenKind::Period()},
+                  {.kind = TokenKind::Identifier(), .text = "foo"},
+                  // newline
+                  {.kind = TokenKind::Error(), .text = "12e"},
+                  {.kind = TokenKind::Plus()},
+                  {.kind = TokenKind::IntegerLiteral(), .text = "1"},
+                  // newline
+                  {.kind = TokenKind::IntegerLiteral(), .text = "13"},
+                  {.kind = TokenKind::Period()},
+                  {.kind = TokenKind::UnderscoreKeyword()},
+              }));
+}
+
 TEST_F(LexerTest, HandlesGarbageCharacters) {
   constexpr char GarbageText[] = "$$💩-$\n$\0$12$";
   auto buffer = Lex(llvm::StringRef(GarbageText, sizeof(GarbageText) - 1));