From a8e4a6932875eea33145e1b7720bcab617c8f490 Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Fri, 26 Feb 2021 17:34:01 -0800 Subject: [PATCH] Initial lexing support for real literals following #143. (#273) --- lexer/token_registry.def | 7 +- lexer/tokenized_buffer.cpp | 646 +++++++++++++++++++++++--------- lexer/tokenized_buffer.h | 43 ++- lexer/tokenized_buffer_test.cpp | 233 +++++++++++- 4 files changed, 751 insertions(+), 178 deletions(-) diff --git a/lexer/token_registry.def b/lexer/token_registry.def index 618f543034df..04bf5876f130 100644 --- a/lexer/token_registry.def +++ b/lexer/token_registry.def @@ -87,11 +87,11 @@ CARBON_SYMBOL_TOKEN(Tilde, "~") // clang-format on #ifndef CARBON_OPENING_GROUP_SYMBOL_TOKEN -#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(Name, Spelling, ClosingName) \ +#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(Name, Spelling, ClosingName) \ CARBON_SYMBOL_TOKEN(Name, Spelling) #endif // clang-format on -CARBON_OPENING_GROUP_SYMBOL_TOKEN(OpenParen, "(", CloseParen) +CARBON_OPENING_GROUP_SYMBOL_TOKEN(OpenParen, "(", CloseParen) CARBON_OPENING_GROUP_SYMBOL_TOKEN(OpenCurlyBrace, "{", CloseCurlyBrace) // clang-format off #undef CARBON_OPENING_GROUP_SYMBOL_TOKEN @@ -101,7 +101,7 @@ CARBON_OPENING_GROUP_SYMBOL_TOKEN(OpenCurlyBrace, "{", CloseCurlyBrace) CARBON_SYMBOL_TOKEN(Name, Spelling) #endif // clang-format on -CARBON_CLOSING_GROUP_SYMBOL_TOKEN(CloseParen, ")", OpenParen) +CARBON_CLOSING_GROUP_SYMBOL_TOKEN(CloseParen, ")", OpenParen) CARBON_CLOSING_GROUP_SYMBOL_TOKEN(CloseCurlyBrace, "}", OpenCurlyBrace) // clang-format off #undef CARBON_CLOSING_GROUP_SYMBOL_TOKEN @@ -156,6 +156,7 @@ CARBON_KEYWORD_TOKEN(XorKeyword, "xor") CARBON_TOKEN(Identifier) CARBON_TOKEN(IntegerLiteral) +CARBON_TOKEN(RealLiteral) CARBON_TOKEN(Error) #undef CARBON_TOKEN diff --git a/lexer/tokenized_buffer.cpp b/lexer/tokenized_buffer.cpp index 0d2b5916b226..8a340b7749c8 100644 --- a/lexer/tokenized_buffer.cpp +++ b/lexer/tokenized_buffer.cpp @@ -21,20 +21,6 @@ namespace Carbon { -static auto TakeLeadingIntegerLiteral(llvm::StringRef source_text) - -> llvm::StringRef { - if (source_text.empty() || !llvm::isDigit(source_text.front())) - return llvm::StringRef(); - - // Greedily consume all following characters that might be part of an integer - // literal. This allows us to produce better diagnostics on invalid literals. - // - // TODO(zygoloid): Update lexical rules to specify that an integer literal - // cannot be immediately followed by another integer literal or a word. - return source_text.take_while( - [](char c) { return llvm::isAlnum(c) || c == '_'; }); -} - struct TrailingComment { static constexpr llvm::StringLiteral ShortName = "syntax-comments"; static constexpr llvm::StringLiteral Message = @@ -95,15 +81,16 @@ struct InvalidDigit { struct Substitutions { char digit; - unsigned radix; + int radix; }; static auto Format(const Substitutions& subst) -> std::string { // TODO: Switch Format to using raw_ostream so we can easily use // llvm::format here. llvm::StringRef digit_str(&subst.digit, 1); return (llvm::Twine("Invalid digit '") + digit_str + "' in " + - (subst.radix == 2 ? "binary" - : subst.radix == 16 ? "hexadecimal" : "decimal") + + (subst.radix == 2 ? "binary" + : subst.radix == 16 ? "hexadecimal" + : "decimal") + " numeric literal.") .str(); } @@ -125,7 +112,7 @@ struct IrregularDigitSeparators { "syntax-irregular-digit-separators"; struct Substitutions { - unsigned radix; + int radix; }; static auto Format(const Substitutions& subst) -> std::string { assert((subst.radix == 10 || subst.radix == 16) && "unexpected radix"); @@ -148,6 +135,31 @@ struct UnknownBaseSpecifier { } }; +struct BinaryRealLiteral { + static constexpr llvm::StringLiteral ShortName = "syntax-invalid-number"; + static constexpr llvm::StringLiteral Message = + "Binary real number literals are not supported."; + + struct Substitutions {}; + static auto Format(const Substitutions&) -> std::string { + return Message.str(); + } +}; + +struct WrongRealLiteralExponent { + static constexpr llvm::StringLiteral ShortName = "syntax-invalid-number"; + + struct Substitutions { + char expected; + }; + static auto Format(const Substitutions& subst) -> std::string { + char expected_str[] = {subst.expected, '\0'}; + return (llvm::Twine("Expected '") + expected_str + + "' to introduce exponent.") + .str(); + } +}; + struct UnrecognizedCharacters { static constexpr llvm::StringLiteral ShortName = "syntax-unrecognized-characters"; @@ -166,6 +178,390 @@ static bool isSpace(char c) { return c == ' ' || c == '\n' || c == '\t'; } +static bool isLower(char c) { return 'a' <= c && c <= 'z'; } + +namespace { +struct NumericLiteral { + llvm::StringRef text; + + // The offset of the '.'. Set to text.size() if none is present. + int radix_point; + + // The offset of the alphabetical character introducing the exponent. In a + // valid literal, this will be an 'e' or a 'p', and may be followed by a '+' + // or a '-', but for error recovery, this may simply be the last lowercase + // letter in the invalid token. Always greater than or equal to radix_point. + // Set to text.size() if none is present. + int exponent; +}; +} // namespace + +static auto TakeLeadingNumericLiteral(llvm::StringRef source_text) + -> NumericLiteral { + NumericLiteral result; + + if (source_text.empty() || !llvm::isDigit(source_text.front())) + return result; + + bool seen_plus_minus = false; + bool seen_radix_point = false; + bool seen_potential_exponent = false; + + // Greedily consume all following characters that might be part of a numeric + // literal. This allows us to produce better diagnostics on invalid literals. + // + // TODO(zygoloid): Update lexical rules to specify that a numeric literal + // cannot be immediately followed by an alphanumeric character. + int i = 1, n = source_text.size(); + for (; i != n; ++i) { + char c = source_text[i]; + if (llvm::isAlnum(c) || c == '_') { + if (isLower(c) && seen_radix_point && !seen_plus_minus) { + result.exponent = i; + seen_potential_exponent = true; + } + continue; + } + + // Exactly one `.` can be part of the literal, but only if it's followed by + // an alphanumeric character. + if (c == '.' && i + 1 != n && llvm::isAlnum(source_text[i + 1]) && + !seen_radix_point) { + result.radix_point = i; + seen_radix_point = true; + continue; + } + + // A `+` or `-` continues the literal only if it's preceded by a lowercase + // letter (which will be 'e' or 'p' or part of an invalid literal) and + // followed by an alphanumeric character. This '+' or '-' cannot be an + // operator because a literal cannot end in a lowercase letter. + if ((c == '+' || c == '-') && seen_potential_exponent && + result.exponent == i - 1 && i + 1 != n && + llvm::isAlnum(source_text[i + 1])) { + // This is not possible because we don't update result.exponent after we + // see a '+' or '-'. + assert(!seen_plus_minus && "should only consume one + or -"); + seen_plus_minus = true; + continue; + } + + break; + } + + result.text = source_text.substr(0, i); + if (!seen_radix_point) + result.radix_point = i; + if (!seen_potential_exponent) + result.exponent = i; + + return result; +} + +namespace { +// Parser for numeric literal tokens. +// +// Responsible for checking that a numeric literal is valid and meaningful and +// either diagnosing or extracting its meaning. +class NumericLiteralParser { + public: + NumericLiteralParser(DiagnosticEmitter& emitter, NumericLiteral literal) + : emitter(emitter), literal(literal) { + int_part = literal.text.substr(0, literal.radix_point); + if (int_part.consume_front("0x")) { + radix = 16; + } else if (int_part.consume_front("0b")) { + radix = 2; + } + + fract_part = literal.text.substr( + literal.radix_point + 1, literal.exponent - literal.radix_point - 1); + + exponent_part = literal.text.substr(literal.exponent + 1); + if (!exponent_part.consume_front("+")) { + exponent_is_negative = exponent_part.consume_front("-"); + } + } + + auto IsInteger() -> bool { + return literal.radix_point == static_cast(literal.text.size()); + } + + enum CheckResult { + // The token is valid. + Valid, + // The token is invalid, but we've diagnosed and recovered from the error. + RecoverableError, + // The token is invalid, and we've diagnosed, but we can't assign meaning + // to it. + UnrecoverableError, + }; + + // Check that the numeric literal token is syntactically valid and + // meaningful, and diagnose if not. + auto Check() -> CheckResult { + if (!CheckLeadingZero() || !CheckIntPart() || !CheckFractionalPart() || + !CheckExponentPart()) + return UnrecoverableError; + return recovered_from_error ? RecoverableError : Valid; + } + + auto GetMantissa() -> llvm::APInt { + const char* end = IsInteger() ? int_part.end() : fract_part.end(); + llvm::StringRef digits(int_part.begin(), end - int_part.begin()); + return ParseInteger(digits, radix, mantissa_needs_cleaning); + } + + auto GetExponent() -> llvm::APInt { + // Compute the effective exponent from the specified exponent, if any, + // and the position of the radix point. + llvm::APInt exponent(64, 0); + if (!exponent_part.empty()) { + exponent = ParseInteger(exponent_part, 10, exponent_needs_cleaning); + + // The exponent is a signed integer, and the number we just parsed is + // non-negative, so ensure we have a wide enough representation to + // include a sign bit. Also make sure the exponent isn't too narrow so + // the calculation below can't lose information through overflow. + if (exponent.isSignBitSet() || exponent.getBitWidth() < 64) { + exponent = exponent.zext(std::max(64u, exponent.getBitWidth() + 1)); + } + if (exponent_is_negative) { + exponent.negate(); + } + } + + // Each character after the decimal point reduces the effective exponent. + int excess_exponent = fract_part.size(); + if (radix == 16) { + excess_exponent *= 4; + } + exponent -= excess_exponent; + if (exponent_is_negative && !exponent.isNegative()) { + // We overflowed. Note that we can only overflow by a little, and only + // from negative to positive, because exponent is at least 64 bits wide + // and excess_exponent is bounded above by four times the size of the + // input buffer, which we assume fits into 32 bits. + exponent = exponent.zext(exponent.getBitWidth() + 1); + exponent.setSignBit(); + } + return exponent; + } + + private: + struct CheckDigitSequenceResult { + bool ok; + bool has_digit_separators = false; + }; + + // Check that a digit sequence is valid: that it contains one or more digits, + // contains only digits in the specified base, and that any digit separators + // are present and correctly positioned. + auto CheckDigitSequence(llvm::StringRef text, int radix, + bool allow_digit_separators = true) + -> CheckDigitSequenceResult { + assert((radix == 2 || radix == 10 || radix == 16) && "unknown radix"); + + std::bitset<256> valid_digits; + if (radix == 2) { + for (char c : "01") + valid_digits[static_cast(c)] = true; + } else if (radix == 10) { + for (char c : "0123456789") + valid_digits[static_cast(c)] = true; + } else { + for (char c : "0123456789ABCDEF") + valid_digits[static_cast(c)] = true; + } + + int num_digit_separators = 0; + + for (int i = 0, n = text.size(); i != n; ++i) { + char c = text[i]; + if (valid_digits[static_cast(c)]) { + continue; + } + + if (c == '_') { + // A digit separator cannot appear at the start of a digit sequence, + // next to another digit separator, or at the end. + if (!allow_digit_separators || i == 0 || text[i - 1] == '_' || + i + 1 == n) { + emitter.EmitError( + [&](InvalidDigitSeparator::Substitutions&) {}); + recovered_from_error = true; + } + ++num_digit_separators; + continue; + } + + emitter.EmitError([&](InvalidDigit::Substitutions& subst) { + subst.digit = c; + subst.radix = radix; + }); + return {.ok = false}; + } + + if (num_digit_separators == static_cast(text.size())) { + emitter.EmitError( + [&](EmptyDigitSequence::Substitutions&) {}); + return {.ok = false}; + } + + // Check that digit separators occur in exactly the expected positions. + if (num_digit_separators && radix != 2) + CheckDigitSeparatorPlacement(text, radix, num_digit_separators); + + return {.ok = true, .has_digit_separators = (num_digit_separators != 0)}; + } + + // Given a number with digit separators, check that the digit separators are + // correctly positioned. + auto CheckDigitSeparatorPlacement(llvm::StringRef text, int radix, + int num_digit_separators) -> void { + assert((radix == 10 || radix == 16) && + "unexpected radix for digit separator checks"); + assert(std::count(text.begin(), text.end(), '_') == num_digit_separators && + "given wrong number of digit separators"); + + auto diagnose_irregular_digit_separators = [&] { + emitter.EmitError( + [&](IrregularDigitSeparators::Substitutions& subst) { + subst.radix = radix; + }); + recovered_from_error = true; + }; + + // For decimal and hexadecimal digit sequences, digit separators must form + // groups of 3 or 4 digits (4 or 5 characters), respectively. + int stride = (radix == 10 ? 4 : 5); + int remaining_digit_separators = num_digit_separators; + for (auto pos = text.end(); pos - text.begin() >= stride; /*in loop*/) { + pos -= stride; + if (*pos != '_') + return diagnose_irregular_digit_separators(); + + --remaining_digit_separators; + } + + // Check there weren't any other digit separators. + if (remaining_digit_separators) + diagnose_irregular_digit_separators(); + }; + + // Check that we don't have a '0' prefix on a non-zero decimal integer. + auto CheckLeadingZero() -> bool { + if (radix == 10 && int_part.startswith("0") && int_part != "0") { + emitter.EmitError( + [&](UnknownBaseSpecifier::Substitutions& subst) {}); + return false; + } + return true; + } + + // Check the integer part (before the '.', if any) is valid. + auto CheckIntPart() -> bool { + auto int_result = CheckDigitSequence(int_part, radix); + mantissa_needs_cleaning |= int_result.has_digit_separators; + return int_result.ok; + } + + // Check the fractional part (after the '.' and before the exponent, if any) + // is valid. + auto CheckFractionalPart() -> bool { + if (IsInteger()) { + return true; + } + + if (radix == 2) { + emitter.EmitError( + [&](BinaryRealLiteral::Substitutions& subst) {}); + recovered_from_error = true; + // Carry on and parse the binary real literal anyway. + } + + // We need to remove a '.' from the mantissa. + mantissa_needs_cleaning = true; + + return CheckDigitSequence(fract_part, radix, + /*allow_digit_separators=*/false) + .ok; + } + + // Check the exponent part (if any) is valid. + auto CheckExponentPart() -> bool { + if (literal.exponent == static_cast(literal.text.size())) { + return true; + } + + char expected_exponent_kind = (radix == 10 ? 'e' : 'p'); + if (literal.text[literal.exponent] != expected_exponent_kind) { + emitter.EmitError( + [&](WrongRealLiteralExponent::Substitutions& subst) { + subst.expected = expected_exponent_kind; + }); + return false; + } + + auto exponent_result = CheckDigitSequence(exponent_part, 10); + exponent_needs_cleaning = exponent_result.has_digit_separators; + return exponent_result.ok; + } + + // Parse a string that is known to be a valid base-radix integer into an + // APInt. If needs_cleaning is true, the string may additionally contain '_' + // and '.' characters that should be ignored. + // + // Ignoring '.' is used when parsing a real literal. For example, when + // parsing 123.456e7, we want to decompose it into an integer mantissa + // (123456) and an exponent (7 - 3 = 2), and this routine is given the + // "123.456" to parse as the mantissa. + static auto ParseInteger(llvm::StringRef digits, int radix, + bool needs_cleaning) -> llvm::APInt { + llvm::SmallString<32> cleaned; + if (needs_cleaning) { + cleaned.reserve(digits.size()); + std::remove_copy_if(digits.begin(), digits.end(), + std::back_inserter(cleaned), + [](char c) { return c == '_' || c == '.'; }); + digits = cleaned; + } + + llvm::APInt value; + if (digits.getAsInteger(radix, value)) { + llvm_unreachable("should never fail"); + } + return value; + } + + private: + DiagnosticEmitter& emitter; + NumericLiteral literal; + + // The radix of the literal: 2, 10, or 16, for a prefix of '0b', no prefix, + // or '0x', respectively. + int radix = 10; + + // The various components of a numeric literal: + // + // [radix] int_part [. fract_part [[ep] [+-] exponent_part]] + llvm::StringRef int_part; + llvm::StringRef fract_part; + llvm::StringRef exponent_part; + + // Do we need to remove any special characters (digit separator or radix + // point) before interpreting the mantissa or exponent as an integer? + bool mantissa_needs_cleaning = false; + bool exponent_needs_cleaning = false; + + // True if we found a `-` before `exponent_part`. + bool exponent_is_negative = false; + + // True if we produced an error but recovered. + bool recovered_from_error = false; +}; +} // namespace + // Implementation of the lexer logic itself. // // The design is that lexing can loop over the source buffer, consuming it into @@ -261,170 +657,58 @@ class TokenizedBuffer::Lexer { return false; } - auto CheckDigitSeparatorPlacement(llvm::StringRef text, unsigned radix, - unsigned num_digit_separators) { - assert((radix == 10 || radix == 16) && - "unexpected radix for digit separator checks"); - assert(std::count(text.begin(), text.end(), '_') == num_digit_separators && - "given wrong number of digit separators"); - - auto diagnose_irregular_digit_separators = [&] { - emitter.EmitError( - [&](IrregularDigitSeparators::Substitutions& subst) { - subst.radix = radix; - }); - buffer.has_errors = true; - }; - - // For decimal and hexadecimal digit sequences, digit separators must form - // groups of 3 or 4 digits (4 or 5 characters), respectively. - unsigned stride = (radix == 10 ? 4 : 5); - unsigned remaining_digit_separators = num_digit_separators; - for (auto pos = text.end(); pos - text.begin() >= stride; /*in loop*/) { - pos -= stride; - if (*pos != '_') - return diagnose_irregular_digit_separators(); - - --remaining_digit_separators; - } - - // Check there weren't any other digit separators. - if (remaining_digit_separators) - diagnose_irregular_digit_separators(); - }; - - struct CheckDigitSequenceResult { - bool ok; - bool has_digit_separators = false; - }; - - auto CheckDigitSequence(llvm::StringRef text, unsigned radix) - -> CheckDigitSequenceResult { - assert((radix == 2 || radix == 10 || radix == 16) && "unknown radix"); - - if (text.empty()) { - emitter.EmitError( - [&](EmptyDigitSequence::Substitutions&) {}); - return {.ok = false}; - } - - std::bitset<256> valid_digits; - if (radix == 2) { - for (char c : "01") - valid_digits[static_cast(c)] = true; - } else if (radix == 10) { - for (char c : "0123456789") - valid_digits[static_cast(c)] = true; - } else { - for (char c : "0123456789ABCDEF") - valid_digits[static_cast(c)] = true; - } - - unsigned num_digit_separators = 0; - - for (std::size_t i = 0, n = text.size(); i != n; ++i) { - char c = text[i]; - if (valid_digits[static_cast(c)]) { - continue; - } - - if (c == '_') { - // A digit separator cannot appear at the start of a digit sequence, - // next to another digit separator, or at the end. - if (i == 0 || text[i - 1] == '_' || i + 1 == n) { - emitter.EmitError( - [&](InvalidDigitSeparator::Substitutions&) {}); - buffer.has_errors = true; - } - ++num_digit_separators; - continue; - } - - emitter.EmitError([&](InvalidDigit::Substitutions& subst) { - subst.digit = c; - subst.radix = radix; - }); - return {.ok = false}; - } - - // Check that digit separators occur in exactly the expected positions. - if (num_digit_separators && radix != 2) - CheckDigitSeparatorPlacement(text, radix, num_digit_separators); - - return {.ok = true, .has_digit_separators = (num_digit_separators != 0)}; - } - - auto LexIntegerLiteral(llvm::StringRef& source_text) -> bool { - llvm::StringRef int_text = TakeLeadingIntegerLiteral(source_text); - if (int_text.empty()) { + auto LexNumericLiteral(llvm::StringRef& source_text) -> bool { + NumericLiteral literal = TakeLeadingNumericLiteral(source_text); + if (literal.text.empty()) { return false; } int int_column = current_column; - current_column += int_text.size(); - source_text = source_text.drop_front(int_text.size()); + current_column += literal.text.size(); + source_text = source_text.drop_front(literal.text.size()); if (!set_indent) { current_line_info->indent = int_column; set_indent = true; } - auto add_error_token_and_continue_lexing = [&] { - buffer.AddToken({ - .kind = TokenKind::Error(), - .token_line = current_line, - .column = int_column, - .error_length = static_cast(int_text.size()), - }); - buffer.has_errors = true; - // Indicate to the caller that we consumed a token. - return true; - }; + NumericLiteralParser literal_parser(emitter, literal); - unsigned radix = 10; - llvm::StringRef digits = int_text; - if (int_text.size() >= 2 && int_text[0] == '0') { - if (int_text[1] == 'x') { - radix = 16; - digits = digits.drop_front(2); - } else if (int_text[1] == 'b') { - radix = 2; - digits = digits.drop_front(2); - } else { - emitter.EmitError( - [&](UnknownBaseSpecifier::Substitutions& subst) {}); - return add_error_token_and_continue_lexing(); - } + switch (literal_parser.Check()) { + case NumericLiteralParser::UnrecoverableError: + buffer.AddToken({ + .kind = TokenKind::Error(), + .token_line = current_line, + .column = int_column, + .error_length = static_cast(literal.text.size()), + }); + buffer.has_errors = true; + return true; + + case NumericLiteralParser::RecoverableError: + buffer.has_errors = true; + break; + + case NumericLiteralParser::Valid: + break; } - llvm::APInt int_value; - - auto result = CheckDigitSequence(digits, radix); - if (!result.ok) { - return add_error_token_and_continue_lexing(); - } - - if (result.has_digit_separators) { - // TODO(zygoloid): Avoid the memory allocation here. - std::string cleaned; - cleaned.reserve(digits.size()); - std::remove_copy_if(digits.begin(), digits.end(), - std::back_inserter(cleaned), - [](char c) { return c == '_'; }); - if (llvm::StringRef(cleaned).getAsInteger(radix, int_value)) { - llvm_unreachable("should never fail"); - } + if (literal_parser.IsInteger()) { + auto token = buffer.AddToken({.kind = TokenKind::IntegerLiteral(), + .token_line = current_line, + .column = int_column}); + buffer.GetTokenInfo(token).literal_index = + buffer.literal_int_storage.size(); + buffer.literal_int_storage.push_back(literal_parser.GetMantissa()); } else { - if (digits.getAsInteger(radix, int_value)) { - llvm_unreachable("should never fail"); - } + auto token = buffer.AddToken({.kind = TokenKind::RealLiteral(), + .token_line = current_line, + .column = int_column}); + buffer.GetTokenInfo(token).literal_index = + buffer.literal_int_storage.size(); + buffer.literal_int_storage.push_back(literal_parser.GetMantissa()); + buffer.literal_int_storage.push_back(literal_parser.GetExponent()); } - - auto token = buffer.AddToken({.kind = TokenKind::IntegerLiteral(), - .token_line = current_line, - .column = int_column}); - buffer.GetTokenInfo(token).literal_index = buffer.int_literals.size(); - buffer.int_literals.push_back(std::move(int_value)); return true; } @@ -619,7 +903,7 @@ auto TokenizedBuffer::Lex(SourceBuffer& source, DiagnosticEmitter& emitter) if (lexer.LexKeywordOrIdentifier(source_text)) { continue; } - if (lexer.LexIntegerLiteral(source_text)) { + if (lexer.LexNumericLiteral(source_text)) { continue; } lexer.LexError(source_text); @@ -660,10 +944,11 @@ auto TokenizedBuffer::GetTokenText(Token token) const -> llvm::StringRef { // Refer back to the source text to preserve oddities like radix or digit // separators the author included. - if (token_info.kind == TokenKind::IntegerLiteral()) { + if (token_info.kind == TokenKind::IntegerLiteral() || + token_info.kind == TokenKind::RealLiteral()) { auto& line_info = GetLineInfo(token_info.token_line); int64_t token_start = line_info.start + token_info.column; - return TakeLeadingIntegerLiteral(source->Text().substr(token_start)); + return TakeLeadingNumericLiteral(source->Text().substr(token_start)).text; } assert(token_info.kind == TokenKind::Identifier() && @@ -678,11 +963,28 @@ auto TokenizedBuffer::GetIdentifier(Token token) const -> Identifier { return token_info.id; } -auto TokenizedBuffer::GetIntegerLiteral(Token token) const -> llvm::APInt { +auto TokenizedBuffer::GetIntegerLiteral(Token token) const + -> const llvm::APInt& { auto& token_info = GetTokenInfo(token); assert(token_info.kind == TokenKind::IntegerLiteral() && "The token must be an integer literal!"); - return int_literals[token_info.literal_index]; + return literal_int_storage[token_info.literal_index]; +} + +auto TokenizedBuffer::GetRealLiteral(Token token) const -> RealLiteralValue { + auto& token_info = GetTokenInfo(token); + assert(token_info.kind == TokenKind::RealLiteral() && + "The token must be a real literal!"); + + // Note that every real literal is at least three characters long, so we can + // safely look at the second character to determine whether we have a decimal + // or hexadecimal literal. + auto& line_info = GetLineInfo(token_info.token_line); + int64_t token_start = line_info.start + token_info.column; + char second_char = source->Text()[token_start + 1]; + bool is_decimal = second_char != 'x' && second_char != 'b'; + + return RealLiteralValue(this, token_info.literal_index, is_decimal); } auto TokenizedBuffer::GetMatchedClosingToken(Token opening_token) const diff --git a/lexer/tokenized_buffer.h b/lexer/tokenized_buffer.h index 74e4401ba791..23305b4396bf 100644 --- a/lexer/tokenized_buffer.h +++ b/lexer/tokenized_buffer.h @@ -186,6 +186,41 @@ class TokenizedBuffer { Token token; }; + // The value of a real literal. + // + // This is either a dyadic fraction (mantissa * 2^exponent) or a decadic + // fraction (mantissa * 10^exponent). + // + // The `TokenizedBuffer` must outlive any `RealLiteralValue`s referring to + // its tokens. + class RealLiteralValue { + const TokenizedBuffer* buffer; + int32_t literal_index; + bool is_decimal; + + public: + // The mantissa, represented as an unsigned integer. + const llvm::APInt& Mantissa() const { + return buffer->literal_int_storage[literal_index]; + } + // The exponent, represented as a signed integer. + const llvm::APInt& Exponent() const { + return buffer->literal_int_storage[literal_index + 1]; + } + // If false, the value is mantissa * 2^exponent. + // If true, the value is mantissa * 10^exponent. + bool IsDecimal() const { return is_decimal; } + + private: + friend class TokenizedBuffer; + + RealLiteralValue(const TokenizedBuffer* buffer, int32_t literal_index, + bool is_decimal) + : buffer(buffer), + literal_index(literal_index), + is_decimal(is_decimal) {} + }; + // Lexes a buffer of source code into a tokenized buffer. // // The provided source buffer must outlive any returned `TokenizedBuffer` @@ -223,7 +258,10 @@ class TokenizedBuffer { [[nodiscard]] auto GetIdentifier(Token token) const -> Identifier; // Returns the value of an `IntegerLiteral()` token. - auto GetIntegerLiteral(Token token) const -> llvm::APInt; + [[nodiscard]] auto GetIntegerLiteral(Token token) const -> const llvm::APInt&; + + // Returns the value of an `RealLiteral()` token. + [[nodiscard]] auto GetRealLiteral(Token token) const -> RealLiteralValue; // Returns the closing token matched with the given opening token. // @@ -361,7 +399,8 @@ class TokenizedBuffer { llvm::SmallVector identifier_infos; - llvm::SmallVector int_literals; + // Storage for integers that form part of the value of a numeric literal. + llvm::SmallVector literal_int_storage; llvm::DenseMap identifier_map; diff --git a/lexer/tokenized_buffer_test.cpp b/lexer/tokenized_buffer_test.cpp index 53543009d8a6..8689989f63b6 100644 --- a/lexer/tokenized_buffer_test.cpp +++ b/lexer/tokenized_buffer_test.cpp @@ -39,7 +39,8 @@ struct LexerTest : ::testing::Test { auto Lex(llvm::Twine text) -> TokenizedBuffer { // TODO: build a full mock for this. - return TokenizedBuffer::Lex(GetSourceBuffer(text), NullDiagnosticEmitter()); + return TokenizedBuffer::Lex(GetSourceBuffer(text), + ConsoleDiagnosticEmitter()); } }; @@ -166,6 +167,7 @@ TEST_F(LexerTest, ValidatesBaseSpecifier) { "00", "0X123", "0o123", "0B1", "007", "123L", "123456789A", "0x", "0b", "0x123abc", "0b011101201001", "0b10A", + "0x_", "0b_", }; for (llvm::StringLiteral literal : invalid) { auto buffer = Lex(literal); @@ -242,6 +244,235 @@ TEST_F(LexerTest, ValidatesIntegerDigitSeparators) { } } +TEST_F(LexerTest, HandlesRealLiteral) { + struct Testcase { + llvm::StringLiteral token; + uint64_t mantissa; + int64_t exponent; + unsigned radix; + }; + Testcase testcases[] = { + // Decimal real literals. + {.token = "0.0", .mantissa = 0, .exponent = -1, .radix = 10}, + {.token = "12.345", .mantissa = 12345, .exponent = -3, .radix = 10}, + {.token = "12.345e6", .mantissa = 12345, .exponent = 3, .radix = 10}, + {.token = "12.345e+6", .mantissa = 12345, .exponent = 3, .radix = 10}, + {.token = "1_234.5e-2", .mantissa = 12345, .exponent = -3, .radix = 10}, + {.token = "1.0e-2_000_000", + .mantissa = 10, + .exponent = -2'000'001, + .radix = 10}, + + // Hexadecimal real literals. + {.token = "0x1_2345_6789.CDEF", + .mantissa = 0x1'2345'6789'CDEF, + .exponent = -16, + .radix = 16}, + {.token = "0x0.0001p4", .mantissa = 1, .exponent = -12, .radix = 16}, + {.token = "0x0.0001p+4", .mantissa = 1, .exponent = -12, .radix = 16}, + {.token = "0x0.0001p-4", .mantissa = 1, .exponent = -20, .radix = 16}, + // The exponent here works out as exactly INT64_MIN. + {.token = "0x1.01p-9223372036854775800", + .mantissa = 0x101, + .exponent = -9223372036854775807L - 1L, + .radix = 16}, + // The exponent here doesn't fit in a signed 64-bit integer until we + // adjust for the radix point. + {.token = "0x1.01p9223372036854775809", + .mantissa = 0x101, + .exponent = 9223372036854775801L, + .radix = 16}, + + // Binary real literals. These are invalid, but we accept them for error + // recovery. + {.token = "0b10_11_01.01", + .mantissa = 0b10110101, + .exponent = -2, + .radix = 2}, + }; + for (Testcase testcase : testcases) { + auto buffer = Lex(testcase.token); + EXPECT_EQ(buffer.HasErrors(), testcase.radix == 2); + ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef{ + {.kind = TokenKind::RealLiteral(), + .line = 1, + .column = 1, + .indent_column = 1, + .text = testcase.token}, + })); + auto token = buffer.Tokens().begin(); + TokenizedBuffer::RealLiteralValue value = buffer.GetRealLiteral(*token); + EXPECT_EQ(value.Mantissa().getZExtValue(), testcase.mantissa); + EXPECT_EQ(value.Exponent().getSExtValue(), testcase.exponent); + EXPECT_EQ(value.IsDecimal(), testcase.radix == 10); + } +} + +TEST_F(LexerTest, HandlesRealLiteralOverflow) { + llvm::StringLiteral input = "0x1.000001p-9223372036854775800"; + auto buffer = Lex(input); + EXPECT_FALSE(buffer.HasErrors()); + ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef{ + {.kind = TokenKind::RealLiteral(), + .line = 1, + .column = 1, + .indent_column = 1, + .text = input}, + })); + auto token = buffer.Tokens().begin(); + TokenizedBuffer::RealLiteralValue value = buffer.GetRealLiteral(*token); + EXPECT_EQ(value.Mantissa(), 0x1000001); + EXPECT_EQ((value.Exponent() + 9223372036854775800).getSExtValue(), -24); + EXPECT_EQ(value.IsDecimal(), false); +} + +TEST_F(LexerTest, ValidatesRealLiterals) { + llvm::StringLiteral invalid_digit_separators[] = { + // Invalid digit separators. + "12_34.5", "123.4_567", "123.456_7", "1_2_3.4", + "123.4e56_78", "0x12_34.5", "0x12.3_4", "0x12.34p5_6", + }; + for (llvm::StringLiteral literal : invalid_digit_separators) { + auto buffer = Lex(literal); + EXPECT_TRUE(buffer.HasErrors()) << literal; + // We expect to produce a token even for a literal containing invalid digit + // separators, for better error recovery. + ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef{ + {.kind = TokenKind::RealLiteral(), + .line = 1, + .column = 1, + .indent_column = 1, + .text = literal}})); + } + + llvm::StringLiteral invalid[] = { + // No digits in integer part. + "0x.0", + "0b.0", + "0x_.0", + "0b_.0", + + // No digits in fractional part. + "0.e", + "0.e0", + "0.e+0", + "0x0.p", + "0x0.p-0", + + // Invalid digits in mantissa. + "123A.4", + "123.4A", + "123A.4e0", + "123.4Ae0", + "0x123ABCDEFG.0", + "0x123.ABCDEFG", + "0x123ABCDEFG.0p0", + "0x123.ABCDEFGp0", + + // Invalid exponent letter. + "0.0f0", + "0.0p0", + "0.0z+0", + "0x0.0e0", + "0x0.0f0", + "0x0.0z-0", + + // No digits in exponent part. + "0.0e", + "0x0.0p", + "0.0e_", + "0x0.0p_", + + // Invalid digits in exponent part. + "0.0eHELLO", + "0.0eA", + "0.0e+A", + "0x0.0pA", + "0x0.0p-A", + }; + for (llvm::StringLiteral literal : invalid) { + auto buffer = Lex(literal); + EXPECT_TRUE(buffer.HasErrors()) << literal; + ASSERT_THAT( + buffer, + HasTokens(llvm::ArrayRef{{.kind = TokenKind::Error(), + .line = 1, + .column = 1, + .indent_column = 1, + .text = literal}})); + } +} + +TEST_F(LexerTest, SplitsNumericLiteralsProperly) { + llvm::StringLiteral source_text = R"( + 1. + .2 + 3.+foo + 4.0-bar + 5.0e+123+456 + 6.0e+1e+2 + 1e7 + 8..10 + 9.0.9.5 + 10.foo + 11.0.foo + 12e+1 + 13._ + )"; + auto buffer = Lex(source_text); + EXPECT_TRUE(buffer.HasErrors()); + EXPECT_THAT(buffer, + HasTokens(llvm::ArrayRef{ + {.kind = TokenKind::IntegerLiteral(), .text = "1"}, + {.kind = TokenKind::Period()}, + // newline + {.kind = TokenKind::Period()}, + {.kind = TokenKind::IntegerLiteral(), .text = "2"}, + // newline + {.kind = TokenKind::IntegerLiteral(), .text = "3"}, + {.kind = TokenKind::Period()}, + {.kind = TokenKind::Plus()}, + {.kind = TokenKind::Identifier(), .text = "foo"}, + // newline + {.kind = TokenKind::RealLiteral(), .text = "4.0"}, + {.kind = TokenKind::Minus()}, + {.kind = TokenKind::Identifier(), .text = "bar"}, + // newline + {.kind = TokenKind::RealLiteral(), .text = "5.0e+123"}, + {.kind = TokenKind::Plus()}, + {.kind = TokenKind::IntegerLiteral(), .text = "456"}, + // newline + {.kind = TokenKind::Error(), .text = "6.0e+1e"}, + {.kind = TokenKind::Plus()}, + {.kind = TokenKind::IntegerLiteral(), .text = "2"}, + // newline + {.kind = TokenKind::Error(), .text = "1e7"}, + // newline + {.kind = TokenKind::IntegerLiteral(), .text = "8"}, + {.kind = TokenKind::Period()}, + {.kind = TokenKind::Period()}, + {.kind = TokenKind::IntegerLiteral(), .text = "10"}, + // newline + {.kind = TokenKind::RealLiteral(), .text = "9.0"}, + {.kind = TokenKind::Period()}, + {.kind = TokenKind::RealLiteral(), .text = "9.5"}, + // newline + {.kind = TokenKind::Error(), .text = "10.foo"}, + // newline + {.kind = TokenKind::RealLiteral(), .text = "11.0"}, + {.kind = TokenKind::Period()}, + {.kind = TokenKind::Identifier(), .text = "foo"}, + // newline + {.kind = TokenKind::Error(), .text = "12e"}, + {.kind = TokenKind::Plus()}, + {.kind = TokenKind::IntegerLiteral(), .text = "1"}, + // newline + {.kind = TokenKind::IntegerLiteral(), .text = "13"}, + {.kind = TokenKind::Period()}, + {.kind = TokenKind::UnderscoreKeyword()}, + })); +} + TEST_F(LexerTest, HandlesGarbageCharacters) { constexpr char GarbageText[] = "$$💩-$\n$\0$12$"; auto buffer = Lex(llvm::StringRef(GarbageText, sizeof(GarbageText) - 1));