diff --git a/toolchain/lex/lex.cpp b/toolchain/lex/lex.cpp index 4760a86b5cf9..21caf0b560ee 100644 --- a/toolchain/lex/lex.cpp +++ b/toolchain/lex/lex.cpp @@ -195,9 +195,6 @@ class [[clang::internal_linkage]] Lexer { auto LexClosingSymbolToken(llvm::StringRef source_text, TokenKind kind, ssize_t& position) -> LexResult; - auto LexSymbolToken(llvm::StringRef source_text, ssize_t& position) - -> LexResult; - // Given a word that has already been lexed, determine whether it is a type // literal and if so form the corresponding token. auto LexWordAsTypeLiteralToken(llvm::StringRef word, int32_t byte_offset) @@ -320,6 +317,31 @@ static constexpr std::array IsIdByteTable = [] { return table; }(); +// A table of booleans classifying the bytes that end a run of unrecognized +// characters forming an error token: alphanumeric characters, `_`, tabs and +// newlines, and any character that is by itself a symbol. +static constexpr std::array IsErrorEndByteTable = [] { + std::array table = {}; + for (char c = '0'; c <= '9'; ++c) { + table[c] = true; + } + for (char c = 'A'; c <= 'Z'; ++c) { + table[c] = true; + } + for (char c = 'a'; c <= 'z'; ++c) { + table[c] = true; + } + table['_'] = true; + table['\t'] = true; + table['\n'] = true; +#define CARBON_SYMBOL_TOKEN(TokenName, Spelling) \ + if (sizeof(Spelling) == 2) { \ + table[(Spelling)[0]] = true; \ + } +#include "toolchain/lex/token_kind.def" + return table; +}(); + // Baseline scalar version, also available for scalar-fallback in SIMD code. // Uses `ssize_t` for performance when indexing in the loop. // @@ -537,18 +559,6 @@ static auto ScanForIdentifierPrefix(llvm::StringRef text) -> llvm::StringRef { return ScanForIdentifierPrefixScalar(text, 0); } -static constexpr std::array OneCharTokenKindTable = [] { - std::array table = {}; -#define CARBON_ONE_CHAR_SYMBOL_TOKEN(TokenName, Spelling) \ - table[(Spelling)[0]] = TokenKind::TokenName; -#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \ - table[(Spelling)[0]] = TokenKind::TokenName; -#define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \ - table[(Spelling)[0]] = TokenKind::TokenName; -#include "toolchain/lex/token_kind.def" - return table; -}(); - // We use a collection of static member functions for table-based dispatch to // lexer methods. These are named static member functions so that they show up // helpfully in profiles and backtraces, but they tend to not contain the @@ -586,7 +596,6 @@ CARBON_PRESERVE_NONE static auto DispatchLexError(Lexer& lexer, /* NOLINTNEXTLINE(readability-avoid-return-with-void-value) */ \ [[clang::musttail]] return DispatchLexError(lexer, source_text, position); \ } -CARBON_DISPATCH_LEX_TOKEN(LexSymbolToken) CARBON_DISPATCH_LEX_TOKEN(LexKeywordOrIdentifier) CARBON_DISPATCH_LEX_TOKEN(LexNumericLiteral) CARBON_DISPATCH_LEX_TOKEN(LexStringLiteral) @@ -604,21 +613,42 @@ CARBON_PRESERVE_NONE static auto DispatchLexHash(Lexer& lexer, } // A set of custom dispatch functions that preselect the symbol token to lex. -#define CARBON_DISPATCH_LEX_SYMBOL_TOKEN(LexMethod) \ - CARBON_PRESERVE_NONE static auto Dispatch##LexMethod##SymbolToken( \ - Lexer& lexer, llvm::StringRef source_text, ssize_t position) -> void { \ - Lexer::LexResult result = lexer.LexMethod##SymbolToken( \ - source_text, \ - OneCharTokenKindTable[static_cast( \ - source_text[position])], \ - position); \ - CARBON_CHECK(result, "Failed to form a token!"); \ - [[clang::musttail]] return DispatchNext(lexer, source_text, position); \ +#define CARBON_DISPATCH_LEX_SYMBOL_TOKEN(LexMethod) \ + template \ + CARBON_PRESERVE_NONE static auto Dispatch##LexMethod##SymbolToken( \ + Lexer& lexer, llvm::StringRef source_text, ssize_t position) -> void { \ + Lexer::LexResult result = lexer.LexMethod##SymbolToken( \ + source_text, TokenKind::Make(Kind), position); \ + CARBON_CHECK(result, "Failed to form a token!"); \ + /* NOLINTNEXTLINE(readability-avoid-return-with-void-value): musttail. */ \ + [[clang::musttail]] return DispatchNext(lexer, source_text, position); \ } CARBON_DISPATCH_LEX_SYMBOL_TOKEN(LexOneChar) CARBON_DISPATCH_LEX_SYMBOL_TOKEN(LexOpening) CARBON_DISPATCH_LEX_SYMBOL_TOKEN(LexClosing) +template +CARBON_PRESERVE_NONE static auto DispatchLexSymbolToken( + Lexer& lexer, llvm::StringRef source_text, ssize_t position) -> void { + CARBON_DCHECK(source_text[position] == FirstChar); + llvm::StringRef tail = source_text.substr(position + 1); +#define CARBON_SYMBOL_TOKEN(Name, Spelling) \ + if constexpr ((Spelling)[0] == FirstChar) { \ + if (tail.starts_with(llvm::StringLiteral(Spelling).substr(1))) { \ + lexer.LexToken(TokenKind::Name, position); \ + /* NOLINTNEXTLINE(readability-avoid-return-with-void-value): musttail */ \ + [[clang::musttail]] return DispatchNext( \ + lexer, source_text, position + (sizeof(Spelling) - 1)); \ + } \ + } +#define CARBON_ONE_CHAR_SYMBOL_TOKEN(TokenName, Spelling) +#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) +#define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) +#include "toolchain/lex/token_kind.def" + // NOLINTNEXTLINE(readability-avoid-return-with-void-value): For musttail. + [[clang::musttail]] return DispatchLexError(lexer, source_text, position); +} + // Define a set of non-token dispatch functions that handle things like // whitespace and comments. #define CARBON_DISPATCH_LEX_NON_TOKEN(LexMethod) \ @@ -648,8 +678,8 @@ CARBON_PRESERVE_NONE static auto DispatchLexCommentOrSlash( } // NOLINTNEXTLINE(readability-avoid-return-with-void-value): For musttail. - [[clang::musttail]] return DispatchLexSymbolToken(lexer, source_text, - position); + [[clang::musttail]] return DispatchLexSymbolToken<'/'>(lexer, source_text, + position); } // Build a table of function pointers that we can use to dispatch to the @@ -692,12 +722,10 @@ static constexpr auto MakeDispatchTable() -> DispatchTableT { } // Symbols have some special dispatching. First, set the first character of - // each symbol token spelling to dispatch to the symbol lexer. We don't - // provide a pre-computed token here, so the symbol lexer will compute the - // exact symbol token kind. We'll override this with more specific dispatch - // below. + // each symbol token spelling to dispatch to the symbol lexer for that + // starting character. We'll override this with more specific dispatch below. #define CARBON_SYMBOL_TOKEN(TokenName, Spelling) \ - table[(Spelling)[0]] = &DispatchLexSymbolToken; + table[(Spelling)[0]] = &DispatchLexSymbolToken<(Spelling)[0]>; #include "toolchain/lex/token_kind.def" // Now special cased single-character symbols that are guaranteed to not @@ -707,11 +735,11 @@ static constexpr auto MakeDispatchTable() -> DispatchTableT { // needs to override some of the generic handling above, and provide a // custom token. #define CARBON_ONE_CHAR_SYMBOL_TOKEN(TokenName, Spelling) \ - table[(Spelling)[0]] = &DispatchLexOneCharSymbolToken; + table[(Spelling)[0]] = &DispatchLexOneCharSymbolToken; #define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \ - table[(Spelling)[0]] = &DispatchLexOpeningSymbolToken; + table[(Spelling)[0]] = &DispatchLexOpeningSymbolToken; #define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \ - table[(Spelling)[0]] = &DispatchLexClosingSymbolToken; + table[(Spelling)[0]] = &DispatchLexClosingSymbolToken; #include "toolchain/lex/token_kind.def" // Override the handling for `/` to consider comments as well as a `/` @@ -1585,27 +1613,6 @@ auto Lexer::LexClosingSymbolToken(llvm::StringRef source_text, TokenKind kind, return token; } -auto Lexer::LexSymbolToken(llvm::StringRef source_text, ssize_t& position) - -> LexResult { - // One character symbols and grouping symbols are handled with dedicated - // dispatch. We only lex the multi-character tokens here. - TokenKind kind = llvm::StringSwitch(source_text.substr(position)) -#define CARBON_SYMBOL_TOKEN(Name, Spelling) \ - .StartsWith(Spelling, TokenKind::Name) -#define CARBON_ONE_CHAR_SYMBOL_TOKEN(TokenName, Spelling) -#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) -#define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) -#include "toolchain/lex/token_kind.def" - .Default(TokenKind::Error); - if (kind == TokenKind::Error) { - return LexResult::NoMatch(); - } - - TokenIndex token = LexToken(kind, position); - position += kind.fixed_spelling().size(); - return token; -} - auto Lexer::LexWordAsTypeLiteralToken(llvm::StringRef word, int32_t byte_offset) -> LexResult { if (word.size() < 2) { @@ -1847,21 +1854,7 @@ auto Lexer::LexError(llvm::StringRef source_text, ssize_t& position) -> LexResult { llvm::StringRef error_text = source_text.substr(position).take_while([](char c) { - if (IsAlnum(c)) { - return false; - } - switch (c) { - case '_': - case '\t': - case '\n': - return false; - default: - break; - } - return llvm::StringSwitch(llvm::StringRef(&c, 1)) -#define CARBON_SYMBOL_TOKEN(Name, Spelling) .StartsWith(Spelling, false) -#include "toolchain/lex/token_kind.def" - .Default(true); + return !IsErrorEndByteTable[static_cast(c)]; }); if (error_text.empty()) { // TODO: Reimplement this to use the lexer properly. In the meantime, diff --git a/toolchain/lex/token_kind.cpp b/toolchain/lex/token_kind.cpp index 66e0276fe8d4..f0f281fdf758 100644 --- a/toolchain/lex/token_kind.cpp +++ b/toolchain/lex/token_kind.cpp @@ -11,72 +11,4 @@ CARBON_DEFINE_ENUM_CLASS_NAMES(TokenKind) { #include "toolchain/lex/token_kind.def" }; -constexpr bool TokenKind::IsSymbol[] = { -#define CARBON_TOKEN(TokenName) false, -#define CARBON_SYMBOL_TOKEN(TokenName, Spelling) true, -#include "toolchain/lex/token_kind.def" -}; - -constexpr bool TokenKind::IsGroupingSymbol[] = { -#define CARBON_TOKEN(TokenName) false, -#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \ - true, -#define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \ - true, -#include "toolchain/lex/token_kind.def" -}; - -constexpr bool TokenKind::IsOpeningSymbol[] = { -#define CARBON_TOKEN(TokenName) false, -#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \ - true, -#include "toolchain/lex/token_kind.def" -}; - -constexpr TokenKind TokenKind::ClosingSymbol[] = { -#define CARBON_TOKEN(TokenName) Error, -#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \ - ClosingName, -#include "toolchain/lex/token_kind.def" -}; - -constexpr bool TokenKind::IsClosingSymbol[] = { -#define CARBON_TOKEN(TokenName) false, -#define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \ - true, -#include "toolchain/lex/token_kind.def" -}; - -constexpr TokenKind TokenKind::OpeningSymbol[] = { -#define CARBON_TOKEN(TokenName) Error, -#define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \ - OpeningName, -#include "toolchain/lex/token_kind.def" -}; - -constexpr bool TokenKind::IsOneCharSymbol[] = { -#define CARBON_TOKEN(TokenName) false, -#define CARBON_ONE_CHAR_SYMBOL_TOKEN(TokenName, Spelling) true, -#include "toolchain/lex/token_kind.def" -}; - -constexpr bool TokenKind::IsKeyword[] = { -#define CARBON_TOKEN(TokenName) false, -#define CARBON_KEYWORD_TOKEN(TokenName, Spelling) true, -#include "toolchain/lex/token_kind.def" -}; - -constexpr llvm::StringLiteral TokenKind::FixedSpelling[] = { -#define CARBON_TOKEN(TokenName) "", -#define CARBON_SYMBOL_TOKEN(TokenName, Spelling) Spelling, -#define CARBON_KEYWORD_TOKEN(TokenName, Spelling) Spelling, -#include "toolchain/lex/token_kind.def" -}; - -constexpr int8_t TokenKind::ExpectedParseTreeSize[] = { -#define CARBON_TOKEN(Name) 1, -#define CARBON_TOKEN_WITH_VIRTUAL_NODE(size) 2, -#include "toolchain/lex/token_kind.def" -}; - } // namespace Carbon::Lex diff --git a/toolchain/lex/token_kind.h b/toolchain/lex/token_kind.h index 19a804fa724d..a58a4e9dce9d 100644 --- a/toolchain/lex/token_kind.h +++ b/toolchain/lex/token_kind.h @@ -42,31 +42,37 @@ class TokenKind : public CARBON_ENUM_BASE(TokenKind) { // inside of other tokens, outside of the contents of other tokens they // don't require any specific characters before or after to distinguish them // in the source. Returns false otherwise. - auto is_symbol() const -> bool { return IsSymbol[AsInt()]; } + constexpr auto is_symbol() const -> bool { return IsSymbol[AsInt()]; } // Test whether this kind of token is a grouping symbol (part of an opening // and closing pair that must always be matched in the token stream). - auto is_grouping_symbol() const -> bool { return IsGroupingSymbol[AsInt()]; } + constexpr auto is_grouping_symbol() const -> bool { + return IsGroupingSymbol[AsInt()]; + } // Test whether this kind of token is an opening symbol for a group. - auto is_opening_symbol() const -> bool { return IsOpeningSymbol[AsInt()]; } + constexpr auto is_opening_symbol() const -> bool { + return IsOpeningSymbol[AsInt()]; + } // Returns the associated closing symbol for an opening symbol. // // The token kind must be an opening symbol. - auto closing_symbol() const -> TokenKind { + constexpr auto closing_symbol() const -> TokenKind { auto result = ClosingSymbol[AsInt()]; CARBON_DCHECK(result != Error, "Only opening symbols are valid!"); return result; } // Test whether this kind of token is a closing symbol for a group. - auto is_closing_symbol() const -> bool { return IsClosingSymbol[AsInt()]; } + constexpr auto is_closing_symbol() const -> bool { + return IsClosingSymbol[AsInt()]; + } // Returns the associated opening symbol for a closing symbol. // // The token kind must be a closing symbol. - auto opening_symbol() const -> TokenKind { + constexpr auto opening_symbol() const -> TokenKind { auto result = OpeningSymbol[AsInt()]; CARBON_DCHECK(result != Error, "Only closing symbols are valid!"); return result; @@ -74,48 +80,50 @@ class TokenKind : public CARBON_ENUM_BASE(TokenKind) { // Test whether this kind of token is a one-character symbol whose character // is not part of any other symbol. - auto is_one_char_symbol() const -> bool { return IsOneCharSymbol[AsInt()]; } + constexpr auto is_one_char_symbol() const -> bool { + return IsOneCharSymbol[AsInt()]; + } // Test whether this kind of token is a keyword. - auto is_keyword() const -> bool { return IsKeyword[AsInt()]; } + constexpr auto is_keyword() const -> bool { return IsKeyword[AsInt()]; } // Test whether this kind of token is a sized type literal. - auto is_sized_type_literal() const -> bool { + constexpr auto is_sized_type_literal() const -> bool { return *this == TokenKind::IntTypeLiteral || *this == TokenKind::UnsignedIntTypeLiteral || *this == TokenKind::FloatTypeLiteral; } // Test whether this kind of token is a dollar int literal. - auto is_dollar_int_literal() const -> bool { + constexpr auto is_dollar_int_literal() const -> bool { return *this == TokenKind::DollarIntLiteral; } // Test whether this kind of token is a word. - auto is_word() const -> bool { + constexpr auto is_word() const -> bool { return *this == TokenKind::Identifier || *this == TokenKind::Underscore || is_keyword() || is_sized_type_literal(); } // Test whether this kind of token is a binding pattern operator. - auto is_binding_pattern_operator() const -> bool { + constexpr auto is_binding_pattern_operator() const -> bool { return *this == TokenKind::Colon || *this == TokenKind::ColonQuestion; } // If this token kind has a fixed spelling when in source code, returns it. // Otherwise returns an empty string. - auto fixed_spelling() const -> llvm::StringLiteral { + constexpr auto fixed_spelling() const -> llvm::StringLiteral { return FixedSpelling[AsInt()]; } // Get the expected number of parse tree nodes that will be created for this // token. - auto expected_max_parse_tree_size() const -> int { + constexpr auto expected_max_parse_tree_size() const -> int { return ExpectedParseTreeSize[AsInt()]; } // Test whether this token kind is in the provided list. - auto IsOneOf(std::initializer_list kinds) const -> bool { + constexpr auto IsOneOf(std::initializer_list kinds) const -> bool { for (TokenKind kind : kinds) { if (*this == kind) { return true; @@ -153,6 +161,74 @@ inline constexpr TokenKind TokenKind::KeywordTokensStorage[] = { inline constexpr llvm::ArrayRef TokenKind::KeywordTokens = KeywordTokensStorage; +inline constexpr bool TokenKind::IsSymbol[] = { +#define CARBON_TOKEN(TokenName) false, +#define CARBON_SYMBOL_TOKEN(TokenName, Spelling) true, +#include "toolchain/lex/token_kind.def" +}; + +inline constexpr bool TokenKind::IsGroupingSymbol[] = { +#define CARBON_TOKEN(TokenName) false, +#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \ + true, +#define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \ + true, +#include "toolchain/lex/token_kind.def" +}; + +inline constexpr bool TokenKind::IsOpeningSymbol[] = { +#define CARBON_TOKEN(TokenName) false, +#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \ + true, +#include "toolchain/lex/token_kind.def" +}; + +inline constexpr TokenKind TokenKind::ClosingSymbol[] = { +#define CARBON_TOKEN(TokenName) Error, +#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \ + ClosingName, +#include "toolchain/lex/token_kind.def" +}; + +inline constexpr bool TokenKind::IsClosingSymbol[] = { +#define CARBON_TOKEN(TokenName) false, +#define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \ + true, +#include "toolchain/lex/token_kind.def" +}; + +inline constexpr TokenKind TokenKind::OpeningSymbol[] = { +#define CARBON_TOKEN(TokenName) Error, +#define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \ + OpeningName, +#include "toolchain/lex/token_kind.def" +}; + +inline constexpr bool TokenKind::IsOneCharSymbol[] = { +#define CARBON_TOKEN(TokenName) false, +#define CARBON_ONE_CHAR_SYMBOL_TOKEN(TokenName, Spelling) true, +#include "toolchain/lex/token_kind.def" +}; + +inline constexpr bool TokenKind::IsKeyword[] = { +#define CARBON_TOKEN(TokenName) false, +#define CARBON_KEYWORD_TOKEN(TokenName, Spelling) true, +#include "toolchain/lex/token_kind.def" +}; + +inline constexpr llvm::StringLiteral TokenKind::FixedSpelling[] = { +#define CARBON_TOKEN(TokenName) "", +#define CARBON_SYMBOL_TOKEN(TokenName, Spelling) Spelling, +#define CARBON_KEYWORD_TOKEN(TokenName, Spelling) Spelling, +#include "toolchain/lex/token_kind.def" +}; + +inline constexpr int8_t TokenKind::ExpectedParseTreeSize[] = { +#define CARBON_TOKEN(Name) 1, +#define CARBON_TOKEN_WITH_VIRTUAL_NODE(size) 2, +#include "toolchain/lex/token_kind.def" +}; + } // namespace Carbon::Lex // We use formatv primarily for diagnostics. In these cases, it's expected that