diff --git a/toolchain/lexer/token_kind.cpp b/toolchain/lexer/token_kind.cpp index c50caa6ead98..f22fb5e47e8e 100644 --- a/toolchain/lexer/token_kind.cpp +++ b/toolchain/lexer/token_kind.cpp @@ -4,9 +4,6 @@ #include "toolchain/lexer/token_kind.h" -#include "common/check.h" -#include "llvm/ADT/StringRef.h" - namespace Carbon { CARBON_DEFINE_ENUM_CLASS_NAMES(TokenKind) = { @@ -14,112 +11,72 @@ CARBON_DEFINE_ENUM_CLASS_NAMES(TokenKind) = { #include "toolchain/lexer/token_kind.def" }; -auto TokenKind::is_symbol() const -> bool { - static constexpr bool Table[] = { +constexpr bool TokenKind::IsSymbol[] = { #define CARBON_TOKEN(TokenName) false, #define CARBON_SYMBOL_TOKEN(TokenName, Spelling) true, #include "toolchain/lexer/token_kind.def" - }; - return Table[AsInt()]; -} +}; -auto TokenKind::is_grouping_symbol() const -> bool { - static constexpr bool Table[] = { +constexpr bool TokenKind::IsGroupingSymbol[] = { #define CARBON_TOKEN(TokenName) false, #define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \ true, #define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \ true, #include "toolchain/lexer/token_kind.def" - }; - return Table[AsInt()]; -} +}; -auto TokenKind::is_opening_symbol() const -> bool { - static constexpr bool Table[] = { +constexpr bool TokenKind::IsOpeningSymbol[] = { #define CARBON_TOKEN(TokenName) false, #define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \ true, #include "toolchain/lexer/token_kind.def" - }; - return Table[AsInt()]; -} +}; -auto TokenKind::closing_symbol() const -> TokenKind { - static constexpr TokenKind Table[] = { +constexpr TokenKind TokenKind::ClosingSymbol[] = { #define CARBON_TOKEN(TokenName) Error, #define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \ ClosingName, #include "toolchain/lexer/token_kind.def" - }; - auto result = Table[AsInt()]; - CARBON_CHECK(result != Error) << "Only opening symbols are valid!"; - return result; -} +}; -auto TokenKind::is_closing_symbol() const -> bool { - static constexpr bool Table[] = { +constexpr bool TokenKind::IsClosingSymbol[] = { #define CARBON_TOKEN(TokenName) false, #define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \ true, #include "toolchain/lexer/token_kind.def" - }; - return Table[AsInt()]; -} +}; -auto TokenKind::opening_symbol() const -> TokenKind { - static constexpr TokenKind Table[] = { +constexpr TokenKind TokenKind::OpeningSymbol[] = { #define CARBON_TOKEN(TokenName) Error, #define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \ OpeningName, #include "toolchain/lexer/token_kind.def" - }; - auto result = Table[AsInt()]; - CARBON_CHECK(result != Error) << "Only closing symbols are valid!"; - return result; -} +}; -auto TokenKind::is_one_char_symbol() const -> bool { - static constexpr bool Table[] = { +constexpr bool TokenKind::IsOneCharSymbol[] = { #define CARBON_TOKEN(TokenName) false, #define CARBON_ONE_CHAR_SYMBOL_TOKEN(TokenName, Spelling) true, #include "toolchain/lexer/token_kind.def" - }; - return Table[AsInt()]; -} +}; -auto TokenKind::is_keyword() const -> bool { - static constexpr bool Table[] = { +constexpr bool TokenKind::IsKeyword[] = { #define CARBON_TOKEN(TokenName) false, #define CARBON_KEYWORD_TOKEN(TokenName, Spelling) true, #include "toolchain/lexer/token_kind.def" - }; - return Table[AsInt()]; -} +}; -auto TokenKind::is_sized_type_literal() const -> bool { - return *this == TokenKind::IntegerTypeLiteral || - *this == TokenKind::UnsignedIntegerTypeLiteral || - *this == TokenKind::FloatingPointTypeLiteral; -} - -auto TokenKind::fixed_spelling() const -> llvm::StringRef { - static constexpr llvm::StringLiteral Table[] = { +constexpr llvm::StringLiteral TokenKind::FixedSpelling[] = { #define CARBON_TOKEN(TokenName) "", #define CARBON_SYMBOL_TOKEN(TokenName, Spelling) Spelling, #define CARBON_KEYWORD_TOKEN(TokenName, Spelling) Spelling, #include "toolchain/lexer/token_kind.def" - }; - return Table[AsInt()]; -} +}; -auto TokenKind::expected_parse_tree_size() const -> int { - static constexpr int8_t Table[] = { +constexpr int8_t TokenKind::ExpectedParseTreeSize[] = { #define CARBON_TOKEN(Name) 1, #define CARBON_TOKEN_WITH_VIRTUAL_NODE(size) 2, #include "toolchain/lexer/token_kind.def" - }; - return Table[AsInt()]; -} +}; } // namespace Carbon diff --git a/toolchain/lexer/token_kind.h b/toolchain/lexer/token_kind.h index 40238d128dde..04615f0ac19e 100644 --- a/toolchain/lexer/token_kind.h +++ b/toolchain/lexer/token_kind.h @@ -7,8 +7,10 @@ #include +#include "common/check.h" #include "common/enum_base.h" #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" #include "llvm/Support/FormatVariadicDetails.h" namespace Carbon { @@ -32,45 +34,69 @@ class TokenKind : public CARBON_ENUM_BASE(TokenKind) { // inside of other tokens, outside of the contents of other tokens they // don't require any specific characters before or after to distinguish them // in the source. Returns false otherwise. - [[nodiscard]] auto is_symbol() const -> bool; + [[nodiscard]] auto is_symbol() const -> bool { return IsSymbol[AsInt()]; } // Test whether this kind of token is a grouping symbol (part of an opening // and closing pair that must always be matched in the token stream). - [[nodiscard]] auto is_grouping_symbol() const -> bool; + [[nodiscard]] auto is_grouping_symbol() const -> bool { + return IsGroupingSymbol[AsInt()]; + } // Test whether this kind of token is an opening symbol for a group. - [[nodiscard]] auto is_opening_symbol() const -> bool; + [[nodiscard]] auto is_opening_symbol() const -> bool { + return IsOpeningSymbol[AsInt()]; + } // Returns the associated closing symbol for an opening symbol. // // The token kind must be an opening symbol. - [[nodiscard]] auto closing_symbol() const -> TokenKind; + [[nodiscard]] auto closing_symbol() const -> TokenKind { + auto result = ClosingSymbol[AsInt()]; + CARBON_CHECK(result != Error) << "Only opening symbols are valid!"; + return result; + } // Test whether this kind of token is a closing symbol for a group. - [[nodiscard]] auto is_closing_symbol() const -> bool; + [[nodiscard]] auto is_closing_symbol() const -> bool { + return IsClosingSymbol[AsInt()]; + } // Returns the associated opening symbol for a closing symbol. // // The token kind must be a closing symbol. - [[nodiscard]] auto opening_symbol() const -> TokenKind; + [[nodiscard]] auto opening_symbol() const -> TokenKind { + auto result = OpeningSymbol[AsInt()]; + CARBON_CHECK(result != Error) << "Only closing symbols are valid!"; + return result; + } // Test whether this kind of token is a one-character symbol whose character // is not part of any other symbol. - [[nodiscard]] auto is_one_char_symbol() const -> bool; + [[nodiscard]] auto is_one_char_symbol() const -> bool { + return IsOneCharSymbol[AsInt()]; + }; // Test whether this kind of token is a keyword. - [[nodiscard]] auto is_keyword() const -> bool; + [[nodiscard]] auto is_keyword() const -> bool { return IsKeyword[AsInt()]; }; // Test whether this kind of token is a sized type literal. - [[nodiscard]] auto is_sized_type_literal() const -> bool; + [[nodiscard]] auto is_sized_type_literal() const -> bool { + return *this == TokenKind::IntegerTypeLiteral || + *this == TokenKind::UnsignedIntegerTypeLiteral || + *this == TokenKind::FloatingPointTypeLiteral; + }; // If this token kind has a fixed spelling when in source code, returns it. // Otherwise returns an empty string. - [[nodiscard]] auto fixed_spelling() const -> llvm::StringRef; + [[nodiscard]] auto fixed_spelling() const -> llvm::StringRef { + return FixedSpelling[AsInt()]; + }; // Get the expected number of parse tree nodes that will be created for this // token. - [[nodiscard]] auto expected_parse_tree_size() const -> int; + [[nodiscard]] auto expected_parse_tree_size() const -> int { + return ExpectedParseTreeSize[AsInt()]; + } // Test whether this token kind is in the provided list. [[nodiscard]] auto IsOneOf(std::initializer_list kinds) const @@ -85,6 +111,20 @@ class TokenKind : public CARBON_ENUM_BASE(TokenKind) { private: static const TokenKind KeywordTokensStorage[]; + + static const bool IsSymbol[]; + static const bool IsGroupingSymbol[]; + static const bool IsOpeningSymbol[]; + static const TokenKind ClosingSymbol[]; + static const bool IsClosingSymbol[]; + static const TokenKind OpeningSymbol[]; + static const bool IsOneCharSymbol[]; + + static const bool IsKeyword[]; + + static const llvm::StringLiteral FixedSpelling[]; + + static const int8_t ExpectedParseTreeSize[]; }; #define CARBON_TOKEN(TokenName) \