Files
carbon-lang/toolchain/lex/token_kind.h
Chandler Carruth 07718a8329 Switch symbol token to be templated on its character to simplify its dispatch (#8041)
Instead of using a table to select a bool parameter, this embeds the
first character as a template parameter and uses `if constexpr` in the
dispatch body to only test multi-character symbols with that start with
that character.

Also template the one-char symbol lexing dispatch routines to remove
runtime lookup in the bool table, and use constexpr property queries of
the token kinds.

Last but not least, add a table to precompute a table at compile time to
synthesize `LexError` based on the last byte.

Benchmark results (./scripts/bench_runner.py --runs=12,
-march=x86-64-v3):

- BM_CompileApiFileDenseDecls<Lang::Carbon, Phase::Lex>/65536: CPU Time:
-5.297%
- BM_CompileApiFileDenseDecls<Lang::Carbon, Phase::Lex>/262144: CPU
Time: -4.936%
- BM_RandomSource: CPU Time: -4.475%, tokens_per_second: +4.684%
- BM_GroupingSymbols/32/32/32: CPU Time: -3.215%, tokens_per_second:
+3.321%

Assisted-by: Antigravity with Gemini
2026-10-28 16:48:52 +00:00

251 lines
8.3 KiB
C++

// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
// Exceptions. See /LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#ifndef CARBON_TOOLCHAIN_LEX_TOKEN_KIND_H_
#define CARBON_TOOLCHAIN_LEX_TOKEN_KIND_H_
#include <cstdint>
#include "common/check.h"
#include "common/enum_base.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Support/FormatVariadicDetails.h"
namespace Carbon::Lex {
CARBON_DEFINE_RAW_ENUM_CLASS(TokenKind, uint8_t) {
#define CARBON_TOKEN(TokenName) CARBON_RAW_ENUM_ENUMERATOR(TokenName)
#include "toolchain/lex/token_kind.def"
};
class TokenKind : public CARBON_ENUM_BASE(TokenKind) {
public:
#define CARBON_TOKEN(TokenName) CARBON_ENUM_CONSTANT_DECL(TokenName)
#include "toolchain/lex/token_kind.def"
// An array of all the keyword tokens.
static const llvm::ArrayRef<TokenKind> KeywordTokens;
using EnumBase::EnumBase;
// Permit creation from RawEnumType for templates.
using EnumBase::Make;
// Permit conversion to integer for use as an array index.
using EnumBase::AsInt;
// Test whether this kind of token is a simple symbol sequence (punctuation,
// not letters) that appears directly in the source text and can be
// unambiguously lexed with `starts_with` logic. While these may appear
// inside of other tokens, outside of the contents of other tokens they
// don't require any specific characters before or after to distinguish them
// in the source. Returns false otherwise.
constexpr auto is_symbol() const -> bool { return IsSymbol[AsInt()]; }
// Test whether this kind of token is a grouping symbol (part of an opening
// and closing pair that must always be matched in the token stream).
constexpr auto is_grouping_symbol() const -> bool {
return IsGroupingSymbol[AsInt()];
}
// Test whether this kind of token is an opening symbol for a group.
constexpr auto is_opening_symbol() const -> bool {
return IsOpeningSymbol[AsInt()];
}
// Returns the associated closing symbol for an opening symbol.
//
// The token kind must be an opening symbol.
constexpr auto closing_symbol() const -> TokenKind {
auto result = ClosingSymbol[AsInt()];
CARBON_DCHECK(result != Error, "Only opening symbols are valid!");
return result;
}
// Test whether this kind of token is a closing symbol for a group.
constexpr auto is_closing_symbol() const -> bool {
return IsClosingSymbol[AsInt()];
}
// Returns the associated opening symbol for a closing symbol.
//
// The token kind must be a closing symbol.
constexpr auto opening_symbol() const -> TokenKind {
auto result = OpeningSymbol[AsInt()];
CARBON_DCHECK(result != Error, "Only closing symbols are valid!");
return result;
}
// Test whether this kind of token is a one-character symbol whose character
// is not part of any other symbol.
constexpr auto is_one_char_symbol() const -> bool {
return IsOneCharSymbol[AsInt()];
}
// Test whether this kind of token is a keyword.
constexpr auto is_keyword() const -> bool { return IsKeyword[AsInt()]; }
// Test whether this kind of token is a sized type literal.
constexpr auto is_sized_type_literal() const -> bool {
return *this == TokenKind::IntTypeLiteral ||
*this == TokenKind::UnsignedIntTypeLiteral ||
*this == TokenKind::FloatTypeLiteral;
}
// Test whether this kind of token is a dollar int literal.
constexpr auto is_dollar_int_literal() const -> bool {
return *this == TokenKind::DollarIntLiteral;
}
// Test whether this kind of token is a word.
constexpr auto is_word() const -> bool {
return *this == TokenKind::Identifier || *this == TokenKind::Underscore ||
is_keyword() || is_sized_type_literal();
}
// Test whether this kind of token is a binding pattern operator.
constexpr auto is_binding_pattern_operator() const -> bool {
return *this == TokenKind::Colon || *this == TokenKind::ColonQuestion;
}
// If this token kind has a fixed spelling when in source code, returns it.
// Otherwise returns an empty string.
constexpr auto fixed_spelling() const -> llvm::StringLiteral {
return FixedSpelling[AsInt()];
}
// Get the expected number of parse tree nodes that will be created for this
// token.
constexpr auto expected_max_parse_tree_size() const -> int {
return ExpectedParseTreeSize[AsInt()];
}
// Test whether this token kind is in the provided list.
constexpr auto IsOneOf(std::initializer_list<TokenKind> kinds) const -> bool {
for (TokenKind kind : kinds) {
if (*this == kind) {
return true;
}
}
return false;
}
private:
static const TokenKind KeywordTokensStorage[];
static const bool IsSymbol[];
static const bool IsGroupingSymbol[];
static const bool IsOpeningSymbol[];
static const TokenKind ClosingSymbol[];
static const bool IsClosingSymbol[];
static const TokenKind OpeningSymbol[];
static const bool IsOneCharSymbol[];
static const bool IsKeyword[];
static const llvm::StringLiteral FixedSpelling[];
static const int8_t ExpectedParseTreeSize[];
};
#define CARBON_TOKEN(TokenName) \
CARBON_ENUM_CONSTANT_DEFINITION(TokenKind, TokenName)
#include "toolchain/lex/token_kind.def"
inline constexpr TokenKind TokenKind::KeywordTokensStorage[] = {
#define CARBON_KEYWORD_TOKEN(TokenName, Spelling) TokenKind::TokenName,
#include "toolchain/lex/token_kind.def"
};
inline constexpr llvm::ArrayRef<TokenKind> TokenKind::KeywordTokens =
KeywordTokensStorage;
inline constexpr bool TokenKind::IsSymbol[] = {
#define CARBON_TOKEN(TokenName) false,
#define CARBON_SYMBOL_TOKEN(TokenName, Spelling) true,
#include "toolchain/lex/token_kind.def"
};
inline constexpr bool TokenKind::IsGroupingSymbol[] = {
#define CARBON_TOKEN(TokenName) false,
#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \
true,
#define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \
true,
#include "toolchain/lex/token_kind.def"
};
inline constexpr bool TokenKind::IsOpeningSymbol[] = {
#define CARBON_TOKEN(TokenName) false,
#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \
true,
#include "toolchain/lex/token_kind.def"
};
inline constexpr TokenKind TokenKind::ClosingSymbol[] = {
#define CARBON_TOKEN(TokenName) Error,
#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \
ClosingName,
#include "toolchain/lex/token_kind.def"
};
inline constexpr bool TokenKind::IsClosingSymbol[] = {
#define CARBON_TOKEN(TokenName) false,
#define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \
true,
#include "toolchain/lex/token_kind.def"
};
inline constexpr TokenKind TokenKind::OpeningSymbol[] = {
#define CARBON_TOKEN(TokenName) Error,
#define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \
OpeningName,
#include "toolchain/lex/token_kind.def"
};
inline constexpr bool TokenKind::IsOneCharSymbol[] = {
#define CARBON_TOKEN(TokenName) false,
#define CARBON_ONE_CHAR_SYMBOL_TOKEN(TokenName, Spelling) true,
#include "toolchain/lex/token_kind.def"
};
inline constexpr bool TokenKind::IsKeyword[] = {
#define CARBON_TOKEN(TokenName) false,
#define CARBON_KEYWORD_TOKEN(TokenName, Spelling) true,
#include "toolchain/lex/token_kind.def"
};
inline constexpr llvm::StringLiteral TokenKind::FixedSpelling[] = {
#define CARBON_TOKEN(TokenName) "",
#define CARBON_SYMBOL_TOKEN(TokenName, Spelling) Spelling,
#define CARBON_KEYWORD_TOKEN(TokenName, Spelling) Spelling,
#include "toolchain/lex/token_kind.def"
};
inline constexpr int8_t TokenKind::ExpectedParseTreeSize[] = {
#define CARBON_TOKEN(Name) 1,
#define CARBON_TOKEN_WITH_VIRTUAL_NODE(size) 2,
#include "toolchain/lex/token_kind.def"
};
} // namespace Carbon::Lex
// We use formatv primarily for diagnostics. In these cases, it's expected that
// the spelling in source code should be used.
template <>
struct llvm::format_provider<Carbon::Lex::TokenKind> {
static void format(const Carbon::Lex::TokenKind& kind, raw_ostream& out,
StringRef /*style*/) {
auto spelling = kind.fixed_spelling();
if (!spelling.empty()) {
out << spelling;
} else {
// Default to the name if there's no fixed spelling.
out << kind;
}
}
};
#endif // CARBON_TOOLCHAIN_LEX_TOKEN_KIND_H_