mirror of
https://github.com/carbon-language/carbon-lang.git
synced 2026-11-17 06:18:13 +00:00
Previously, the code would try each form of lexing and let that sub-lexing routine reject the code. This was very branch heavy and also hard to optimize -- lots of hard to inline function calls, etc. However, it's really nice to keep the different categories of lexing broken out into their own functions rather than flattening this into a huge state machine. So this creates a miniature explicit state machine by building a table of function pointers that wrap the methods on the lexer. The main lexing loop simply indexes this table with the first byte of the source code, and calls the dispatch function pointer returned. The result is that the main lex loop is *incredibly* tight code, and the only time spent appears to be stalls waiting on memory for the next byte. =] As part of this, optimize symbol lexing specifically by recognizing all the symbols that are exactly one character -- IE, we don't even need to look at the *next* character, there is no max-munch or anything else. For these, we pre-detect the exact token kind and hand that into the symbol lexing routine to avoid re-computing it. The symbols in this category are really frequent symbols in practice like `,` and `;`, so this seems likely worthwhile in practice. The one-byte-dispatch should also be reasonably extendable in the future. For example, I suspect this is the likely hot-path for non-ASCII lexing, where we see the UTF-8 marker at the start of a token and most (if not all) of the token is non-ASCII. We can use this table to dispatch immediately to an optimized routine dedicated to UTF-8 processing, without any slowdown for other inputs. The benchmark results are best for keyword lexing because that is the fastest thing to lex -- it goes form 25 mt/s to 30 mt/s. Other improvements are less dramatic, but I think this is still worthwhile because it gives a really strong basis for both expanded functionality without performance hit and further optimizations. --------- Co-authored-by: Richard Smith <richard@metafoo.co.uk>
126 lines
3.7 KiB
C++
126 lines
3.7 KiB
C++
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
|
|
// Exceptions. See /LICENSE for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
|
|
#include "toolchain/lexer/token_kind.h"
|
|
|
|
#include "common/check.h"
|
|
#include "llvm/ADT/StringRef.h"
|
|
|
|
namespace Carbon {
|
|
|
|
CARBON_DEFINE_ENUM_CLASS_NAMES(TokenKind) = {
|
|
#define CARBON_TOKEN(TokenName) CARBON_ENUM_CLASS_NAME_STRING(TokenName)
|
|
#include "toolchain/lexer/token_kind.def"
|
|
};
|
|
|
|
auto TokenKind::is_symbol() const -> bool {
|
|
static constexpr bool Table[] = {
|
|
#define CARBON_TOKEN(TokenName) false,
|
|
#define CARBON_SYMBOL_TOKEN(TokenName, Spelling) true,
|
|
#include "toolchain/lexer/token_kind.def"
|
|
};
|
|
return Table[AsInt()];
|
|
}
|
|
|
|
auto TokenKind::is_grouping_symbol() const -> bool {
|
|
static constexpr bool Table[] = {
|
|
#define CARBON_TOKEN(TokenName) false,
|
|
#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \
|
|
true,
|
|
#define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \
|
|
true,
|
|
#include "toolchain/lexer/token_kind.def"
|
|
};
|
|
return Table[AsInt()];
|
|
}
|
|
|
|
auto TokenKind::is_opening_symbol() const -> bool {
|
|
static constexpr bool Table[] = {
|
|
#define CARBON_TOKEN(TokenName) false,
|
|
#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \
|
|
true,
|
|
#include "toolchain/lexer/token_kind.def"
|
|
};
|
|
return Table[AsInt()];
|
|
}
|
|
|
|
auto TokenKind::closing_symbol() const -> TokenKind {
|
|
static constexpr TokenKind Table[] = {
|
|
#define CARBON_TOKEN(TokenName) Error,
|
|
#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \
|
|
ClosingName,
|
|
#include "toolchain/lexer/token_kind.def"
|
|
};
|
|
auto result = Table[AsInt()];
|
|
CARBON_CHECK(result != Error) << "Only opening symbols are valid!";
|
|
return result;
|
|
}
|
|
|
|
auto TokenKind::is_closing_symbol() const -> bool {
|
|
static constexpr bool Table[] = {
|
|
#define CARBON_TOKEN(TokenName) false,
|
|
#define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \
|
|
true,
|
|
#include "toolchain/lexer/token_kind.def"
|
|
};
|
|
return Table[AsInt()];
|
|
}
|
|
|
|
auto TokenKind::opening_symbol() const -> TokenKind {
|
|
static constexpr TokenKind Table[] = {
|
|
#define CARBON_TOKEN(TokenName) Error,
|
|
#define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \
|
|
OpeningName,
|
|
#include "toolchain/lexer/token_kind.def"
|
|
};
|
|
auto result = Table[AsInt()];
|
|
CARBON_CHECK(result != Error) << "Only closing symbols are valid!";
|
|
return result;
|
|
}
|
|
|
|
auto TokenKind::is_one_char_symbol() const -> bool {
|
|
static constexpr bool Table[] = {
|
|
#define CARBON_TOKEN(TokenName) false,
|
|
#define CARBON_ONE_CHAR_SYMBOL_TOKEN(TokenName, Spelling) true,
|
|
#include "toolchain/lexer/token_kind.def"
|
|
};
|
|
return Table[AsInt()];
|
|
}
|
|
|
|
auto TokenKind::is_keyword() const -> bool {
|
|
static constexpr bool Table[] = {
|
|
#define CARBON_TOKEN(TokenName) false,
|
|
#define CARBON_KEYWORD_TOKEN(TokenName, Spelling) true,
|
|
#include "toolchain/lexer/token_kind.def"
|
|
};
|
|
return Table[AsInt()];
|
|
}
|
|
|
|
auto TokenKind::is_sized_type_literal() const -> bool {
|
|
return *this == TokenKind::IntegerTypeLiteral ||
|
|
*this == TokenKind::UnsignedIntegerTypeLiteral ||
|
|
*this == TokenKind::FloatingPointTypeLiteral;
|
|
}
|
|
|
|
auto TokenKind::fixed_spelling() const -> llvm::StringRef {
|
|
static constexpr llvm::StringLiteral Table[] = {
|
|
#define CARBON_TOKEN(TokenName) "",
|
|
#define CARBON_SYMBOL_TOKEN(TokenName, Spelling) Spelling,
|
|
#define CARBON_KEYWORD_TOKEN(TokenName, Spelling) Spelling,
|
|
#include "toolchain/lexer/token_kind.def"
|
|
};
|
|
return Table[AsInt()];
|
|
}
|
|
|
|
auto TokenKind::expected_parse_tree_size() const -> int {
|
|
static constexpr int8_t Table[] = {
|
|
#define CARBON_TOKEN(Name) 1,
|
|
#define CARBON_TOKEN_WITH_VIRTUAL_NODE(size) 2,
|
|
#include "toolchain/lexer/token_kind.def"
|
|
};
|
|
return Table[AsInt()];
|
|
}
|
|
|
|
} // namespace Carbon
|