carbon-language/carbon-lang
1
0
Fork 0
mirror of https://github.com/carbon-language/carbon-lang.git synced 2026-09-24 20:20:10 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Switch TokenKind to EnumBase (#2509)

Browse Source 
This shouldn't have any behavior change, it's just using #2504

Co-authored-by: Chandler Carruth <chandlerc@gmail.com>
This commit is contained in:
Jon Ross-Perkins
2023-01-05 14:11:30 -08:00
committed by GitHub
co-authored by Chandler Carruth
parent 97634a5e91
commit 78ac6cb7d1
12 changed files with 576 additions and 640 deletions
Download Patch File Download Diff File Expand all files Collapse all files
toolchain/lexer/BUILD
+1
View File
@@ -14,6 +14,7 @@ cc_library(
deps = [
"//common:check",
"//common:ostream",
"//toolchain/common:enum_base",
"@llvm-project//llvm:Support",
],
)
toolchain/lexer/token_kind.cpp
+20 -23
View File
@@ -9,13 +9,10 @@
namespace Carbon {
auto TokenKind::name() const -> llvm::StringRef {
static constexpr llvm::StringLiteral Names[] = {
#define CARBON_TOKEN(TokenName) #TokenName,
CARBON_DEFINE_ENUM_CLASS_NAMES(TokenKind) = {
#define CARBON_TOKEN(TokenName) CARBON_ENUM_CLASS_NAME_STRING(TokenName)
#include "toolchain/lexer/token_kind.def"
};
return Names[static_cast<int>(kind_value_)];
}
};
auto TokenKind::is_symbol() const -> bool {
static constexpr bool Table[] = {
@@ -23,7 +20,7 @@ auto TokenKind::is_symbol() const -> bool {
#define CARBON_SYMBOL_TOKEN(TokenName, Spelling) true,
#include "toolchain/lexer/token_kind.def"
};
return Table[static_cast<int>(kind_value_)];
return Table[AsInt()];
}
auto TokenKind::is_grouping_symbol() const -> bool {
@@ -35,7 +32,7 @@ auto TokenKind::is_grouping_symbol() const -> bool {
true,
#include "toolchain/lexer/token_kind.def"
};
return Table[static_cast<int>(kind_value_)];
return Table[AsInt()];
}
auto TokenKind::is_opening_symbol() const -> bool {
@@ -45,18 +42,18 @@ auto TokenKind::is_opening_symbol() const -> bool {
true,
#include "toolchain/lexer/token_kind.def"
};
return Table[static_cast<int>(kind_value_)];
return Table[AsInt()];
}
auto TokenKind::closing_symbol() const -> TokenKind {
static constexpr TokenKind Table[] = {
#define CARBON_TOKEN(TokenName) Error(),
#define CARBON_TOKEN(TokenName) Error,
#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \
ClosingName(),
ClosingName,
#include "toolchain/lexer/token_kind.def"
};
auto result = Table[static_cast<int>(kind_value_)];
CARBON_CHECK(result != Error()) << "Only opening symbols are valid!";
auto result = Table[AsInt()];
CARBON_CHECK(result != Error) << "Only opening symbols are valid!";
return result;
}
@@ -67,18 +64,18 @@ auto TokenKind::is_closing_symbol() const -> bool {
true,
#include "toolchain/lexer/token_kind.def"
};
return Table[static_cast<int>(kind_value_)];
return Table[AsInt()];
}
auto TokenKind::opening_symbol() const -> TokenKind {
static constexpr TokenKind Table[] = {
#define CARBON_TOKEN(TokenName) Error(),
#define CARBON_TOKEN(TokenName) Error,
#define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \
OpeningName(),
OpeningName,
#include "toolchain/lexer/token_kind.def"
};
auto result = Table[static_cast<int>(kind_value_)];
CARBON_CHECK(result != Error()) << "Only closing symbols are valid!";
auto result = Table[AsInt()];
CARBON_CHECK(result != Error) << "Only closing symbols are valid!";
return result;
}
@@ -88,13 +85,13 @@ auto TokenKind::is_keyword() const -> bool {
#define CARBON_KEYWORD_TOKEN(TokenName, Spelling) true,
#include "toolchain/lexer/token_kind.def"
};
return Table[static_cast<int>(kind_value_)];
return Table[AsInt()];
}
auto TokenKind::is_sized_type_literal() const -> bool {
return *this == TokenKind::IntegerTypeLiteral() ||
*this == TokenKind::UnsignedIntegerTypeLiteral() ||
*this == TokenKind::FloatingPointTypeLiteral();
return *this == TokenKind::IntegerTypeLiteral ||
*this == TokenKind::UnsignedIntegerTypeLiteral ||
*this == TokenKind::FloatingPointTypeLiteral;
}
auto TokenKind::fixed_spelling() const -> llvm::StringRef {
@@ -104,7 +101,7 @@ auto TokenKind::fixed_spelling() const -> llvm::StringRef {
#define CARBON_KEYWORD_TOKEN(TokenName, Spelling) Spelling,
#include "toolchain/lexer/token_kind.def"
};
return Table[static_cast<int>(kind_value_)];
return Table[AsInt()];
}
} // namespace Carbon
toolchain/lexer/token_kind.h
+13 -34
View File
@@ -6,38 +6,22 @@
#define CARBON_TOOLCHAIN_LEXER_TOKEN_KIND_H_
#include <cstdint>
#include <initializer_list>
#include <iterator>
#include "common/ostream.h"
#include "llvm/ADT/StringRef.h"
#include "toolchain/common/enum_base.h"
namespace Carbon {
class TokenKind {
// Note that this must be declared earlier in the class so that its type can
// be used, for example in the conversion operator.
enum class KindEnum : int8_t {
#define CARBON_TOKEN(TokenName) TokenName,
CARBON_DEFINE_RAW_ENUM_CLASS(TokenKind, uint8_t) {
#define CARBON_TOKEN(TokenName) CARBON_RAW_ENUM_ENUMERATOR(TokenName)
#include "toolchain/lexer/token_kind.def"
};
};
class TokenKind : public CARBON_ENUM_BASE(TokenKind) {
public:
// The formatting for this macro is weird due to a `clang-format` bug. See
// https://bugs.llvm.org/show_bug.cgi?id=48320 for details.
#define CARBON_TOKEN(TokenName) \
static constexpr auto TokenName()->TokenKind { \
return TokenKind(KindEnum::TokenName); \
}
#define CARBON_TOKEN(TokenName) CARBON_ENUM_CONSTANT_DECLARATION(TokenName)
#include "toolchain/lexer/token_kind.def"
// The default constructor is deleted as objects of this type should always be
// constructed using the above factory functions for each unique kind.
TokenKind() = delete;
// Get a friendly name for the token for logging or debugging.
[[nodiscard]] auto name() const -> llvm::StringRef;
// Test whether this kind of token is a simple symbol sequence (punctuation,
// not letters) that appears directly in the source text and can be
// unambiguously lexed with `starts_with` logic. While these may appear
@@ -87,21 +71,16 @@ class TokenKind {
return false;
}
// Enable conversion to our private enum, including in a `constexpr` context,
// to enable usage in `switch` and `case`. The enum remains private and
// nothing else should be using this.
// NOLINTNEXTLINE(google-explicit-constructor)
constexpr operator KindEnum() const { return kind_value_; }
// Prints the TokenKind, typically for diagnostics.
// Override the EnumBase printing to use the fixed spelling rather than the
// name for tokens as this better corresponds to the source code the
// represent.
void Print(llvm::raw_ostream& out) const { out << fixed_spelling(); }
private:
constexpr explicit TokenKind(KindEnum kind_value) : kind_value_(kind_value) {}
KindEnum kind_value_;
};
#define CARBON_TOKEN(TokenName) \
CARBON_ENUM_CONSTANT_DEFINITION(TokenKind, TokenName)
#include "toolchain/lexer/token_kind.def"
} // namespace Carbon
#endif // CARBON_TOOLCHAIN_LEXER_TOKEN_KIND_H_
toolchain/lexer/token_kind_test.cpp
+45 -47
View File
@@ -25,57 +25,55 @@ constexpr llvm::StringLiteral SymbolRegex =
// exception: `Self`.
constexpr llvm::StringLiteral KeywordRegex = "[a-z_]+|Self";
#define CARBON_TOKEN(TokenName) \
TEST(TokenKindTest, TokenName) { \
EXPECT_EQ(#TokenName, TokenKind::TokenName().name()); \
EXPECT_FALSE(TokenKind::TokenName().is_symbol()); \
EXPECT_FALSE(TokenKind::TokenName().is_keyword()); \
EXPECT_EQ("", TokenKind::TokenName().fixed_spelling()); \
#define CARBON_TOKEN(TokenName) \
TEST(TokenKindTest, TokenName) { \
EXPECT_EQ(#TokenName, TokenKind::TokenName.name()); \
EXPECT_FALSE(TokenKind::TokenName.is_symbol()); \
EXPECT_FALSE(TokenKind::TokenName.is_keyword()); \
EXPECT_EQ("", TokenKind::TokenName.fixed_spelling()); \
}
#define CARBON_SYMBOL_TOKEN(TokenName, Spelling) \
TEST(TokenKindTest, TokenName) { \
EXPECT_EQ(#TokenName, TokenKind::TokenName().name()); \
EXPECT_TRUE(TokenKind::TokenName().is_symbol()); \
EXPECT_FALSE(TokenKind::TokenName().is_grouping_symbol()); \
EXPECT_FALSE(TokenKind::TokenName().is_opening_symbol()); \
EXPECT_FALSE(TokenKind::TokenName().is_closing_symbol()); \
EXPECT_FALSE(TokenKind::TokenName().is_keyword()); \
EXPECT_EQ(Spelling, TokenKind::TokenName().fixed_spelling()); \
EXPECT_THAT(Spelling, MatchesRegex(SymbolRegex.str())); \
#define CARBON_SYMBOL_TOKEN(TokenName, Spelling) \
TEST(TokenKindTest, TokenName) { \
EXPECT_EQ(#TokenName, TokenKind::TokenName.name()); \
EXPECT_TRUE(TokenKind::TokenName.is_symbol()); \
EXPECT_FALSE(TokenKind::TokenName.is_grouping_symbol()); \
EXPECT_FALSE(TokenKind::TokenName.is_opening_symbol()); \
EXPECT_FALSE(TokenKind::TokenName.is_closing_symbol()); \
EXPECT_FALSE(TokenKind::TokenName.is_keyword()); \
EXPECT_EQ(Spelling, TokenKind::TokenName.fixed_spelling()); \
EXPECT_THAT(Spelling, MatchesRegex(SymbolRegex.str())); \
}
#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \
TEST(TokenKindTest, TokenName) { \
EXPECT_EQ(#TokenName, TokenKind::TokenName().name()); \
EXPECT_TRUE(TokenKind::TokenName().is_symbol()); \
EXPECT_TRUE(TokenKind::TokenName().is_grouping_symbol()); \
EXPECT_TRUE(TokenKind::TokenName().is_opening_symbol()); \
EXPECT_EQ(TokenKind::ClosingName(), \
TokenKind::TokenName().closing_symbol()); \
EXPECT_FALSE(TokenKind::TokenName().is_closing_symbol()); \
EXPECT_FALSE(TokenKind::TokenName().is_keyword()); \
EXPECT_EQ(Spelling, TokenKind::TokenName().fixed_spelling()); \
EXPECT_THAT(Spelling, MatchesRegex(SymbolRegex.str())); \
#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \
TEST(TokenKindTest, TokenName) { \
EXPECT_EQ(#TokenName, TokenKind::TokenName.name()); \
EXPECT_TRUE(TokenKind::TokenName.is_symbol()); \
EXPECT_TRUE(TokenKind::TokenName.is_grouping_symbol()); \
EXPECT_TRUE(TokenKind::TokenName.is_opening_symbol()); \
EXPECT_EQ(TokenKind::ClosingName, TokenKind::TokenName.closing_symbol()); \
EXPECT_FALSE(TokenKind::TokenName.is_closing_symbol()); \
EXPECT_FALSE(TokenKind::TokenName.is_keyword()); \
EXPECT_EQ(Spelling, TokenKind::TokenName.fixed_spelling()); \
EXPECT_THAT(Spelling, MatchesRegex(SymbolRegex.str())); \
}
#define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \
TEST(TokenKindTest, TokenName) { \
EXPECT_EQ(#TokenName, TokenKind::TokenName().name()); \
EXPECT_TRUE(TokenKind::TokenName().is_symbol()); \
EXPECT_TRUE(TokenKind::TokenName().is_grouping_symbol()); \
EXPECT_FALSE(TokenKind::TokenName().is_opening_symbol()); \
EXPECT_TRUE(TokenKind::TokenName().is_closing_symbol()); \
EXPECT_EQ(TokenKind::OpeningName(), \
TokenKind::TokenName().opening_symbol()); \
EXPECT_FALSE(TokenKind::TokenName().is_keyword()); \
EXPECT_EQ(Spelling, TokenKind::TokenName().fixed_spelling()); \
EXPECT_THAT(Spelling, MatchesRegex(SymbolRegex.str())); \
#define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \
TEST(TokenKindTest, TokenName) { \
EXPECT_EQ(#TokenName, TokenKind::TokenName.name()); \
EXPECT_TRUE(TokenKind::TokenName.is_symbol()); \
EXPECT_TRUE(TokenKind::TokenName.is_grouping_symbol()); \
EXPECT_FALSE(TokenKind::TokenName.is_opening_symbol()); \
EXPECT_TRUE(TokenKind::TokenName.is_closing_symbol()); \
EXPECT_EQ(TokenKind::OpeningName, TokenKind::TokenName.opening_symbol()); \
EXPECT_FALSE(TokenKind::TokenName.is_keyword()); \
EXPECT_EQ(Spelling, TokenKind::TokenName.fixed_spelling()); \
EXPECT_THAT(Spelling, MatchesRegex(SymbolRegex.str())); \
}
#define CARBON_KEYWORD_TOKEN(TokenName, Spelling) \
TEST(TokenKindTest, TokenName) { \
EXPECT_EQ(#TokenName, TokenKind::TokenName().name()); \
EXPECT_FALSE(TokenKind::TokenName().is_symbol()); \
EXPECT_TRUE(TokenKind::TokenName().is_keyword()); \
EXPECT_EQ(Spelling, TokenKind::TokenName().fixed_spelling()); \
EXPECT_THAT(Spelling, MatchesRegex(KeywordRegex.str())); \
#define CARBON_KEYWORD_TOKEN(TokenName, Spelling) \
TEST(TokenKindTest, TokenName) { \
EXPECT_EQ(#TokenName, TokenKind::TokenName.name()); \
EXPECT_FALSE(TokenKind::TokenName.is_symbol()); \
EXPECT_TRUE(TokenKind::TokenName.is_keyword()); \
EXPECT_EQ(Spelling, TokenKind::TokenName.fixed_spelling()); \
EXPECT_THAT(Spelling, MatchesRegex(KeywordRegex.str())); \
}
#include "toolchain/lexer/token_kind.def"
toolchain/lexer/tokenized_buffer.cpp
+35 -35
View File
@@ -203,7 +203,7 @@ class TokenizedBuffer::Lexer {
return VariantMatch(
literal->ComputeValue(emitter_),
[&](LexedNumericLiteral::IntegerValue&& value) {
auto token = buffer_->AddToken({.kind = TokenKind::IntegerLiteral(),
auto token = buffer_->AddToken({.kind = TokenKind::IntegerLiteral,
.token_line = current_line_,
.column = int_column});
buffer_->GetTokenInfo(token).literal_index =
@@ -212,7 +212,7 @@ class TokenizedBuffer::Lexer {
return token;
},
[&](LexedNumericLiteral::RealValue&& value) {
auto token = buffer_->AddToken({.kind = TokenKind::RealLiteral(),
auto token = buffer_->AddToken({.kind = TokenKind::RealLiteral,
.token_line = current_line_,
.column = int_column});
buffer_->GetTokenInfo(token).literal_index =
@@ -225,7 +225,7 @@ class TokenizedBuffer::Lexer {
},
[&](LexedNumericLiteral::UnrecoverableError) {
auto token = buffer_->AddToken({
.kind = TokenKind::Error(),
.kind = TokenKind::Error,
.token_line = current_line_,
.column = int_column,
.error_length = token_size,
@@ -270,7 +270,7 @@ class TokenizedBuffer::Lexer {
if (literal->is_terminated()) {
auto token =
buffer_->AddToken({.kind = TokenKind::StringLiteral(),
buffer_->AddToken({.kind = TokenKind::StringLiteral,
.token_line = string_line,
.column = string_column,
.literal_index = static_cast<int32_t>(
@@ -282,7 +282,7 @@ class TokenizedBuffer::Lexer {
CARBON_DIAGNOSTIC(UnterminatedString, Error,
"String is missing a terminator.");
emitter_.Emit(literal->text().begin(), UnterminatedString);
return buffer_->AddToken({.kind = TokenKind::Error(),
return buffer_->AddToken({.kind = TokenKind::Error,
.token_line = string_line,
.column = string_column,
.error_length = literal_size});
@@ -292,10 +292,10 @@ class TokenizedBuffer::Lexer {
auto LexSymbolToken(llvm::StringRef& source_text) -> LexResult {
TokenKind kind = llvm::StringSwitch<TokenKind>(source_text)
#define CARBON_SYMBOL_TOKEN(Name, Spelling) \
.StartsWith(Spelling, TokenKind::Name())
.StartsWith(Spelling, TokenKind::Name)
#include "toolchain/lexer/token_kind.def"
.Default(TokenKind::Error());
if (kind == TokenKind::Error()) {
.Default(TokenKind::Error);
if (kind == TokenKind::Error) {
return LexResult::NoMatch();
}
@@ -328,7 +328,7 @@ class TokenizedBuffer::Lexer {
// Check that there is a matching opening symbol before we consume this as
// a closing symbol.
if (open_groups_.empty()) {
closing_token_info.kind = TokenKind::Error();
closing_token_info.kind = TokenKind::Error;
closing_token_info.error_length = kind.fixed_spelling().size();
CARBON_DIAGNOSTIC(
@@ -363,13 +363,13 @@ class TokenizedBuffer::Lexer {
std::optional<TokenKind> kind;
switch (word.front()) {
case 'i':
kind = TokenKind::IntegerTypeLiteral();
kind = TokenKind::IntegerTypeLiteral;
break;
case 'u':
kind = TokenKind::UnsignedIntegerTypeLiteral();
kind = TokenKind::UnsignedIntegerTypeLiteral;
break;
case 'f':
kind = TokenKind::FloatingPointTypeLiteral();
kind = TokenKind::FloatingPointTypeLiteral;
break;
default:
return LexResult::NoMatch();
@@ -378,7 +378,7 @@ class TokenizedBuffer::Lexer {
llvm::StringRef suffix = word.substr(1);
if (!CanLexInteger(emitter_, suffix)) {
return buffer_->AddToken(
{.kind = TokenKind::Error(),
{.kind = TokenKind::Error,
.token_line = current_line_,
.column = column,
.error_length = static_cast<int32_t>(word.size())});
@@ -399,7 +399,7 @@ class TokenizedBuffer::Lexer {
// Closes all open groups that cannot remain open across the symbol `K`.
// Users may pass `Error` to close all open groups.
auto CloseInvalidOpenGroups(TokenKind kind) -> void {
if (!kind.is_closing_symbol() && kind != TokenKind::Error()) {
if (!kind.is_closing_symbol() && kind != TokenKind::Error) {
return;
}
@@ -471,17 +471,17 @@ class TokenizedBuffer::Lexer {
// Check if the text matches a keyword token, and if so use that.
TokenKind kind = llvm::StringSwitch<TokenKind>(identifier_text)
#define CARBON_KEYWORD_TOKEN(Name, Spelling) .Case(Spelling, TokenKind::Name())
#define CARBON_KEYWORD_TOKEN(Name, Spelling) .Case(Spelling, TokenKind::Name)
#include "toolchain/lexer/token_kind.def"
.Default(TokenKind::Error());
if (kind != TokenKind::Error()) {
.Default(TokenKind::Error);
if (kind != TokenKind::Error) {
return buffer_->AddToken({.kind = kind,
.token_line = current_line_,
.column = identifier_column});
}
// Otherwise we have a generic identifier.
return buffer_->AddToken({.kind = TokenKind::Identifier(),
return buffer_->AddToken({.kind = TokenKind::Identifier,
.token_line = current_line_,
.column = identifier_column,
.id = GetOrCreateIdentifier(identifier_text)});
@@ -510,7 +510,7 @@ class TokenizedBuffer::Lexer {
}
auto token = buffer_->AddToken(
{.kind = TokenKind::Error(),
{.kind = TokenKind::Error,
.token_line = current_line_,
.column = current_column_,
.error_length = static_cast<int32_t>(error_text.size())});
@@ -524,7 +524,7 @@ class TokenizedBuffer::Lexer {
}
auto AddEndOfFileToken() -> void {
buffer_->AddToken({.kind = TokenKind::EndOfFile(),
buffer_->AddToken({.kind = TokenKind::EndOfFile,
.token_line = current_line_,
.column = current_column_});
}
@@ -576,7 +576,7 @@ auto TokenizedBuffer::Lex(SourceBuffer& source, DiagnosticConsumer& consumer)
// The end-of-file token is always considered to be whitespace.
lexer.NoteWhitespace();
lexer.CloseInvalidOpenGroups(TokenKind::Error());
lexer.CloseInvalidOpenGroups(TokenKind::Error);
lexer.AddEndOfFileToken();
if (error_tracking_consumer.seen_error()) {
@@ -609,7 +609,7 @@ auto TokenizedBuffer::GetTokenText(Token token) const -> llvm::StringRef {
return fixed_spelling;
}
if (token_info.kind == TokenKind::Error()) {
if (token_info.kind == TokenKind::Error) {
const auto& line_info = GetLineInfo(token_info.token_line);
int64_t token_start = line_info.start + token_info.column;
return source_->text().substr(token_start, token_info.error_length);
@@ -617,8 +617,8 @@ auto TokenizedBuffer::GetTokenText(Token token) const -> llvm::StringRef {
// Refer back to the source text to preserve oddities like radix or digit
// separators the author included.
if (token_info.kind == TokenKind::IntegerLiteral() ||
token_info.kind == TokenKind::RealLiteral()) {
if (token_info.kind == TokenKind::IntegerLiteral ||
token_info.kind == TokenKind::RealLiteral) {
const auto& line_info = GetLineInfo(token_info.token_line);
int64_t token_start = line_info.start + token_info.column;
std::optional<LexedNumericLiteral> relexed_token =
@@ -629,7 +629,7 @@ auto TokenizedBuffer::GetTokenText(Token token) const -> llvm::StringRef {
// Refer back to the source text to find the original spelling, including
// escape sequences etc.
if (token_info.kind == TokenKind::StringLiteral()) {
if (token_info.kind == TokenKind::StringLiteral) {
const auto& line_info = GetLineInfo(token_info.token_line);
int64_t token_start = line_info.start + token_info.column;
std::optional<LexedStringLiteral> relexed_token =
@@ -648,18 +648,18 @@ auto TokenizedBuffer::GetTokenText(Token token) const -> llvm::StringRef {
return llvm::StringRef(suffix.data() - 1, suffix.size() + 1);
}
if (token_info.kind == TokenKind::EndOfFile()) {
if (token_info.kind == TokenKind::EndOfFile) {
return llvm::StringRef();
}
CARBON_CHECK(token_info.kind == TokenKind::Identifier())
CARBON_CHECK(token_info.kind == TokenKind::Identifier)
<< token_info.kind.name();
return GetIdentifierText(token_info.id);
}
auto TokenizedBuffer::GetIdentifier(Token token) const -> Identifier {
const auto& token_info = GetTokenInfo(token);
CARBON_CHECK(token_info.kind == TokenKind::Identifier())
CARBON_CHECK(token_info.kind == TokenKind::Identifier)
<< token_info.kind.name();
return token_info.id;
}
@@ -667,14 +667,14 @@ auto TokenizedBuffer::GetIdentifier(Token token) const -> Identifier {
auto TokenizedBuffer::GetIntegerLiteral(Token token) const
-> const llvm::APInt& {
const auto& token_info = GetTokenInfo(token);
CARBON_CHECK(token_info.kind == TokenKind::IntegerLiteral())
CARBON_CHECK(token_info.kind == TokenKind::IntegerLiteral)
<< token_info.kind.name();
return literal_int_storage_[token_info.literal_index];
}
auto TokenizedBuffer::GetRealLiteral(Token token) const -> RealLiteralValue {
const auto& token_info = GetTokenInfo(token);
CARBON_CHECK(token_info.kind == TokenKind::RealLiteral())
CARBON_CHECK(token_info.kind == TokenKind::RealLiteral)
<< token_info.kind.name();
// Note that every real literal is at least three characters long, so we can
@@ -690,7 +690,7 @@ auto TokenizedBuffer::GetRealLiteral(Token token) const -> RealLiteralValue {
auto TokenizedBuffer::GetStringLiteral(Token token) const -> llvm::StringRef {
const auto& token_info = GetTokenInfo(token);
CARBON_CHECK(token_info.kind == TokenKind::StringLiteral())
CARBON_CHECK(token_info.kind == TokenKind::StringLiteral)
<< token_info.kind.name();
return literal_string_storage_[token_info.literal_index];
}
@@ -826,18 +826,18 @@ auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream, Token token,
token_text);
switch (token_info.kind) {
case TokenKind::Identifier():
case TokenKind::Identifier:
output_stream << ", identifier: " << GetIdentifier(token).index;
break;
case TokenKind::IntegerLiteral():
case TokenKind::IntegerLiteral:
output_stream << ", value: `";
GetIntegerLiteral(token).print(output_stream, /*isSigned=*/false);
output_stream << "`";
break;
case TokenKind::RealLiteral():
case TokenKind::RealLiteral:
output_stream << ", value: `" << GetRealLiteral(token) << "`";
break;
case TokenKind::StringLiteral():
case TokenKind::StringLiteral:
output_stream << ", value: `" << GetStringLiteral(token) << "`";
break;
default:
toolchain/lexer/tokenized_buffer_test.cpp
+239 -267
View File
@@ -50,56 +50,41 @@ class LexerTest : public ::testing::Test {
TEST_F(LexerTest, HandlesEmptyBuffer) {
auto buffer = Lex("");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{{TokenKind::EndOfFile()}}));
EXPECT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{{TokenKind::EndOfFile}}));
}
TEST_F(LexerTest, TracksLinesAndColumns) {
auto buffer = Lex("\n ;;\n ;;;\n x\"foo\" '''baz\n a\n ''' y");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::Semi(),
.line = 2,
.column = 3,
.indent_column = 3},
{.kind = TokenKind::Semi(),
.line = 2,
.column = 4,
.indent_column = 3},
{.kind = TokenKind::Semi(),
.line = 3,
.column = 4,
.indent_column = 4},
{.kind = TokenKind::Semi(),
.line = 3,
.column = 5,
.indent_column = 4},
{.kind = TokenKind::Semi(),
.line = 3,
.column = 6,
.indent_column = 4},
{.kind = TokenKind::Identifier(),
.line = 4,
.column = 4,
.indent_column = 4,
.text = "x"},
{.kind = TokenKind::StringLiteral(),
.line = 4,
.column = 5,
.indent_column = 4},
{.kind = TokenKind::StringLiteral(),
.line = 4,
.column = 11,
.indent_column = 4},
{.kind = TokenKind::Identifier(),
.line = 6,
.column = 6,
.indent_column = 11,
.text = "y"},
{.kind = TokenKind::EndOfFile(), .line = 6, .column = 7},
}));
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::Semi, .line = 2, .column = 3, .indent_column = 3},
{.kind = TokenKind::Semi, .line = 2, .column = 4, .indent_column = 3},
{.kind = TokenKind::Semi, .line = 3, .column = 4, .indent_column = 4},
{.kind = TokenKind::Semi, .line = 3, .column = 5, .indent_column = 4},
{.kind = TokenKind::Semi, .line = 3, .column = 6, .indent_column = 4},
{.kind = TokenKind::Identifier,
.line = 4,
.column = 4,
.indent_column = 4,
.text = "x"},
{.kind = TokenKind::StringLiteral,
.line = 4,
.column = 5,
.indent_column = 4},
{.kind = TokenKind::StringLiteral,
.line = 4,
.column = 11,
.indent_column = 4},
{.kind = TokenKind::Identifier,
.line = 6,
.column = 6,
.indent_column = 11,
.text = "y"},
{.kind = TokenKind::EndOfFile, .line = 6, .column = 7},
}));
}
TEST_F(LexerTest, HandlesNumericLiteral) {
@@ -107,51 +92,51 @@ TEST_F(LexerTest, HandlesNumericLiteral) {
EXPECT_FALSE(buffer.has_errors());
ASSERT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::IntegerLiteral(),
{.kind = TokenKind::IntegerLiteral,
.line = 1,
.column = 1,
.indent_column = 1,
.text = "12"},
{.kind = TokenKind::Minus(),
{.kind = TokenKind::Minus,
.line = 1,
.column = 3,
.indent_column = 1},
{.kind = TokenKind::IntegerLiteral(),
{.kind = TokenKind::IntegerLiteral,
.line = 1,
.column = 4,
.indent_column = 1,
.text = "578"},
{.kind = TokenKind::IntegerLiteral(),
{.kind = TokenKind::IntegerLiteral,
.line = 2,
.column = 3,
.indent_column = 3,
.text = "1"},
{.kind = TokenKind::IntegerLiteral(),
{.kind = TokenKind::IntegerLiteral,
.line = 2,
.column = 6,
.indent_column = 3,
.text = "2"},
{.kind = TokenKind::IntegerLiteral(),
{.kind = TokenKind::IntegerLiteral,
.line = 3,
.column = 1,
.indent_column = 1,
.text = "0x12_3ABC"},
{.kind = TokenKind::IntegerLiteral(),
{.kind = TokenKind::IntegerLiteral,
.line = 4,
.column = 1,
.indent_column = 1,
.text = "0b10_10_11"},
{.kind = TokenKind::IntegerLiteral(),
{.kind = TokenKind::IntegerLiteral,
.line = 5,
.column = 1,
.indent_column = 1,
.text = "1_234_567"},
{.kind = TokenKind::RealLiteral(),
{.kind = TokenKind::RealLiteral,
.line = 6,
.column = 1,
.indent_column = 1,
.text = "1.5e9"},
{.kind = TokenKind::EndOfFile(), .line = 6, .column = 6},
{.kind = TokenKind::EndOfFile, .line = 6, .column = 6},
}));
auto token_12 = buffer.tokens().begin();
EXPECT_EQ(buffer.GetIntegerLiteral(*token_12), 12);
@@ -179,32 +164,32 @@ TEST_F(LexerTest, HandlesInvalidNumericLiterals) {
EXPECT_TRUE(buffer.has_errors());
ASSERT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::Error(),
{.kind = TokenKind::Error,
.line = 1,
.column = 1,
.indent_column = 1,
.text = "14x"},
{.kind = TokenKind::IntegerLiteral(),
{.kind = TokenKind::IntegerLiteral,
.line = 1,
.column = 5,
.indent_column = 1,
.text = "15_49"},
{.kind = TokenKind::Error(),
{.kind = TokenKind::Error,
.line = 1,
.column = 11,
.indent_column = 1,
.text = "0x3.5q"},
{.kind = TokenKind::RealLiteral(),
{.kind = TokenKind::RealLiteral,
.line = 1,
.column = 18,
.indent_column = 1,
.text = "0x3_4.5_6"},
{.kind = TokenKind::Error(),
{.kind = TokenKind::Error,
.line = 1,
.column = 28,
.indent_column = 1,
.text = "0ops"},
{.kind = TokenKind::EndOfFile(), .line = 1, .column = 32},
{.kind = TokenKind::EndOfFile, .line = 1, .column = 32},
}));
}
@@ -226,58 +211,57 @@ TEST_F(LexerTest, SplitsNumericLiteralsProperly) {
)";
auto buffer = Lex(source_text);
EXPECT_TRUE(buffer.has_errors());
EXPECT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::IntegerLiteral(), .text = "1"},
{.kind = TokenKind::Period()},
// newline
{.kind = TokenKind::Period()},
{.kind = TokenKind::IntegerLiteral(), .text = "2"},
// newline
{.kind = TokenKind::IntegerLiteral(), .text = "3"},
{.kind = TokenKind::Period()},
{.kind = TokenKind::Plus()},
{.kind = TokenKind::Identifier(), .text = "foo"},
// newline
{.kind = TokenKind::RealLiteral(), .text = "4.0"},
{.kind = TokenKind::Minus()},
{.kind = TokenKind::Identifier(), .text = "bar"},
// newline
{.kind = TokenKind::RealLiteral(), .text = "5.0e+123"},
{.kind = TokenKind::Plus()},
{.kind = TokenKind::IntegerLiteral(), .text = "456"},
// newline
{.kind = TokenKind::Error(), .text = "6.0e+1e"},
{.kind = TokenKind::Plus()},
{.kind = TokenKind::IntegerLiteral(), .text = "2"},
// newline
{.kind = TokenKind::Error(), .text = "1e7"},
// newline
{.kind = TokenKind::IntegerLiteral(), .text = "8"},
{.kind = TokenKind::Period()},
{.kind = TokenKind::Period()},
{.kind = TokenKind::IntegerLiteral(), .text = "10"},
// newline
{.kind = TokenKind::RealLiteral(), .text = "9.0"},
{.kind = TokenKind::Period()},
{.kind = TokenKind::RealLiteral(), .text = "9.5"},
// newline
{.kind = TokenKind::Error(), .text = "10.foo"},
// newline
{.kind = TokenKind::RealLiteral(), .text = "11.0"},
{.kind = TokenKind::Period()},
{.kind = TokenKind::Identifier(), .text = "foo"},
// newline
{.kind = TokenKind::Error(), .text = "12e"},
{.kind = TokenKind::Plus()},
{.kind = TokenKind::IntegerLiteral(), .text = "1"},
// newline
{.kind = TokenKind::IntegerLiteral(), .text = "13"},
{.kind = TokenKind::Period()},
{.kind = TokenKind::Underscore()},
// newline
{.kind = TokenKind::EndOfFile()},
}));
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::IntegerLiteral, .text = "1"},
{.kind = TokenKind::Period},
// newline
{.kind = TokenKind::Period},
{.kind = TokenKind::IntegerLiteral, .text = "2"},
// newline
{.kind = TokenKind::IntegerLiteral, .text = "3"},
{.kind = TokenKind::Period},
{.kind = TokenKind::Plus},
{.kind = TokenKind::Identifier, .text = "foo"},
// newline
{.kind = TokenKind::RealLiteral, .text = "4.0"},
{.kind = TokenKind::Minus},
{.kind = TokenKind::Identifier, .text = "bar"},
// newline
{.kind = TokenKind::RealLiteral, .text = "5.0e+123"},
{.kind = TokenKind::Plus},
{.kind = TokenKind::IntegerLiteral, .text = "456"},
// newline
{.kind = TokenKind::Error, .text = "6.0e+1e"},
{.kind = TokenKind::Plus},
{.kind = TokenKind::IntegerLiteral, .text = "2"},
// newline
{.kind = TokenKind::Error, .text = "1e7"},
// newline
{.kind = TokenKind::IntegerLiteral, .text = "8"},
{.kind = TokenKind::Period},
{.kind = TokenKind::Period},
{.kind = TokenKind::IntegerLiteral, .text = "10"},
// newline
{.kind = TokenKind::RealLiteral, .text = "9.0"},
{.kind = TokenKind::Period},
{.kind = TokenKind::RealLiteral, .text = "9.5"},
// newline
{.kind = TokenKind::Error, .text = "10.foo"},
// newline
{.kind = TokenKind::RealLiteral, .text = "11.0"},
{.kind = TokenKind::Period},
{.kind = TokenKind::Identifier, .text = "foo"},
// newline
{.kind = TokenKind::Error, .text = "12e"},
{.kind = TokenKind::Plus},
{.kind = TokenKind::IntegerLiteral, .text = "1"},
// newline
{.kind = TokenKind::IntegerLiteral, .text = "13"},
{.kind = TokenKind::Period},
{.kind = TokenKind::Underscore},
// newline
{.kind = TokenKind::EndOfFile},
}));
}
TEST_F(LexerTest, HandlesGarbageCharacters) {
@@ -287,32 +271,29 @@ TEST_F(LexerTest, HandlesGarbageCharacters) {
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::Error(),
{.kind = TokenKind::Error,
.line = 1,
.column = 1,
// 💩 takes 4 bytes, and we count column as bytes offset.
.text = llvm::StringRef("$$💩", 6)},
{.kind = TokenKind::Minus(), .line = 1, .column = 7},
{.kind = TokenKind::Error(), .line = 1, .column = 8, .text = "$"},
{.kind = TokenKind::Minus, .line = 1, .column = 7},
{.kind = TokenKind::Error, .line = 1, .column = 8, .text = "$"},
// newline
{.kind = TokenKind::Error(),
{.kind = TokenKind::Error,
.line = 2,
.column = 1,
.text = llvm::StringRef("$\0$", 3)},
{.kind = TokenKind::IntegerLiteral(),
{.kind = TokenKind::IntegerLiteral,
.line = 2,
.column = 4,
.text = "12"},
{.kind = TokenKind::Error(), .line = 2, .column = 6, .text = "$"},
{.kind = TokenKind::Error, .line = 2, .column = 6, .text = "$"},
// newline
{.kind = TokenKind::Backslash(),
.line = 3,
.column = 1,
.text = "\\"},
{.kind = TokenKind::Error(), .line = 3, .column = 2, .text = "\"\\"},
{.kind = TokenKind::Backslash, .line = 3, .column = 1, .text = "\\"},
{.kind = TokenKind::Error, .line = 3, .column = 2, .text = "\"\\"},
// newline
{.kind = TokenKind::Error(), .line = 4, .column = 1, .text = "\"x"},
{.kind = TokenKind::EndOfFile(), .line = 4, .column = 3},
{.kind = TokenKind::Error, .line = 4, .column = 1, .text = "\"x"},
{.kind = TokenKind::EndOfFile, .line = 4, .column = 3},
}));
}
@@ -323,39 +304,39 @@ TEST_F(LexerTest, Symbols) {
auto buffer = Lex("<<<");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::LessLess()},
{TokenKind::Less()},
{TokenKind::EndOfFile()},
{TokenKind::LessLess},
{TokenKind::Less},
{TokenKind::EndOfFile},
}));
buffer = Lex("<<=>>");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::LessLessEqual()},
{TokenKind::GreaterGreater()},
{TokenKind::EndOfFile()},
{TokenKind::LessLessEqual},
{TokenKind::GreaterGreater},
{TokenKind::EndOfFile},
}));
buffer = Lex("< <=> >");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::Less()},
{TokenKind::LessEqualGreater()},
{TokenKind::Greater()},
{TokenKind::EndOfFile()},
{TokenKind::Less},
{TokenKind::LessEqualGreater},
{TokenKind::Greater},
{TokenKind::EndOfFile},
}));
buffer = Lex("\\/?@&^!");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::Backslash()},
{TokenKind::Slash()},
{TokenKind::Question()},
{TokenKind::At()},
{TokenKind::Amp()},
{TokenKind::Caret()},
{TokenKind::Exclaim()},
{TokenKind::EndOfFile()},
{TokenKind::Backslash},
{TokenKind::Slash},
{TokenKind::Question},
{TokenKind::At},
{TokenKind::Amp},
{TokenKind::Caret},
{TokenKind::Exclaim},
{TokenKind::EndOfFile},
}));
}
@@ -363,23 +344,23 @@ TEST_F(LexerTest, Parens) {
auto buffer = Lex("()");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::OpenParen()},
{TokenKind::CloseParen()},
{TokenKind::EndOfFile()},
{TokenKind::OpenParen},
{TokenKind::CloseParen},
{TokenKind::EndOfFile},
}));
buffer = Lex("((()()))");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::OpenParen()},
{TokenKind::OpenParen()},
{TokenKind::OpenParen()},
{TokenKind::CloseParen()},
{TokenKind::OpenParen()},
{TokenKind::CloseParen()},
{TokenKind::CloseParen()},
{TokenKind::CloseParen()},
{TokenKind::EndOfFile()},
{TokenKind::OpenParen},
{TokenKind::OpenParen},
{TokenKind::OpenParen},
{TokenKind::CloseParen},
{TokenKind::OpenParen},
{TokenKind::CloseParen},
{TokenKind::CloseParen},
{TokenKind::CloseParen},
{TokenKind::EndOfFile},
}));
}
@@ -387,23 +368,23 @@ TEST_F(LexerTest, CurlyBraces) {
auto buffer = Lex("{}");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::OpenCurlyBrace()},
{TokenKind::CloseCurlyBrace()},
{TokenKind::EndOfFile()},
{TokenKind::OpenCurlyBrace},
{TokenKind::CloseCurlyBrace},
{TokenKind::EndOfFile},
}));
buffer = Lex("{{{}{}}}");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::OpenCurlyBrace()},
{TokenKind::OpenCurlyBrace()},
{TokenKind::OpenCurlyBrace()},
{TokenKind::CloseCurlyBrace()},
{TokenKind::OpenCurlyBrace()},
{TokenKind::CloseCurlyBrace()},
{TokenKind::CloseCurlyBrace()},
{TokenKind::CloseCurlyBrace()},
{TokenKind::EndOfFile()},
{TokenKind::OpenCurlyBrace},
{TokenKind::OpenCurlyBrace},
{TokenKind::OpenCurlyBrace},
{TokenKind::CloseCurlyBrace},
{TokenKind::OpenCurlyBrace},
{TokenKind::CloseCurlyBrace},
{TokenKind::CloseCurlyBrace},
{TokenKind::CloseCurlyBrace},
{TokenKind::EndOfFile},
}));
}
@@ -425,7 +406,7 @@ TEST_F(LexerTest, MatchingGroups) {
EXPECT_EQ(open_curly_token,
buffer.GetMatchedOpeningToken(close_curly_token));
auto eof_token = *it++;
EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::EndOfFile());
EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::EndOfFile);
EXPECT_EQ(buffer.tokens().end(), it);
}
@@ -488,7 +469,7 @@ TEST_F(LexerTest, MatchingGroups) {
buffer.GetMatchedOpeningToken(inner_close_paren_token));
auto eof_token = *it++;
EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::EndOfFile());
EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::EndOfFile);
EXPECT_EQ(buffer.tokens().end(), it);
}
}
@@ -498,16 +479,16 @@ TEST_F(LexerTest, MismatchedGroups) {
EXPECT_TRUE(buffer.has_errors());
EXPECT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::OpenCurlyBrace()},
{.kind = TokenKind::CloseCurlyBrace(), .recovery = true},
{TokenKind::EndOfFile()},
{TokenKind::OpenCurlyBrace},
{.kind = TokenKind::CloseCurlyBrace, .recovery = true},
{TokenKind::EndOfFile},
}));
buffer = Lex("}");
EXPECT_TRUE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::Error(), .text = "}"},
{TokenKind::EndOfFile()},
{.kind = TokenKind::Error, .text = "}"},
{TokenKind::EndOfFile},
}));
buffer = Lex("{(}");
@@ -515,11 +496,11 @@ TEST_F(LexerTest, MismatchedGroups) {
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::OpenCurlyBrace(), .column = 1},
{.kind = TokenKind::OpenParen(), .column = 2},
{.kind = TokenKind::CloseParen(), .column = 3, .recovery = true},
{.kind = TokenKind::CloseCurlyBrace(), .column = 3},
{TokenKind::EndOfFile()},
{.kind = TokenKind::OpenCurlyBrace, .column = 1},
{.kind = TokenKind::OpenParen, .column = 2},
{.kind = TokenKind::CloseParen, .column = 3, .recovery = true},
{.kind = TokenKind::CloseCurlyBrace, .column = 3},
{TokenKind::EndOfFile},
}));
buffer = Lex(")({)");
@@ -527,12 +508,12 @@ TEST_F(LexerTest, MismatchedGroups) {
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::Error(), .column = 1, .text = ")"},
{.kind = TokenKind::OpenParen(), .column = 2},
{.kind = TokenKind::OpenCurlyBrace(), .column = 3},
{.kind = TokenKind::CloseCurlyBrace(), .column = 4, .recovery = true},
{.kind = TokenKind::CloseParen(), .column = 4},
{TokenKind::EndOfFile()},
{.kind = TokenKind::Error, .column = 1, .text = ")"},
{.kind = TokenKind::OpenParen, .column = 2},
{.kind = TokenKind::OpenCurlyBrace, .column = 3},
{.kind = TokenKind::CloseCurlyBrace, .column = 4, .recovery = true},
{.kind = TokenKind::CloseParen, .column = 4},
{TokenKind::EndOfFile},
}));
}
@@ -575,63 +556,55 @@ TEST_F(LexerTest, Keywords) {
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::Fn(), .column = 4, .indent_column = 4},
{TokenKind::EndOfFile()},
{.kind = TokenKind::Fn, .column = 4, .indent_column = 4},
{TokenKind::EndOfFile},
}));
buffer = Lex("and or not if else for return var break continue _");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{TokenKind::And()},
{TokenKind::Or()},
{TokenKind::Not()},
{TokenKind::If()},
{TokenKind::Else()},
{TokenKind::For()},
{TokenKind::Return()},
{TokenKind::Var()},
{TokenKind::Break()},
{TokenKind::Continue()},
{TokenKind::Underscore()},
{TokenKind::EndOfFile()},
{TokenKind::And},
{TokenKind::Or},
{TokenKind::Not},
{TokenKind::If},
{TokenKind::Else},
{TokenKind::For},
{TokenKind::Return},
{TokenKind::Var},
{TokenKind::Break},
{TokenKind::Continue},
{TokenKind::Underscore},
{TokenKind::EndOfFile},
}));
}
TEST_F(LexerTest, Comments) {
auto buffer = Lex(" ;\n // foo\n ;\n");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::Semi(),
.line = 1,
.column = 2,
.indent_column = 2},
{.kind = TokenKind::Semi(),
.line = 3,
.column = 3,
.indent_column = 3},
{.kind = TokenKind::EndOfFile(), .line = 3, .column = 4},
}));
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::Semi, .line = 1, .column = 2, .indent_column = 2},
{.kind = TokenKind::Semi, .line = 3, .column = 3, .indent_column = 3},
{.kind = TokenKind::EndOfFile, .line = 3, .column = 4},
}));
buffer = Lex("// foo\n//\n// bar");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{{TokenKind::EndOfFile()}}));
EXPECT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{{TokenKind::EndOfFile}}));
// Make sure weird characters aren't a problem.
buffer = Lex(" // foo#$!^?@-_💩🍫⃠ [̲̅$̲̅(̲̅ ͡° ͜ʖ ͡°̲̅)̲̅$̲̅]");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{{TokenKind::EndOfFile()}}));
EXPECT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{{TokenKind::EndOfFile}}));
// Make sure we can lex a comment at the end of the input.
buffer = Lex("//");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{{TokenKind::EndOfFile()}}));
EXPECT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{{TokenKind::EndOfFile}}));
}
TEST_F(LexerTest, InvalidComments) {
@@ -651,35 +624,34 @@ TEST_F(LexerTest, Identifiers) {
auto buffer = Lex(" foobar");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::Identifier(),
{.kind = TokenKind::Identifier,
.column = 4,
.indent_column = 4,
.text = "foobar"},
{TokenKind::EndOfFile()},
{TokenKind::EndOfFile},
}));
// Check different kinds of identifier character sequences.
buffer = Lex("_foo_bar");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::Identifier(), .text = "_foo_bar"},
{TokenKind::EndOfFile()},
{.kind = TokenKind::Identifier, .text = "_foo_bar"},
{TokenKind::EndOfFile},
}));
buffer = Lex("foo2bar00");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::Identifier(), .text = "foo2bar00"},
{TokenKind::EndOfFile()},
}));
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::Identifier, .text = "foo2bar00"},
{TokenKind::EndOfFile},
}));
// Check that we can parse identifiers that start with a keyword.
buffer = Lex("fnord");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::Identifier(), .text = "fnord"},
{TokenKind::EndOfFile()},
{.kind = TokenKind::Identifier, .text = "fnord"},
{TokenKind::EndOfFile},
}));
// Check multiple identifiers with indent and interning.
@@ -687,33 +659,33 @@ TEST_F(LexerTest, Identifiers) {
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::Identifier(),
{.kind = TokenKind::Identifier,
.line = 1,
.column = 4,
.indent_column = 4,
.text = "foo"},
{.kind = TokenKind::Semi()},
{.kind = TokenKind::Identifier(),
{.kind = TokenKind::Semi},
{.kind = TokenKind::Identifier,
.line = 1,
.column = 8,
.indent_column = 4,
.text = "bar"},
{.kind = TokenKind::Identifier(),
{.kind = TokenKind::Identifier,
.line = 2,
.column = 1,
.indent_column = 1,
.text = "bar"},
{.kind = TokenKind::Identifier(),
{.kind = TokenKind::Identifier,
.line = 3,
.column = 3,
.indent_column = 3,
.text = "foo"},
{.kind = TokenKind::Identifier(),
{.kind = TokenKind::Identifier,
.line = 3,
.column = 7,
.indent_column = 3,
.text = "foo"},
{.kind = TokenKind::EndOfFile(), .line = 3, .column = 10},
{.kind = TokenKind::EndOfFile, .line = 3, .column = 10},
}));
}
@@ -739,32 +711,32 @@ TEST_F(LexerTest, StringLiterals) {
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::StringLiteral(),
{.kind = TokenKind::StringLiteral,
.line = 2,
.column = 5,
.indent_column = 5,
.string_contents = {"hello world\n"}},
{.kind = TokenKind::StringLiteral(),
{.kind = TokenKind::StringLiteral,
.line = 4,
.column = 5,
.indent_column = 5,
.string_contents = {" test \xAB\n"}},
{.kind = TokenKind::Identifier(),
{.kind = TokenKind::Identifier,
.line = 7,
.column = 10,
.indent_column = 5,
.text = "trailing"},
{.kind = TokenKind::StringLiteral(),
{.kind = TokenKind::StringLiteral,
.line = 9,
.column = 7,
.indent_column = 7,
.string_contents = {"\""}},
{.kind = TokenKind::StringLiteral(),
{.kind = TokenKind::StringLiteral,
.line = 11,
.column = 5,
.indent_column = 5,
.string_contents = llvm::StringLiteral::withInnerNUL("\0")},
{.kind = TokenKind::StringLiteral(),
{.kind = TokenKind::StringLiteral,
.line = 13,
.column = 5,
.indent_column = 5,
@@ -772,22 +744,22 @@ TEST_F(LexerTest, StringLiterals) {
// """x""" is three string literals, not one invalid
// attempt at a block string literal.
{.kind = TokenKind::StringLiteral(),
{.kind = TokenKind::StringLiteral,
.line = 15,
.column = 5,
.indent_column = 5,
.string_contents = {""}},
{.kind = TokenKind::StringLiteral(),
{.kind = TokenKind::StringLiteral,
.line = 15,
.column = 7,
.indent_column = 5,
.string_contents = {"x"}},
{.kind = TokenKind::StringLiteral(),
{.kind = TokenKind::StringLiteral,
.line = 15,
.column = 10,
.indent_column = 5,
.string_contents = {""}},
{.kind = TokenKind::EndOfFile(), .line = 16, .column = 3},
{.kind = TokenKind::EndOfFile, .line = 16, .column = 3},
}));
}
@@ -818,7 +790,7 @@ TEST_F(LexerTest, InvalidStringLiterals) {
// We should have formed at least one error token.
bool found_error = false;
for (TokenizedBuffer::Token token : buffer.tokens()) {
if (buffer.GetKind(token) == TokenKind::Error()) {
if (buffer.GetKind(token) == TokenKind::Error) {
found_error = true;
break;
}
@@ -839,81 +811,81 @@ TEST_F(LexerTest, TypeLiterals) {
EXPECT_FALSE(buffer.has_errors());
ASSERT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::Identifier(),
{.kind = TokenKind::Identifier,
.line = 2,
.column = 5,
.indent_column = 5,
.text = {"i0"}},
{.kind = TokenKind::IntegerTypeLiteral(),
{.kind = TokenKind::IntegerTypeLiteral,
.line = 2,
.column = 8,
.indent_column = 5,
.text = {"i1"}},
{.kind = TokenKind::IntegerTypeLiteral(),
{.kind = TokenKind::IntegerTypeLiteral,
.line = 2,
.column = 11,
.indent_column = 5,
.text = {"i20"}},
{.kind = TokenKind::IntegerTypeLiteral(),
{.kind = TokenKind::IntegerTypeLiteral,
.line = 2,
.column = 15,
.indent_column = 5,
.text = {"i999999999999"}},
{.kind = TokenKind::Identifier(),
{.kind = TokenKind::Identifier,
.line = 2,
.column = 29,
.indent_column = 5,
.text = {"i0x1"}},
{.kind = TokenKind::Identifier(),
{.kind = TokenKind::Identifier,
.line = 3,
.column = 5,
.indent_column = 5,
.text = {"u0"}},
{.kind = TokenKind::UnsignedIntegerTypeLiteral(),
{.kind = TokenKind::UnsignedIntegerTypeLiteral,
.line = 3,
.column = 8,
.indent_column = 5,
.text = {"u1"}},
{.kind = TokenKind::UnsignedIntegerTypeLiteral(),
{.kind = TokenKind::UnsignedIntegerTypeLiteral,
.line = 3,
.column = 11,
.indent_column = 5,
.text = {"u64"}},
{.kind = TokenKind::Identifier(),
{.kind = TokenKind::Identifier,
.line = 3,
.column = 15,
.indent_column = 5,
.text = {"u64b"}},
{.kind = TokenKind::FloatingPointTypeLiteral(),
{.kind = TokenKind::FloatingPointTypeLiteral,
.line = 4,
.column = 5,
.indent_column = 5,
.text = {"f32"}},
{.kind = TokenKind::FloatingPointTypeLiteral(),
{.kind = TokenKind::FloatingPointTypeLiteral,
.line = 4,
.column = 9,
.indent_column = 5,
.text = {"f80"}},
{.kind = TokenKind::FloatingPointTypeLiteral(),
{.kind = TokenKind::FloatingPointTypeLiteral,
.line = 4,
.column = 13,
.indent_column = 5,
.text = {"f1"}},
{.kind = TokenKind::Identifier(),
{.kind = TokenKind::Identifier,
.line = 4,
.column = 16,
.indent_column = 5,
.text = {"fi"}},
{.kind = TokenKind::Identifier(),
{.kind = TokenKind::Identifier,
.line = 5,
.column = 5,
.indent_column = 5,
.text = {"s1"}},
{.kind = TokenKind::EndOfFile(), .line = 6, .column = 3},
{.kind = TokenKind::EndOfFile, .line = 6, .column = 3},
}));
auto token_i1 = buffer.tokens().begin() + 1;
@@ -949,12 +921,12 @@ TEST_F(LexerTest, TypeLiteralTooManyDigits) {
ASSERT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::Error(),
{.kind = TokenKind::Error,
.line = 1,
.column = 1,
.indent_column = 1,
.text = {code}},
{.kind = TokenKind::EndOfFile(), .line = 1, .column = Count + 2},
{.kind = TokenKind::EndOfFile, .line = 1, .column = Count + 2},
}));
}
toolchain/lexer/tokenized_buffer_test_helpers.h
+2 -2
View File
@@ -132,8 +132,8 @@ MATCHER_P(HasTokens, raw_all_expected, "") {
}
CARBON_CHECK(!expected.string_contents ||
expected.kind == TokenKind::StringLiteral());
if (expected.string_contents && actual_kind == TokenKind::StringLiteral()) {
expected.kind == TokenKind::StringLiteral);
if (expected.string_contents && actual_kind == TokenKind::StringLiteral) {
llvm::StringRef actual_contents = buffer.GetStringLiteral(token);
if (actual_contents != *expected.string_contents) {
*result_listener << "\nToken " << index << " has contents `"
toolchain/parser/parse_node_kind.h
+1 -1
View File
@@ -9,7 +9,7 @@
namespace Carbon {
CARBON_DEFINE_RAW_ENUM_CLASS(ParseNodeKind, uint8_t){
CARBON_DEFINE_RAW_ENUM_CLASS(ParseNodeKind, uint8_t) {
#define CARBON_PARSE_NODE_KIND(Name) CARBON_RAW_ENUM_ENUMERATOR(Name)
#include "toolchain/parser/parse_node_kind.def"
};
toolchain/parser/parser.cpp
+107 -111
View File
@@ -95,7 +95,7 @@ Parser::Parser(ParseTree& tree, TokenizedBuffer& tokens,
stack_context_(ParseContext::File) {
CARBON_CHECK(position_ != end_) << "Empty TokenizedBuffer";
--end_;
CARBON_CHECK(tokens_->GetKind(*end_) == TokenKind::EndOfFile())
CARBON_CHECK(tokens_->GetKind(*end_) == TokenKind::EndOfFile)
<< "TokenizedBuffer should end with EndOfFile, ended with "
<< tokens_->GetKind(*end_).name();
}
@@ -121,7 +121,7 @@ auto Parser::AddNode(ParseNodeKind kind, TokenizedBuffer::Token token,
auto Parser::ConsumeAndAddOpenParen(TokenizedBuffer::Token default_token,
ParseNodeKind start_kind) -> void {
if (auto open_paren = ConsumeIf(TokenKind::OpenParen())) {
if (auto open_paren = ConsumeIf(TokenKind::OpenParen)) {
AddLeafNode(start_kind, *open_paren, /*has_error=*/false);
} else {
emitter_->Emit(*position_, ExpectedParenAfter,
@@ -136,9 +136,9 @@ auto Parser::ConsumeAndAddCloseParen(StateStackEntry state,
// introducer.
auto expected_paren = *(TokenizedBuffer::TokenIterator(state.token) + 1);
if (tokens_->GetKind(expected_paren) != TokenKind::OpenParen()) {
if (tokens_->GetKind(expected_paren) != TokenKind::OpenParen) {
AddNode(close_kind, state.token, state.subtree_start, /*has_error=*/true);
} else if (auto close_token = ConsumeIf(TokenKind::CloseParen())) {
} else if (auto close_token = ConsumeIf(TokenKind::CloseParen)) {
AddNode(close_kind, *close_token, state.subtree_start, state.has_error);
} else {
// TODO: Include the location of the matching open_paren in the diagnostic.
@@ -187,7 +187,7 @@ auto Parser::FindNextOf(std::initializer_list<TokenKind> desired_kinds)
}
// Step to the next token at the current bracketing level.
if (kind.is_closing_symbol() || kind == TokenKind::EndOfFile()) {
if (kind.is_closing_symbol() || kind == TokenKind::EndOfFile) {
// There are no more tokens at this level.
return std::nullopt;
} else if (kind.is_opening_symbol()) {
@@ -233,7 +233,7 @@ auto Parser::SkipPastLikelyEnd(TokenizedBuffer::Token skip_root)
};
do {
if (PositionIs(TokenKind::CloseCurlyBrace())) {
if (PositionIs(TokenKind::CloseCurlyBrace)) {
// Immediately bail out if we hit an unmatched close curly, this will
// pop us up a level of the syntax grouping.
return std::nullopt;
@@ -241,7 +241,7 @@ auto Parser::SkipPastLikelyEnd(TokenizedBuffer::Token skip_root)
// We assume that a semicolon is always intended to be the end of the
// current construct.
if (auto semi = ConsumeIf(TokenKind::Semi())) {
if (auto semi = ConsumeIf(TokenKind::Semi)) {
return semi;
}
@@ -269,7 +269,7 @@ auto Parser::HandleCodeBlockState() -> void {
PopAndDiscardState();
PushState(ParserState::CodeBlockFinish);
if (ConsumeAndAddLeafNodeIf(TokenKind::OpenCurlyBrace(),
if (ConsumeAndAddLeafNodeIf(TokenKind::OpenCurlyBrace,
ParseNodeKind::CodeBlockStart)) {
PushState(ParserState::StatementScopeLoop);
} else {
@@ -287,27 +287,27 @@ auto Parser::HandleCodeBlockState() -> void {
// Determines whether the given token is considered to be the start of an
// operand according to the rules for infix operator parsing.
static auto IsAssumedStartOfOperand(TokenKind kind) -> bool {
return kind.IsOneOf({TokenKind::OpenParen(), TokenKind::Identifier(),
TokenKind::IntegerLiteral(), TokenKind::RealLiteral(),
TokenKind::StringLiteral()});
return kind.IsOneOf({TokenKind::OpenParen, TokenKind::Identifier,
TokenKind::IntegerLiteral, TokenKind::RealLiteral,
TokenKind::StringLiteral});
}
// Determines whether the given token is considered to be the end of an
// operand according to the rules for infix operator parsing.
static auto IsAssumedEndOfOperand(TokenKind kind) -> bool {
return kind.IsOneOf({TokenKind::CloseParen(), TokenKind::CloseCurlyBrace(),
TokenKind::CloseSquareBracket(), TokenKind::Identifier(),
TokenKind::IntegerLiteral(), TokenKind::RealLiteral(),
TokenKind::StringLiteral()});
return kind.IsOneOf({TokenKind::CloseParen, TokenKind::CloseCurlyBrace,
TokenKind::CloseSquareBracket, TokenKind::Identifier,
TokenKind::IntegerLiteral, TokenKind::RealLiteral,
TokenKind::StringLiteral});
}
// Determines whether the given token could possibly be the start of an
// operand. This is conservatively correct, and will never incorrectly return
// `false`, but can incorrectly return `true`.
static auto IsPossibleStartOfOperand(TokenKind kind) -> bool {
return !kind.IsOneOf({TokenKind::CloseParen(), TokenKind::CloseCurlyBrace(),
TokenKind::CloseSquareBracket(), TokenKind::Comma(),
TokenKind::Semi(), TokenKind::Colon()});
return !kind.IsOneOf({TokenKind::CloseParen, TokenKind::CloseCurlyBrace,
TokenKind::CloseSquareBracket, TokenKind::Comma,
TokenKind::Semi, TokenKind::Colon});
}
auto Parser::IsLexicallyValidInfixOperator() -> bool {
@@ -405,7 +405,7 @@ auto Parser::DiagnoseOperatorFixity(OperatorFixity fixity) -> void {
auto Parser::ConsumeListToken(ParseNodeKind comma_kind, TokenKind close_kind,
bool already_has_error) -> ListTokenKind {
if (!PositionIs(TokenKind::Comma()) && !PositionIs(close_kind)) {
if (!PositionIs(TokenKind::Comma) && !PositionIs(close_kind)) {
// Don't error a second time on the same element.
if (!already_has_error) {
CARBON_DIAGNOSTIC(UnexpectedTokenAfterListElement, Error,
@@ -415,7 +415,7 @@ auto Parser::ConsumeListToken(ParseNodeKind comma_kind, TokenKind close_kind,
}
// Recover from the invalid token.
auto end_of_element = FindNextOf({TokenKind::Comma(), close_kind});
auto end_of_element = FindNextOf({TokenKind::Comma, close_kind});
// The lexer guarantees that parentheses are balanced.
CARBON_CHECK(end_of_element)
<< "missing matching `" << close_kind.opening_symbol() << "` for `"
@@ -458,9 +458,9 @@ auto Parser::HandleBraceExpressionState() -> void {
PushState(state);
CARBON_CHECK(ConsumeAndAddLeafNodeIf(
TokenKind::OpenCurlyBrace(),
TokenKind::OpenCurlyBrace,
ParseNodeKind::StructLiteralOrStructTypeLiteralStart));
if (!PositionIs(TokenKind::CloseCurlyBrace())) {
if (!PositionIs(TokenKind::CloseCurlyBrace)) {
PushState(ParserState::BraceExpressionParameterAsUnknown);
}
}
@@ -506,7 +506,7 @@ auto Parser::HandleBraceExpressionParameterError(StateStackEntry state,
auto Parser::HandleBraceExpressionParameter(BraceExpressionKind kind) -> void {
auto state = PopState();
if (!PositionIs(TokenKind::Period())) {
if (!PositionIs(TokenKind::Period)) {
HandleBraceExpressionParameterError(state, kind);
return;
}
@@ -536,10 +536,9 @@ auto Parser::HandleBraceExpressionParameterAfterDesignator(
auto state = PopState();
if (state.has_error) {
auto recovery_pos = FindNextOf(
{TokenKind::Equal(), TokenKind::Colon(), TokenKind::Comma()});
if (!recovery_pos ||
tokens_->GetKind(*recovery_pos) == TokenKind::Comma()) {
auto recovery_pos =
FindNextOf({TokenKind::Equal, TokenKind::Colon, TokenKind::Comma});
if (!recovery_pos || tokens_->GetKind(*recovery_pos) == TokenKind::Comma) {
state.state = BraceExpressionKindToParserState(
kind, ParserState::BraceExpressionParameterFinishAsType,
ParserState::BraceExpressionParameterFinishAsValue,
@@ -552,9 +551,9 @@ auto Parser::HandleBraceExpressionParameterAfterDesignator(
// Work out the kind of this element.
auto elem_kind = BraceExpressionKind::Unknown;
if (PositionIs(TokenKind::Colon())) {
if (PositionIs(TokenKind::Colon)) {
elem_kind = BraceExpressionKind::Type;
} else if (PositionIs(TokenKind::Equal())) {
} else if (PositionIs(TokenKind::Equal)) {
elem_kind = BraceExpressionKind::Value;
}
// Unknown kinds and changes between type and value are errors.
@@ -618,7 +617,7 @@ auto Parser::HandleBraceExpressionParameterFinish(BraceExpressionKind kind)
state.token, state.subtree_start, /*has_error=*/false);
}
if (ConsumeListToken(ParseNodeKind::StructComma, TokenKind::CloseCurlyBrace(),
if (ConsumeListToken(ParseNodeKind::StructComma, TokenKind::CloseCurlyBrace,
state.has_error) == ListTokenKind::Comma) {
PushState(BraceExpressionKindToParserState(
kind, ParserState::BraceExpressionParameterAsType,
@@ -667,7 +666,7 @@ auto Parser::HandleCallExpressionState() -> void {
AddNode(ParseNodeKind::CallExpressionStart, Consume(), state.subtree_start,
state.has_error);
if (!PositionIs(TokenKind::CloseParen())) {
if (!PositionIs(TokenKind::CloseParen)) {
PushState(ParserState::CallExpressionParameterFinish);
PushState(ParserState::Expression);
}
@@ -681,7 +680,7 @@ auto Parser::HandleCallExpressionParameterFinishState() -> void {
}
if (ConsumeListToken(ParseNodeKind::CallExpressionComma,
TokenKind::CloseParen(),
TokenKind::CloseParen,
state.has_error) == ListTokenKind::Comma) {
PushState(ParserState::CallExpressionParameterFinish);
PushState(ParserState::Expression);
@@ -699,7 +698,7 @@ auto Parser::HandleCodeBlockFinishState() -> void {
auto state = PopState();
// If the block started with an open curly, this is a close curly.
if (tokens_->GetKind(state.token) == TokenKind::OpenCurlyBrace()) {
if (tokens_->GetKind(state.token) == TokenKind::OpenCurlyBrace) {
AddNode(ParseNodeKind::CodeBlock, Consume(), state.subtree_start,
state.has_error);
} else {
@@ -712,28 +711,28 @@ auto Parser::HandleDeclarationLoopState() -> void {
// This maintains the current state unless we're at the end of the file.
switch (PositionKind()) {
case TokenKind::EndOfFile(): {
case TokenKind::EndOfFile: {
PopAndDiscardState();
break;
}
case TokenKind::Fn(): {
case TokenKind::Fn: {
PushState(ParserState::FunctionIntroducer);
AddLeafNode(ParseNodeKind::FunctionIntroducer, Consume());
break;
}
case TokenKind::Package(): {
case TokenKind::Package: {
PushState(ParserState::Package);
break;
}
case TokenKind::Semi(): {
case TokenKind::Semi: {
AddLeafNode(ParseNodeKind::EmptyDeclaration, Consume());
break;
}
case TokenKind::Var(): {
case TokenKind::Var: {
PushState(ParserState::VarAsSemicolon);
break;
}
case TokenKind::Interface(): {
case TokenKind::Interface: {
PushState(ParserState::InterfaceIntroducer);
++position_;
break;
@@ -754,7 +753,7 @@ auto Parser::HandleDeclarationLoopState() -> void {
auto Parser::HandleDeducedParameterListFinishState() -> void {
auto state = PopState();
CARBON_CHECK(tokens_->GetKind(*position_) == TokenKind::CloseSquareBracket())
CARBON_CHECK(tokens_->GetKind(*position_) == TokenKind::CloseSquareBracket)
<< "Expected current token to be: `]`, found: "
<< tokens_->GetKind(state.token);
AddNode(ParseNodeKind::DeducedParameterList, Consume(), state.subtree_start,
@@ -765,9 +764,9 @@ auto Parser::HandleDesignator(bool as_struct) -> void {
auto state = PopState();
// `.` identifier
auto dot = ConsumeChecked(TokenKind::Period());
auto dot = ConsumeChecked(TokenKind::Period);
if (!ConsumeAndAddLeafNodeIf(TokenKind::Identifier(),
if (!ConsumeAndAddLeafNodeIf(TokenKind::Identifier,
ParseNodeKind::DesignatedName)) {
CARBON_DIAGNOSTIC(ExpectedIdentifierAfterDot, Error,
"Expected identifier after `.`.");
@@ -837,32 +836,32 @@ auto Parser::HandleExpressionInPostfixState() -> void {
// expression tree, such as an identifier or literal, or a parenthesized
// expression.
switch (PositionKind()) {
case TokenKind::Identifier(): {
case TokenKind::Identifier: {
AddLeafNode(ParseNodeKind::NameReference, Consume());
PushState(state);
break;
}
case TokenKind::IntegerLiteral():
case TokenKind::RealLiteral():
case TokenKind::StringLiteral():
case TokenKind::IntegerTypeLiteral():
case TokenKind::UnsignedIntegerTypeLiteral():
case TokenKind::FloatingPointTypeLiteral(): {
case TokenKind::IntegerLiteral:
case TokenKind::RealLiteral:
case TokenKind::StringLiteral:
case TokenKind::IntegerTypeLiteral:
case TokenKind::UnsignedIntegerTypeLiteral:
case TokenKind::FloatingPointTypeLiteral: {
AddLeafNode(ParseNodeKind::Literal, Consume());
PushState(state);
break;
}
case TokenKind::OpenCurlyBrace(): {
case TokenKind::OpenCurlyBrace: {
PushState(state);
PushState(ParserState::BraceExpression);
break;
}
case TokenKind::OpenParen(): {
case TokenKind::OpenParen: {
PushState(state);
PushState(ParserState::ParenExpression);
break;
}
case TokenKind::SelfType(): {
case TokenKind::SelfType: {
AddLeafNode(ParseNodeKind::SelfType, Consume());
PushState(state);
break;
@@ -884,13 +883,13 @@ auto Parser::HandleExpressionInPostfixLoopState() -> void {
state.token = *position_;
switch (PositionKind()) {
case TokenKind::Period(): {
case TokenKind::Period: {
PushState(state);
state.state = ParserState::DesignatorAsExpression;
PushState(state);
break;
}
case TokenKind::OpenParen(): {
case TokenKind::OpenParen: {
PushState(state);
state.state = ParserState::CallExpression;
PushState(state);
@@ -982,7 +981,7 @@ auto Parser::HandleExpressionLoopForPrefixState() -> void {
auto Parser::HandleExpressionStatementFinishState() -> void {
auto state = PopState();
if (auto semi = ConsumeIf(TokenKind::Semi())) {
if (auto semi = ConsumeIf(TokenKind::Semi)) {
AddNode(ParseNodeKind::ExpressionStatement, *semi, state.subtree_start,
state.has_error);
return;
@@ -1018,7 +1017,7 @@ auto Parser::HandleFunctionError(StateStackEntry state,
auto Parser::HandleFunctionIntroducerState() -> void {
auto state = PopState();
if (!ConsumeAndAddLeafNodeIf(TokenKind::Identifier(),
if (!ConsumeAndAddLeafNodeIf(TokenKind::Identifier,
ParseNodeKind::DeclaredName)) {
CARBON_DIAGNOSTIC(ExpectedFunctionName, Error,
"Expected function name after `fn` keyword.");
@@ -1034,12 +1033,12 @@ auto Parser::HandleFunctionIntroducerState() -> void {
PushState(state);
// If there are deduced params handle them next.
if (PositionIs(TokenKind::OpenSquareBracket())) {
if (PositionIs(TokenKind::OpenSquareBracket)) {
PushState(ParserState::DeducedParameterListFinish);
// This is for sure a `[`, we can safely create the corresponding node.
AddLeafNode(ParseNodeKind::DeducedParameterListStart, Consume());
if (PositionIs(TokenKind::CloseSquareBracket())) {
if (PositionIs(TokenKind::CloseSquareBracket)) {
return;
}
@@ -1055,7 +1054,7 @@ auto Parser::HandleFunctionIntroducerState() -> void {
auto Parser::HandleFunctionAfterDeducedParameterListState() -> void {
auto state = PopState();
if (!PositionIs(TokenKind::OpenParen())) {
if (!PositionIs(TokenKind::OpenParen)) {
CARBON_DIAGNOSTIC(ExpectedFunctionParams, Error,
"Expected `(` after function name.");
emitter_->Emit(*position_, ExpectedFunctionParams);
@@ -1070,7 +1069,7 @@ auto Parser::HandleFunctionAfterDeducedParameterListState() -> void {
PushState(ParserState::FunctionParameterListFinish);
AddLeafNode(ParseNodeKind::ParameterListStart, Consume());
if (!PositionIs(TokenKind::CloseParen())) {
if (!PositionIs(TokenKind::CloseParen)) {
PushState(ParserState::FunctionParameter);
}
}
@@ -1089,8 +1088,7 @@ auto Parser::HandleFunctionParameterFinishState() -> void {
ReturnErrorOnState();
}
if (ConsumeListToken(ParseNodeKind::ParameterListComma,
TokenKind::CloseParen(),
if (ConsumeListToken(ParseNodeKind::ParameterListComma, TokenKind::CloseParen,
state.has_error) == ListTokenKind::Comma) {
PushState(ParserState::PatternAsFunctionParameter);
}
@@ -1099,7 +1097,7 @@ auto Parser::HandleFunctionParameterFinishState() -> void {
auto Parser::HandleFunctionParameterListFinishState() -> void {
auto state = PopState();
AddNode(ParseNodeKind::ParameterList, ConsumeChecked(TokenKind::CloseParen()),
AddNode(ParseNodeKind::ParameterList, ConsumeChecked(TokenKind::CloseParen),
state.subtree_start, state.has_error);
}
@@ -1112,7 +1110,7 @@ auto Parser::HandleFunctionAfterParameterListState() -> void {
// If there is a return type, parse the expression before adding the return
// type nod.e
if (PositionIs(TokenKind::MinusGreater())) {
if (PositionIs(TokenKind::MinusGreater)) {
PushState(ParserState::FunctionReturnTypeFinish);
++position_;
PushStateForExpression(PrecedenceGroup::ForType());
@@ -1130,12 +1128,12 @@ auto Parser::HandleFunctionSignatureFinishState() -> void {
auto state = PopState();
switch (PositionKind()) {
case TokenKind::Semi(): {
case TokenKind::Semi: {
AddNode(ParseNodeKind::FunctionDeclaration, Consume(),
state.subtree_start, state.has_error);
break;
}
case TokenKind::OpenCurlyBrace(): {
case TokenKind::OpenCurlyBrace: {
if (stack_context_ == ParseContext::Interface) {
CARBON_DIAGNOSTIC(
MethodImplNotAllowed, Error,
@@ -1180,7 +1178,7 @@ auto Parser::HandleInterfaceIntroducerState() -> void {
<< "TODO: Support nesting.";
stack_context_ = ParseContext::Interface;
if (!ConsumeAndAddLeafNodeIf(TokenKind::Identifier(),
if (!ConsumeAndAddLeafNodeIf(TokenKind::Identifier,
ParseNodeKind::DeclaredName)) {
CARBON_DIAGNOSTIC(ExpectedInterfaceName, Error,
"Expected interface name after `interface` keyword.");
@@ -1196,7 +1194,7 @@ auto Parser::HandleInterfaceIntroducerState() -> void {
bool parse_body = true;
if (!PositionIs(TokenKind::OpenCurlyBrace())) {
if (!PositionIs(TokenKind::OpenCurlyBrace)) {
CARBON_DIAGNOSTIC(ExpectedInterfaceOpenCurlyBrace, Error,
"Expected `{{` to start interface definition.");
emitter_->Emit(*position_, ExpectedInterfaceOpenCurlyBrace);
@@ -1220,7 +1218,7 @@ auto Parser::HandleInterfaceDefinitionLoopState() -> void {
// definition.
switch (PositionKind()) {
case TokenKind::CloseCurlyBrace(): {
case TokenKind::CloseCurlyBrace: {
auto state = PopState();
AddNode(ParseNodeKind::InterfaceBody, Consume(), state.subtree_start,
@@ -1228,7 +1226,7 @@ auto Parser::HandleInterfaceDefinitionLoopState() -> void {
break;
}
case TokenKind::Fn(): {
case TokenKind::Fn: {
PushState(ParserState::FunctionIntroducer);
AddLeafNode(ParseNodeKind::FunctionIntroducer, Consume());
break;
@@ -1265,7 +1263,7 @@ auto Parser::HandlePackageState() -> void {
/*has_error=*/true);
};
if (!ConsumeAndAddLeafNodeIf(TokenKind::Identifier(),
if (!ConsumeAndAddLeafNodeIf(TokenKind::Identifier,
ParseNodeKind::DeclaredName)) {
CARBON_DIAGNOSTIC(ExpectedIdentifierAfterPackage, Error,
"Expected identifier after `package`.");
@@ -1275,10 +1273,10 @@ auto Parser::HandlePackageState() -> void {
}
bool library_parsed = false;
if (auto library_token = ConsumeIf(TokenKind::Library())) {
if (auto library_token = ConsumeIf(TokenKind::Library)) {
auto library_start = tree_->size();
if (!ConsumeAndAddLeafNodeIf(TokenKind::StringLiteral(),
if (!ConsumeAndAddLeafNodeIf(TokenKind::StringLiteral,
ParseNodeKind::Literal)) {
CARBON_DIAGNOSTIC(
ExpectedLibraryName, Error,
@@ -1294,16 +1292,16 @@ auto Parser::HandlePackageState() -> void {
}
switch (auto api_or_impl_token = tokens_->GetKind(*(position_))) {
case TokenKind::Api(): {
case TokenKind::Api: {
AddLeafNode(ParseNodeKind::PackageApi, Consume());
break;
}
case TokenKind::Impl(): {
case TokenKind::Impl: {
AddLeafNode(ParseNodeKind::PackageImpl, Consume());
break;
}
default: {
if (!library_parsed && api_or_impl_token == TokenKind::StringLiteral()) {
if (!library_parsed && api_or_impl_token == TokenKind::StringLiteral) {
// If we come acroess a string literal and we didn't parse `library
// "..."` yet, then most probably the user forgot to add `library`
// before the library name.
@@ -1320,7 +1318,7 @@ auto Parser::HandlePackageState() -> void {
}
}
if (!PositionIs(TokenKind::Semi())) {
if (!PositionIs(TokenKind::Semi)) {
CARBON_DIAGNOSTIC(ExpectedSemiToEndPackageDirective, Error,
"Expected `;` to end package directive.");
emitter_->Emit(*position_, ExpectedSemiToEndPackageDirective);
@@ -1370,9 +1368,9 @@ auto Parser::HandleParenExpressionState() -> void {
// Advance past the open paren.
AddLeafNode(ParseNodeKind::ParenExpressionOrTupleLiteralStart,
ConsumeChecked(TokenKind::OpenParen()));
ConsumeChecked(TokenKind::OpenParen));
if (PositionIs(TokenKind::CloseParen())) {
if (PositionIs(TokenKind::CloseParen)) {
state.state = ParserState::ParenExpressionFinishAsTuple;
PushState(state);
} else {
@@ -1386,9 +1384,8 @@ auto Parser::HandleParenExpressionState() -> void {
auto Parser::HandleParenExpressionParameterFinish(bool as_tuple) -> void {
auto state = PopState();
auto list_token_kind =
ConsumeListToken(ParseNodeKind::TupleLiteralComma,
TokenKind::CloseParen(), state.has_error);
auto list_token_kind = ConsumeListToken(
ParseNodeKind::TupleLiteralComma, TokenKind::CloseParen, state.has_error);
if (list_token_kind == ListTokenKind::Close) {
return;
}
@@ -1442,8 +1439,8 @@ auto Parser::HandlePattern(PatternKind pattern_kind) -> void {
state.state = ParserState::PatternFinish;
// Handle an invalid pattern introducer for parameters and variables.
if (!PositionIs(TokenKind::Identifier()) ||
tokens_->GetKind(*(position_ + 1)) != TokenKind::Colon()) {
if (!PositionIs(TokenKind::Identifier) ||
tokens_->GetKind(*(position_ + 1)) != TokenKind::Colon) {
switch (pattern_kind) {
case PatternKind::Parameter: {
CARBON_DIAGNOSTIC(ExpectedParameterName, Error,
@@ -1514,8 +1511,8 @@ auto Parser::HandleSelfPatternState() -> void {
// self `:` type
auto possible_self_param =
(PositionIs(TokenKind::SelfParameter()) &&
tokens_->GetKind(*(position_ + 1)) == TokenKind::Colon());
(PositionIs(TokenKind::SelfParameter) &&
tokens_->GetKind(*(position_ + 1)) == TokenKind::Colon);
if (possible_self_param) {
// Ensure the finish state always follows.
@@ -1533,9 +1530,9 @@ auto Parser::HandleSelfPatternState() -> void {
// addr self `:` type
auto possible_addr_self_param =
(PositionIs(TokenKind::Addr()) &&
tokens_->GetKind(*(position_ + 1)) == TokenKind::SelfParameter() &&
tokens_->GetKind(*(position_ + 2)) == TokenKind::Colon());
(PositionIs(TokenKind::Addr) &&
tokens_->GetKind(*(position_ + 1)) == TokenKind::SelfParameter &&
tokens_->GetKind(*(position_ + 2)) == TokenKind::Colon);
if (possible_addr_self_param) {
// Ensure the finish state always follows.
@@ -1560,7 +1557,7 @@ auto Parser::HandleSelfPatternState() -> void {
// Try to recover by skipping to the next `]`.
if (auto next_close_square_bracket =
FindNextOf({TokenKind::CloseSquareBracket()});
FindNextOf({TokenKind::CloseSquareBracket});
next_close_square_bracket) {
SkipTo(*next_close_square_bracket);
}
@@ -1572,35 +1569,35 @@ auto Parser::HandleStatementState() -> void {
PopAndDiscardState();
switch (PositionKind()) {
case TokenKind::Break(): {
case TokenKind::Break: {
PushState(ParserState::StatementBreakFinish);
AddLeafNode(ParseNodeKind::BreakStatementStart, Consume());
break;
}
case TokenKind::Continue(): {
case TokenKind::Continue: {
PushState(ParserState::StatementContinueFinish);
AddLeafNode(ParseNodeKind::ContinueStatementStart, Consume());
break;
}
case TokenKind::For(): {
case TokenKind::For: {
PushState(ParserState::StatementForFinish);
PushState(ParserState::StatementForHeader);
++position_;
break;
}
case TokenKind::If(): {
case TokenKind::If: {
PushState(ParserState::StatementIf);
break;
}
case TokenKind::Return(): {
case TokenKind::Return: {
PushState(ParserState::StatementReturn);
break;
}
case TokenKind::Var(): {
case TokenKind::Var: {
PushState(ParserState::VarAsSemicolon);
break;
}
case TokenKind::While(): {
case TokenKind::While: {
PushState(ParserState::StatementWhile);
break;
}
@@ -1627,7 +1624,7 @@ auto Parser::HandleStatementForHeaderState() -> void {
state.state = ParserState::StatementForHeaderIn;
if (PositionIs(TokenKind::Var())) {
if (PositionIs(TokenKind::Var)) {
PushState(state);
PushState(ParserState::VarAsFor);
} else {
@@ -1635,7 +1632,7 @@ auto Parser::HandleStatementForHeaderState() -> void {
"Expected `var` declaration.");
emitter_->Emit(*position_, ExpectedVariableDeclaration);
if (auto next_in = FindNextOf({TokenKind::In()})) {
if (auto next_in = FindNextOf({TokenKind::In})) {
SkipTo(*next_in);
++position_;
}
@@ -1686,13 +1683,13 @@ auto Parser::HandleStatementIfConditionFinishState() -> void {
auto Parser::HandleStatementIfThenBlockFinishState() -> void {
auto state = PopState();
if (ConsumeAndAddLeafNodeIf(TokenKind::Else(),
if (ConsumeAndAddLeafNodeIf(TokenKind::Else,
ParseNodeKind::IfStatementElse)) {
state.state = ParserState::StatementIfElseBlockFinish;
PushState(state);
// `else if` is permitted as a special case.
PushState(PositionIs(TokenKind::If()) ? ParserState::StatementIf
: ParserState::CodeBlock);
PushState(PositionIs(TokenKind::If) ? ParserState::StatementIf
: ParserState::CodeBlock);
} else {
AddNode(ParseNodeKind::IfStatement, state.token, state.subtree_start,
state.has_error);
@@ -1708,7 +1705,7 @@ auto Parser::HandleStatementIfElseBlockFinishState() -> void {
auto Parser::HandleStatementKeywordFinish(ParseNodeKind node_kind) -> void {
auto state = PopState();
auto semi = ConsumeIf(TokenKind::Semi());
auto semi = ConsumeIf(TokenKind::Semi);
if (!semi) {
CARBON_DIAGNOSTIC(ExpectedSemiAfter, Error, "Expected `;` after `{0}`.",
TokenKind);
@@ -1731,7 +1728,7 @@ auto Parser::HandleStatementReturnState() -> void {
PushState(state);
AddLeafNode(ParseNodeKind::ReturnStatementStart, Consume());
if (!PositionIs(TokenKind::Semi())) {
if (!PositionIs(TokenKind::Semi)) {
PushState(ParserState::Expression);
}
}
@@ -1744,7 +1741,7 @@ auto Parser::HandleStatementScopeLoopState() -> void {
// This maintains the current state until we're at the end of the scope.
auto token_kind = PositionKind();
if (token_kind == TokenKind::CloseCurlyBrace()) {
if (token_kind == TokenKind::CloseCurlyBrace) {
auto state = PopState();
if (state.has_error) {
ReturnErrorOnState();
@@ -1802,13 +1799,12 @@ auto Parser::HandleVarAfterPatternState() -> void {
auto state = PopState();
if (state.has_error) {
if (auto after_pattern =
FindNextOf({TokenKind::Equal(), TokenKind::Semi()})) {
if (auto after_pattern = FindNextOf({TokenKind::Equal, TokenKind::Semi})) {
SkipTo(*after_pattern);
}
}
if (auto equals = ConsumeIf(TokenKind::Equal())) {
if (auto equals = ConsumeIf(TokenKind::Equal)) {
AddLeafNode(ParseNodeKind::VariableInitializer, *equals);
PushState(ParserState::Expression);
}
@@ -1818,7 +1814,7 @@ auto Parser::HandleVarFinishAsSemicolonState() -> void {
auto state = PopState();
auto end_token = state.token;
if (PositionIs(TokenKind::Semi())) {
if (PositionIs(TokenKind::Semi)) {
end_token = Consume();
} else {
emitter_->Emit(*position_, ExpectedSemiAfterExpression);
@@ -1835,9 +1831,9 @@ auto Parser::HandleVarFinishAsForState() -> void {
auto state = PopState();
auto end_token = state.token;
if (PositionIs(TokenKind::In())) {
if (PositionIs(TokenKind::In)) {
end_token = Consume();
} else if (PositionIs(TokenKind::Colon())) {
} else if (PositionIs(TokenKind::Colon)) {
CARBON_DIAGNOSTIC(ExpectedInNotColon, Error,
"`:` should be replaced by `in`.");
emitter_->Emit(*position_, ExpectedInNotColon);
toolchain/parser/precedence.cpp
+55 -55
View File
@@ -196,18 +196,18 @@ auto PrecedenceGroup::ForType() -> PrecedenceGroup {
auto PrecedenceGroup::ForLeading(TokenKind kind)
-> std::optional<PrecedenceGroup> {
switch (kind) {
case TokenKind::Star():
case TokenKind::Star:
return PrecedenceGroup(TermPrefix);
case TokenKind::Not():
case TokenKind::Not:
return PrecedenceGroup(LogicalPrefix);
case TokenKind::Minus():
case TokenKind::MinusMinus():
case TokenKind::PlusPlus():
case TokenKind::Minus:
case TokenKind::MinusMinus:
case TokenKind::PlusPlus:
return PrecedenceGroup(NumericPrefix);
case TokenKind::Tilde():
case TokenKind::Tilde:
return PrecedenceGroup(BitwisePrefix);
default:
@@ -219,92 +219,92 @@ auto PrecedenceGroup::ForTrailing(TokenKind kind, bool infix)
-> std::optional<Trailing> {
switch (kind) {
// Assignment operators.
case TokenKind::Equal():
case TokenKind::Equal:
return Trailing{.level = SimpleAssignment, .is_binary = true};
case TokenKind::PlusEqual():
case TokenKind::MinusEqual():
case TokenKind::StarEqual():
case TokenKind::SlashEqual():
case TokenKind::PercentEqual():
case TokenKind::AmpEqual():
case TokenKind::PipeEqual():
case TokenKind::GreaterGreaterEqual():
case TokenKind::LessLessEqual():
case TokenKind::PlusEqual:
case TokenKind::MinusEqual:
case TokenKind::StarEqual:
case TokenKind::SlashEqual:
case TokenKind::PercentEqual:
case TokenKind::AmpEqual:
case TokenKind::PipeEqual:
case TokenKind::GreaterGreaterEqual:
case TokenKind::LessLessEqual:
return Trailing{.level = CompoundAssignment, .is_binary = true};
// Logical operators.
case TokenKind::And():
case TokenKind::And:
return Trailing{.level = LogicalAnd, .is_binary = true};
case TokenKind::Or():
case TokenKind::Or:
return Trailing{.level = LogicalOr, .is_binary = true};
// Bitwise operators.
case TokenKind::Amp():
case TokenKind::Amp:
return Trailing{.level = BitwiseAnd, .is_binary = true};
case TokenKind::Pipe():
case TokenKind::Pipe:
return Trailing{.level = BitwiseOr, .is_binary = true};
case TokenKind::Xor():
case TokenKind::Xor:
return Trailing{.level = BitwiseXor, .is_binary = true};
case TokenKind::GreaterGreater():
case TokenKind::LessLess():
case TokenKind::GreaterGreater:
case TokenKind::LessLess:
return Trailing{.level = BitShift, .is_binary = true};
// Relational operators.
case TokenKind::EqualEqual():
case TokenKind::ExclaimEqual():
case TokenKind::Less():
case TokenKind::LessEqual():
case TokenKind::Greater():
case TokenKind::GreaterEqual():
case TokenKind::LessEqualGreater():
case TokenKind::EqualEqual:
case TokenKind::ExclaimEqual:
case TokenKind::Less:
case TokenKind::LessEqual:
case TokenKind::Greater:
case TokenKind::GreaterEqual:
case TokenKind::LessEqualGreater:
return Trailing{.level = Relational, .is_binary = true};
// Additive operators.
case TokenKind::Plus():
case TokenKind::Minus():
case TokenKind::Plus:
case TokenKind::Minus:
return Trailing{.level = Additive, .is_binary = true};
// Multiplicative operators.
case TokenKind::Slash():
case TokenKind::Slash:
return Trailing{.level = Multiplicative, .is_binary = true};
case TokenKind::Percent():
case TokenKind::Percent:
return Trailing{.level = Modulo, .is_binary = true};
// `*` could be multiplication or pointer type formation.
case TokenKind::Star():
case TokenKind::Star:
return infix ? Trailing{.level = Multiplicative, .is_binary = true}
: Trailing{.level = TypePostfix, .is_binary = false};
// Postfix operators.
case TokenKind::MinusMinus():
case TokenKind::PlusPlus():
case TokenKind::MinusMinus:
case TokenKind::PlusPlus:
return Trailing{.level = NumericPostfix, .is_binary = false};
// Prefix-only operators.
case TokenKind::Tilde():
case TokenKind::Not():
case TokenKind::Tilde:
case TokenKind::Not:
break;
// Symbolic tokens that might be operators eventually.
case TokenKind::Backslash():
case TokenKind::Caret():
case TokenKind::CaretEqual():
case TokenKind::Comma():
case TokenKind::TildeEqual():
case TokenKind::Exclaim():
case TokenKind::LessGreater():
case TokenKind::Question():
case TokenKind::Colon():
case TokenKind::Backslash:
case TokenKind::Caret:
case TokenKind::CaretEqual:
case TokenKind::Comma:
case TokenKind::TildeEqual:
case TokenKind::Exclaim:
case TokenKind::LessGreater:
case TokenKind::Question:
case TokenKind::Colon:
break;
// Symbolic tokens that are intentionally not operators.
case TokenKind::At():
case TokenKind::LessMinus():
case TokenKind::MinusGreater():
case TokenKind::EqualGreater():
case TokenKind::ColonEqual():
case TokenKind::Period():
case TokenKind::Semi():
case TokenKind::At:
case TokenKind::LessMinus:
case TokenKind::MinusGreater:
case TokenKind::EqualGreater:
case TokenKind::ColonEqual:
case TokenKind::Period:
case TokenKind::Semi:
break;
default:
toolchain/parser/precedence_test.cpp
+54 -61
View File
@@ -15,140 +15,133 @@ namespace {
using ::testing::Eq;
TEST(PrecedenceTest, OperatorsAreRecognized) {
EXPECT_TRUE(PrecedenceGroup::ForLeading(TokenKind::Minus()).has_value());
EXPECT_TRUE(PrecedenceGroup::ForLeading(TokenKind::Tilde()).has_value());
EXPECT_FALSE(PrecedenceGroup::ForLeading(TokenKind::Slash()).has_value());
EXPECT_FALSE(
PrecedenceGroup::ForLeading(TokenKind::Identifier()).has_value());
EXPECT_TRUE(PrecedenceGroup::ForLeading(TokenKind::Minus).has_value());
EXPECT_TRUE(PrecedenceGroup::ForLeading(TokenKind::Tilde).has_value());
EXPECT_FALSE(PrecedenceGroup::ForLeading(TokenKind::Slash).has_value());
EXPECT_FALSE(PrecedenceGroup::ForLeading(TokenKind::Identifier).has_value());
EXPECT_TRUE(
PrecedenceGroup::ForTrailing(TokenKind::Minus(), false).has_value());
PrecedenceGroup::ForTrailing(TokenKind::Minus, false).has_value());
EXPECT_FALSE(
PrecedenceGroup::ForTrailing(TokenKind::Tilde(), false).has_value());
EXPECT_TRUE(
PrecedenceGroup::ForTrailing(TokenKind::Slash(), true).has_value());
PrecedenceGroup::ForTrailing(TokenKind::Tilde, false).has_value());
EXPECT_TRUE(PrecedenceGroup::ForTrailing(TokenKind::Slash, true).has_value());
EXPECT_FALSE(
PrecedenceGroup::ForTrailing(TokenKind::Identifier(), false).has_value());
PrecedenceGroup::ForTrailing(TokenKind::Identifier, false).has_value());
EXPECT_TRUE(
PrecedenceGroup::ForTrailing(TokenKind::Minus(), true)->is_binary);
EXPECT_TRUE(PrecedenceGroup::ForTrailing(TokenKind::Minus, true)->is_binary);
EXPECT_FALSE(
PrecedenceGroup::ForTrailing(TokenKind::MinusMinus(), false)->is_binary);
PrecedenceGroup::ForTrailing(TokenKind::MinusMinus, false)->is_binary);
}
TEST(PrecedenceTest, InfixVsPostfix) {
// A trailing `-` is always infix; a trailing `--` is always postfix.
EXPECT_TRUE(
PrecedenceGroup::ForTrailing(TokenKind::Minus(), false)->is_binary);
EXPECT_TRUE(PrecedenceGroup::ForTrailing(TokenKind::Minus, false)->is_binary);
EXPECT_FALSE(
PrecedenceGroup::ForTrailing(TokenKind::MinusMinus(), true)->is_binary);
PrecedenceGroup::ForTrailing(TokenKind::MinusMinus, true)->is_binary);
// A trailing `*` is interpreted based on context.
EXPECT_TRUE(PrecedenceGroup::ForTrailing(TokenKind::Star(), true)->is_binary);
EXPECT_FALSE(
PrecedenceGroup::ForTrailing(TokenKind::Star(), false)->is_binary);
EXPECT_TRUE(PrecedenceGroup::ForTrailing(TokenKind::Star, true)->is_binary);
EXPECT_FALSE(PrecedenceGroup::ForTrailing(TokenKind::Star, false)->is_binary);
// Infix `*` can appear in type contexts; binary `*` cannot.
EXPECT_THAT(PrecedenceGroup::GetPriority(
PrecedenceGroup::ForTrailing(TokenKind::Star(), true)->level,
PrecedenceGroup::ForTrailing(TokenKind::Star, true)->level,
PrecedenceGroup::ForType()),
Eq(OperatorPriority::Ambiguous));
EXPECT_THAT(PrecedenceGroup::GetPriority(
PrecedenceGroup::ForTrailing(TokenKind::Star(), false)->level,
PrecedenceGroup::ForTrailing(TokenKind::Star, false)->level,
PrecedenceGroup::ForType()),
Eq(OperatorPriority::LeftFirst));
// Binary `*` can appear in `+` contexts; binary `*` cannot.
EXPECT_THAT(PrecedenceGroup::GetPriority(
PrecedenceGroup::ForTrailing(TokenKind::Star(), true)->level,
PrecedenceGroup::ForTrailing(TokenKind::Plus(), true)->level),
PrecedenceGroup::ForTrailing(TokenKind::Star, true)->level,
PrecedenceGroup::ForTrailing(TokenKind::Plus, true)->level),
Eq(OperatorPriority::LeftFirst));
EXPECT_THAT(PrecedenceGroup::GetPriority(
PrecedenceGroup::ForTrailing(TokenKind::Star(), false)->level,
PrecedenceGroup::ForTrailing(TokenKind::Plus(), true)->level),
PrecedenceGroup::ForTrailing(TokenKind::Star, false)->level,
PrecedenceGroup::ForTrailing(TokenKind::Plus, true)->level),
Eq(OperatorPriority::Ambiguous));
}
TEST(PrecedenceTest, Associativity) {
EXPECT_THAT(
PrecedenceGroup::ForLeading(TokenKind::Minus())->GetAssociativity(),
Eq(Associativity::RightToLeft));
EXPECT_THAT(PrecedenceGroup::ForTrailing(TokenKind::PlusPlus(), false)
EXPECT_THAT(PrecedenceGroup::ForLeading(TokenKind::Minus)->GetAssociativity(),
Eq(Associativity::RightToLeft));
EXPECT_THAT(PrecedenceGroup::ForTrailing(TokenKind::PlusPlus, false)
->level.GetAssociativity(),
Eq(Associativity::LeftToRight));
EXPECT_THAT(PrecedenceGroup::ForTrailing(TokenKind::Plus(), true)
EXPECT_THAT(PrecedenceGroup::ForTrailing(TokenKind::Plus, true)
->level.GetAssociativity(),
Eq(Associativity::LeftToRight));
EXPECT_THAT(PrecedenceGroup::ForTrailing(TokenKind::Equal(), true)
EXPECT_THAT(PrecedenceGroup::ForTrailing(TokenKind::Equal, true)
->level.GetAssociativity(),
Eq(Associativity::RightToLeft));
EXPECT_THAT(PrecedenceGroup::ForTrailing(TokenKind::PlusEqual(), true)
EXPECT_THAT(PrecedenceGroup::ForTrailing(TokenKind::PlusEqual, true)
->level.GetAssociativity(),
Eq(Associativity::None));
}
TEST(PrecedenceTest, DirectRelations) {
EXPECT_THAT(PrecedenceGroup::GetPriority(
PrecedenceGroup::ForTrailing(TokenKind::Star(), true)->level,
PrecedenceGroup::ForTrailing(TokenKind::Plus(), true)->level),
PrecedenceGroup::ForTrailing(TokenKind::Star, true)->level,
PrecedenceGroup::ForTrailing(TokenKind::Plus, true)->level),
Eq(OperatorPriority::LeftFirst));
EXPECT_THAT(PrecedenceGroup::GetPriority(
PrecedenceGroup::ForTrailing(TokenKind::Plus(), true)->level,
PrecedenceGroup::ForTrailing(TokenKind::Star(), true)->level),
PrecedenceGroup::ForTrailing(TokenKind::Plus, true)->level,
PrecedenceGroup::ForTrailing(TokenKind::Star, true)->level),
Eq(OperatorPriority::RightFirst));
EXPECT_THAT(PrecedenceGroup::GetPriority(
PrecedenceGroup::ForTrailing(TokenKind::Amp(), true)->level,
PrecedenceGroup::ForTrailing(TokenKind::Less(), true)->level),
PrecedenceGroup::ForTrailing(TokenKind::Amp, true)->level,
PrecedenceGroup::ForTrailing(TokenKind::Less, true)->level),
Eq(OperatorPriority::LeftFirst));
EXPECT_THAT(PrecedenceGroup::GetPriority(
PrecedenceGroup::ForTrailing(TokenKind::Less(), true)->level,
PrecedenceGroup::ForTrailing(TokenKind::Amp(), true)->level),
PrecedenceGroup::ForTrailing(TokenKind::Less, true)->level,
PrecedenceGroup::ForTrailing(TokenKind::Amp, true)->level),
Eq(OperatorPriority::RightFirst));
}
TEST(PrecedenceTest, IndirectRelations) {
EXPECT_THAT(PrecedenceGroup::GetPriority(
PrecedenceGroup::ForTrailing(TokenKind::Star(), true)->level,
PrecedenceGroup::ForTrailing(TokenKind::Or(), true)->level),
PrecedenceGroup::ForTrailing(TokenKind::Star, true)->level,
PrecedenceGroup::ForTrailing(TokenKind::Or, true)->level),
Eq(OperatorPriority::LeftFirst));
EXPECT_THAT(PrecedenceGroup::GetPriority(
PrecedenceGroup::ForTrailing(TokenKind::Or(), true)->level,
PrecedenceGroup::ForTrailing(TokenKind::Star(), true)->level),
PrecedenceGroup::ForTrailing(TokenKind::Or, true)->level,
PrecedenceGroup::ForTrailing(TokenKind::Star, true)->level),
Eq(OperatorPriority::RightFirst));
EXPECT_THAT(
PrecedenceGroup::GetPriority(
*PrecedenceGroup::ForLeading(TokenKind::Tilde()),
PrecedenceGroup::ForTrailing(TokenKind::Equal(), true)->level),
Eq(OperatorPriority::LeftFirst));
EXPECT_THAT(PrecedenceGroup::GetPriority(
PrecedenceGroup::ForTrailing(TokenKind::Equal(), true)->level,
*PrecedenceGroup::ForLeading(TokenKind::Tilde())),
*PrecedenceGroup::ForLeading(TokenKind::Tilde),
PrecedenceGroup::ForTrailing(TokenKind::Equal, true)->level),
Eq(OperatorPriority::LeftFirst));
EXPECT_THAT(PrecedenceGroup::GetPriority(
PrecedenceGroup::ForTrailing(TokenKind::Equal, true)->level,
*PrecedenceGroup::ForLeading(TokenKind::Tilde)),
Eq(OperatorPriority::RightFirst));
}
TEST(PrecedenceTest, IncomparableOperators) {
EXPECT_THAT(PrecedenceGroup::GetPriority(
*PrecedenceGroup::ForLeading(TokenKind::Tilde()),
*PrecedenceGroup::ForLeading(TokenKind::Not())),
*PrecedenceGroup::ForLeading(TokenKind::Tilde),
*PrecedenceGroup::ForLeading(TokenKind::Not)),
Eq(OperatorPriority::Ambiguous));
EXPECT_THAT(PrecedenceGroup::GetPriority(
*PrecedenceGroup::ForLeading(TokenKind::Tilde()),
*PrecedenceGroup::ForLeading(TokenKind::Minus())),
*PrecedenceGroup::ForLeading(TokenKind::Tilde),
*PrecedenceGroup::ForLeading(TokenKind::Minus)),
Eq(OperatorPriority::Ambiguous));
EXPECT_THAT(PrecedenceGroup::GetPriority(
*PrecedenceGroup::ForLeading(TokenKind::Minus()),
PrecedenceGroup::ForTrailing(TokenKind::Amp(), true)->level),
*PrecedenceGroup::ForLeading(TokenKind::Minus),
PrecedenceGroup::ForTrailing(TokenKind::Amp, true)->level),
Eq(OperatorPriority::Ambiguous));
EXPECT_THAT(
PrecedenceGroup::GetPriority(
PrecedenceGroup::ForTrailing(TokenKind::Equal(), true)->level,
PrecedenceGroup::ForTrailing(TokenKind::PipeEqual(), true)->level),
PrecedenceGroup::ForTrailing(TokenKind::Equal, true)->level,
PrecedenceGroup::ForTrailing(TokenKind::PipeEqual, true)->level),
Eq(OperatorPriority::Ambiguous));
EXPECT_THAT(PrecedenceGroup::GetPriority(
PrecedenceGroup::ForTrailing(TokenKind::Plus(), true)->level,
PrecedenceGroup::ForTrailing(TokenKind::Amp(), true)->level),
PrecedenceGroup::ForTrailing(TokenKind::Plus, true)->level,
PrecedenceGroup::ForTrailing(TokenKind::Amp, true)->level),
Eq(OperatorPriority::Ambiguous));
}
toolchain/semantics/semantics_parse_tree_handler.cpp
+4 -4
View File
@@ -447,7 +447,7 @@ auto SemanticsParseTreeHandler::HandleInfixOperator(ParseTree::Node parse_node)
// Figure out the operator for the token.
auto token = parse_tree_->node_token(parse_node);
switch (auto token_kind = tokens_->GetKind(token)) {
case TokenKind::Plus():
case TokenKind::Plus:
Push(parse_node, SemanticsNode::MakeBinaryOperatorAdd(
parse_node, result_type, lhs_id, rhs_id));
break;
@@ -475,18 +475,18 @@ auto SemanticsParseTreeHandler::HandleLiteral(ParseTree::Node parse_node)
-> void {
auto token = parse_tree_->node_token(parse_node);
switch (auto token_kind = tokens_->GetKind(token)) {
case TokenKind::IntegerLiteral(): {
case TokenKind::IntegerLiteral: {
auto id =
semantics_->AddIntegerLiteral(tokens_->GetIntegerLiteral(token));
Push(parse_node, SemanticsNode::MakeIntegerLiteral(parse_node, id));
break;
}
case TokenKind::RealLiteral(): {
case TokenKind::RealLiteral: {
// TODO: Add storage of the Real literal.
Push(parse_node, SemanticsNode::MakeRealLiteral(parse_node));
break;
}
case TokenKind::IntegerTypeLiteral(): {
case TokenKind::IntegerTypeLiteral: {
auto text = tokens_->GetTokenText(token);
CARBON_CHECK(text == "i32") << "Currently only i32 is allowed";
Push(parse_node, SemanticsNodeId::MakeBuiltinReference(