diff --git a/toolchain/lexer/BUILD b/toolchain/lexer/BUILD index 8ba3b57bf680..d1e6b660b9b6 100644 --- a/toolchain/lexer/BUILD +++ b/toolchain/lexer/BUILD @@ -14,6 +14,7 @@ cc_library( deps = [ "//common:check", "//common:ostream", + "//toolchain/common:enum_base", "@llvm-project//llvm:Support", ], ) diff --git a/toolchain/lexer/token_kind.cpp b/toolchain/lexer/token_kind.cpp index df56bfb406f7..ac262ce6902b 100644 --- a/toolchain/lexer/token_kind.cpp +++ b/toolchain/lexer/token_kind.cpp @@ -9,13 +9,10 @@ namespace Carbon { -auto TokenKind::name() const -> llvm::StringRef { - static constexpr llvm::StringLiteral Names[] = { -#define CARBON_TOKEN(TokenName) #TokenName, +CARBON_DEFINE_ENUM_CLASS_NAMES(TokenKind) = { +#define CARBON_TOKEN(TokenName) CARBON_ENUM_CLASS_NAME_STRING(TokenName) #include "toolchain/lexer/token_kind.def" - }; - return Names[static_cast(kind_value_)]; -} +}; auto TokenKind::is_symbol() const -> bool { static constexpr bool Table[] = { @@ -23,7 +20,7 @@ auto TokenKind::is_symbol() const -> bool { #define CARBON_SYMBOL_TOKEN(TokenName, Spelling) true, #include "toolchain/lexer/token_kind.def" }; - return Table[static_cast(kind_value_)]; + return Table[AsInt()]; } auto TokenKind::is_grouping_symbol() const -> bool { @@ -35,7 +32,7 @@ auto TokenKind::is_grouping_symbol() const -> bool { true, #include "toolchain/lexer/token_kind.def" }; - return Table[static_cast(kind_value_)]; + return Table[AsInt()]; } auto TokenKind::is_opening_symbol() const -> bool { @@ -45,18 +42,18 @@ auto TokenKind::is_opening_symbol() const -> bool { true, #include "toolchain/lexer/token_kind.def" }; - return Table[static_cast(kind_value_)]; + return Table[AsInt()]; } auto TokenKind::closing_symbol() const -> TokenKind { static constexpr TokenKind Table[] = { -#define CARBON_TOKEN(TokenName) Error(), +#define CARBON_TOKEN(TokenName) Error, #define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \ - ClosingName(), + ClosingName, #include "toolchain/lexer/token_kind.def" }; - auto result = Table[static_cast(kind_value_)]; - CARBON_CHECK(result != Error()) << "Only opening symbols are valid!"; + auto result = Table[AsInt()]; + CARBON_CHECK(result != Error) << "Only opening symbols are valid!"; return result; } @@ -67,18 +64,18 @@ auto TokenKind::is_closing_symbol() const -> bool { true, #include "toolchain/lexer/token_kind.def" }; - return Table[static_cast(kind_value_)]; + return Table[AsInt()]; } auto TokenKind::opening_symbol() const -> TokenKind { static constexpr TokenKind Table[] = { -#define CARBON_TOKEN(TokenName) Error(), +#define CARBON_TOKEN(TokenName) Error, #define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \ - OpeningName(), + OpeningName, #include "toolchain/lexer/token_kind.def" }; - auto result = Table[static_cast(kind_value_)]; - CARBON_CHECK(result != Error()) << "Only closing symbols are valid!"; + auto result = Table[AsInt()]; + CARBON_CHECK(result != Error) << "Only closing symbols are valid!"; return result; } @@ -88,13 +85,13 @@ auto TokenKind::is_keyword() const -> bool { #define CARBON_KEYWORD_TOKEN(TokenName, Spelling) true, #include "toolchain/lexer/token_kind.def" }; - return Table[static_cast(kind_value_)]; + return Table[AsInt()]; } auto TokenKind::is_sized_type_literal() const -> bool { - return *this == TokenKind::IntegerTypeLiteral() || - *this == TokenKind::UnsignedIntegerTypeLiteral() || - *this == TokenKind::FloatingPointTypeLiteral(); + return *this == TokenKind::IntegerTypeLiteral || + *this == TokenKind::UnsignedIntegerTypeLiteral || + *this == TokenKind::FloatingPointTypeLiteral; } auto TokenKind::fixed_spelling() const -> llvm::StringRef { @@ -104,7 +101,7 @@ auto TokenKind::fixed_spelling() const -> llvm::StringRef { #define CARBON_KEYWORD_TOKEN(TokenName, Spelling) Spelling, #include "toolchain/lexer/token_kind.def" }; - return Table[static_cast(kind_value_)]; + return Table[AsInt()]; } } // namespace Carbon diff --git a/toolchain/lexer/token_kind.h b/toolchain/lexer/token_kind.h index 2441c3477354..ff1b871e3b0d 100644 --- a/toolchain/lexer/token_kind.h +++ b/toolchain/lexer/token_kind.h @@ -6,38 +6,22 @@ #define CARBON_TOOLCHAIN_LEXER_TOKEN_KIND_H_ #include -#include -#include #include "common/ostream.h" -#include "llvm/ADT/StringRef.h" +#include "toolchain/common/enum_base.h" namespace Carbon { -class TokenKind { - // Note that this must be declared earlier in the class so that its type can - // be used, for example in the conversion operator. - enum class KindEnum : int8_t { -#define CARBON_TOKEN(TokenName) TokenName, +CARBON_DEFINE_RAW_ENUM_CLASS(TokenKind, uint8_t) { +#define CARBON_TOKEN(TokenName) CARBON_RAW_ENUM_ENUMERATOR(TokenName) #include "toolchain/lexer/token_kind.def" - }; +}; +class TokenKind : public CARBON_ENUM_BASE(TokenKind) { public: - // The formatting for this macro is weird due to a `clang-format` bug. See - // https://bugs.llvm.org/show_bug.cgi?id=48320 for details. -#define CARBON_TOKEN(TokenName) \ - static constexpr auto TokenName()->TokenKind { \ - return TokenKind(KindEnum::TokenName); \ - } +#define CARBON_TOKEN(TokenName) CARBON_ENUM_CONSTANT_DECLARATION(TokenName) #include "toolchain/lexer/token_kind.def" - // The default constructor is deleted as objects of this type should always be - // constructed using the above factory functions for each unique kind. - TokenKind() = delete; - - // Get a friendly name for the token for logging or debugging. - [[nodiscard]] auto name() const -> llvm::StringRef; - // Test whether this kind of token is a simple symbol sequence (punctuation, // not letters) that appears directly in the source text and can be // unambiguously lexed with `starts_with` logic. While these may appear @@ -87,21 +71,16 @@ class TokenKind { return false; } - // Enable conversion to our private enum, including in a `constexpr` context, - // to enable usage in `switch` and `case`. The enum remains private and - // nothing else should be using this. - // NOLINTNEXTLINE(google-explicit-constructor) - constexpr operator KindEnum() const { return kind_value_; } - - // Prints the TokenKind, typically for diagnostics. + // Override the EnumBase printing to use the fixed spelling rather than the + // name for tokens as this better corresponds to the source code the + // represent. void Print(llvm::raw_ostream& out) const { out << fixed_spelling(); } - - private: - constexpr explicit TokenKind(KindEnum kind_value) : kind_value_(kind_value) {} - - KindEnum kind_value_; }; +#define CARBON_TOKEN(TokenName) \ + CARBON_ENUM_CONSTANT_DEFINITION(TokenKind, TokenName) +#include "toolchain/lexer/token_kind.def" + } // namespace Carbon #endif // CARBON_TOOLCHAIN_LEXER_TOKEN_KIND_H_ diff --git a/toolchain/lexer/token_kind_test.cpp b/toolchain/lexer/token_kind_test.cpp index 67a3ade33279..0d9fa3eb862a 100644 --- a/toolchain/lexer/token_kind_test.cpp +++ b/toolchain/lexer/token_kind_test.cpp @@ -25,57 +25,55 @@ constexpr llvm::StringLiteral SymbolRegex = // exception: `Self`. constexpr llvm::StringLiteral KeywordRegex = "[a-z_]+|Self"; -#define CARBON_TOKEN(TokenName) \ - TEST(TokenKindTest, TokenName) { \ - EXPECT_EQ(#TokenName, TokenKind::TokenName().name()); \ - EXPECT_FALSE(TokenKind::TokenName().is_symbol()); \ - EXPECT_FALSE(TokenKind::TokenName().is_keyword()); \ - EXPECT_EQ("", TokenKind::TokenName().fixed_spelling()); \ +#define CARBON_TOKEN(TokenName) \ + TEST(TokenKindTest, TokenName) { \ + EXPECT_EQ(#TokenName, TokenKind::TokenName.name()); \ + EXPECT_FALSE(TokenKind::TokenName.is_symbol()); \ + EXPECT_FALSE(TokenKind::TokenName.is_keyword()); \ + EXPECT_EQ("", TokenKind::TokenName.fixed_spelling()); \ } -#define CARBON_SYMBOL_TOKEN(TokenName, Spelling) \ - TEST(TokenKindTest, TokenName) { \ - EXPECT_EQ(#TokenName, TokenKind::TokenName().name()); \ - EXPECT_TRUE(TokenKind::TokenName().is_symbol()); \ - EXPECT_FALSE(TokenKind::TokenName().is_grouping_symbol()); \ - EXPECT_FALSE(TokenKind::TokenName().is_opening_symbol()); \ - EXPECT_FALSE(TokenKind::TokenName().is_closing_symbol()); \ - EXPECT_FALSE(TokenKind::TokenName().is_keyword()); \ - EXPECT_EQ(Spelling, TokenKind::TokenName().fixed_spelling()); \ - EXPECT_THAT(Spelling, MatchesRegex(SymbolRegex.str())); \ +#define CARBON_SYMBOL_TOKEN(TokenName, Spelling) \ + TEST(TokenKindTest, TokenName) { \ + EXPECT_EQ(#TokenName, TokenKind::TokenName.name()); \ + EXPECT_TRUE(TokenKind::TokenName.is_symbol()); \ + EXPECT_FALSE(TokenKind::TokenName.is_grouping_symbol()); \ + EXPECT_FALSE(TokenKind::TokenName.is_opening_symbol()); \ + EXPECT_FALSE(TokenKind::TokenName.is_closing_symbol()); \ + EXPECT_FALSE(TokenKind::TokenName.is_keyword()); \ + EXPECT_EQ(Spelling, TokenKind::TokenName.fixed_spelling()); \ + EXPECT_THAT(Spelling, MatchesRegex(SymbolRegex.str())); \ } -#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \ - TEST(TokenKindTest, TokenName) { \ - EXPECT_EQ(#TokenName, TokenKind::TokenName().name()); \ - EXPECT_TRUE(TokenKind::TokenName().is_symbol()); \ - EXPECT_TRUE(TokenKind::TokenName().is_grouping_symbol()); \ - EXPECT_TRUE(TokenKind::TokenName().is_opening_symbol()); \ - EXPECT_EQ(TokenKind::ClosingName(), \ - TokenKind::TokenName().closing_symbol()); \ - EXPECT_FALSE(TokenKind::TokenName().is_closing_symbol()); \ - EXPECT_FALSE(TokenKind::TokenName().is_keyword()); \ - EXPECT_EQ(Spelling, TokenKind::TokenName().fixed_spelling()); \ - EXPECT_THAT(Spelling, MatchesRegex(SymbolRegex.str())); \ +#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \ + TEST(TokenKindTest, TokenName) { \ + EXPECT_EQ(#TokenName, TokenKind::TokenName.name()); \ + EXPECT_TRUE(TokenKind::TokenName.is_symbol()); \ + EXPECT_TRUE(TokenKind::TokenName.is_grouping_symbol()); \ + EXPECT_TRUE(TokenKind::TokenName.is_opening_symbol()); \ + EXPECT_EQ(TokenKind::ClosingName, TokenKind::TokenName.closing_symbol()); \ + EXPECT_FALSE(TokenKind::TokenName.is_closing_symbol()); \ + EXPECT_FALSE(TokenKind::TokenName.is_keyword()); \ + EXPECT_EQ(Spelling, TokenKind::TokenName.fixed_spelling()); \ + EXPECT_THAT(Spelling, MatchesRegex(SymbolRegex.str())); \ } -#define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \ - TEST(TokenKindTest, TokenName) { \ - EXPECT_EQ(#TokenName, TokenKind::TokenName().name()); \ - EXPECT_TRUE(TokenKind::TokenName().is_symbol()); \ - EXPECT_TRUE(TokenKind::TokenName().is_grouping_symbol()); \ - EXPECT_FALSE(TokenKind::TokenName().is_opening_symbol()); \ - EXPECT_TRUE(TokenKind::TokenName().is_closing_symbol()); \ - EXPECT_EQ(TokenKind::OpeningName(), \ - TokenKind::TokenName().opening_symbol()); \ - EXPECT_FALSE(TokenKind::TokenName().is_keyword()); \ - EXPECT_EQ(Spelling, TokenKind::TokenName().fixed_spelling()); \ - EXPECT_THAT(Spelling, MatchesRegex(SymbolRegex.str())); \ +#define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \ + TEST(TokenKindTest, TokenName) { \ + EXPECT_EQ(#TokenName, TokenKind::TokenName.name()); \ + EXPECT_TRUE(TokenKind::TokenName.is_symbol()); \ + EXPECT_TRUE(TokenKind::TokenName.is_grouping_symbol()); \ + EXPECT_FALSE(TokenKind::TokenName.is_opening_symbol()); \ + EXPECT_TRUE(TokenKind::TokenName.is_closing_symbol()); \ + EXPECT_EQ(TokenKind::OpeningName, TokenKind::TokenName.opening_symbol()); \ + EXPECT_FALSE(TokenKind::TokenName.is_keyword()); \ + EXPECT_EQ(Spelling, TokenKind::TokenName.fixed_spelling()); \ + EXPECT_THAT(Spelling, MatchesRegex(SymbolRegex.str())); \ } -#define CARBON_KEYWORD_TOKEN(TokenName, Spelling) \ - TEST(TokenKindTest, TokenName) { \ - EXPECT_EQ(#TokenName, TokenKind::TokenName().name()); \ - EXPECT_FALSE(TokenKind::TokenName().is_symbol()); \ - EXPECT_TRUE(TokenKind::TokenName().is_keyword()); \ - EXPECT_EQ(Spelling, TokenKind::TokenName().fixed_spelling()); \ - EXPECT_THAT(Spelling, MatchesRegex(KeywordRegex.str())); \ +#define CARBON_KEYWORD_TOKEN(TokenName, Spelling) \ + TEST(TokenKindTest, TokenName) { \ + EXPECT_EQ(#TokenName, TokenKind::TokenName.name()); \ + EXPECT_FALSE(TokenKind::TokenName.is_symbol()); \ + EXPECT_TRUE(TokenKind::TokenName.is_keyword()); \ + EXPECT_EQ(Spelling, TokenKind::TokenName.fixed_spelling()); \ + EXPECT_THAT(Spelling, MatchesRegex(KeywordRegex.str())); \ } #include "toolchain/lexer/token_kind.def" diff --git a/toolchain/lexer/tokenized_buffer.cpp b/toolchain/lexer/tokenized_buffer.cpp index 4fc7b3d15882..f7b448a7ace3 100644 --- a/toolchain/lexer/tokenized_buffer.cpp +++ b/toolchain/lexer/tokenized_buffer.cpp @@ -203,7 +203,7 @@ class TokenizedBuffer::Lexer { return VariantMatch( literal->ComputeValue(emitter_), [&](LexedNumericLiteral::IntegerValue&& value) { - auto token = buffer_->AddToken({.kind = TokenKind::IntegerLiteral(), + auto token = buffer_->AddToken({.kind = TokenKind::IntegerLiteral, .token_line = current_line_, .column = int_column}); buffer_->GetTokenInfo(token).literal_index = @@ -212,7 +212,7 @@ class TokenizedBuffer::Lexer { return token; }, [&](LexedNumericLiteral::RealValue&& value) { - auto token = buffer_->AddToken({.kind = TokenKind::RealLiteral(), + auto token = buffer_->AddToken({.kind = TokenKind::RealLiteral, .token_line = current_line_, .column = int_column}); buffer_->GetTokenInfo(token).literal_index = @@ -225,7 +225,7 @@ class TokenizedBuffer::Lexer { }, [&](LexedNumericLiteral::UnrecoverableError) { auto token = buffer_->AddToken({ - .kind = TokenKind::Error(), + .kind = TokenKind::Error, .token_line = current_line_, .column = int_column, .error_length = token_size, @@ -270,7 +270,7 @@ class TokenizedBuffer::Lexer { if (literal->is_terminated()) { auto token = - buffer_->AddToken({.kind = TokenKind::StringLiteral(), + buffer_->AddToken({.kind = TokenKind::StringLiteral, .token_line = string_line, .column = string_column, .literal_index = static_cast( @@ -282,7 +282,7 @@ class TokenizedBuffer::Lexer { CARBON_DIAGNOSTIC(UnterminatedString, Error, "String is missing a terminator."); emitter_.Emit(literal->text().begin(), UnterminatedString); - return buffer_->AddToken({.kind = TokenKind::Error(), + return buffer_->AddToken({.kind = TokenKind::Error, .token_line = string_line, .column = string_column, .error_length = literal_size}); @@ -292,10 +292,10 @@ class TokenizedBuffer::Lexer { auto LexSymbolToken(llvm::StringRef& source_text) -> LexResult { TokenKind kind = llvm::StringSwitch(source_text) #define CARBON_SYMBOL_TOKEN(Name, Spelling) \ - .StartsWith(Spelling, TokenKind::Name()) + .StartsWith(Spelling, TokenKind::Name) #include "toolchain/lexer/token_kind.def" - .Default(TokenKind::Error()); - if (kind == TokenKind::Error()) { + .Default(TokenKind::Error); + if (kind == TokenKind::Error) { return LexResult::NoMatch(); } @@ -328,7 +328,7 @@ class TokenizedBuffer::Lexer { // Check that there is a matching opening symbol before we consume this as // a closing symbol. if (open_groups_.empty()) { - closing_token_info.kind = TokenKind::Error(); + closing_token_info.kind = TokenKind::Error; closing_token_info.error_length = kind.fixed_spelling().size(); CARBON_DIAGNOSTIC( @@ -363,13 +363,13 @@ class TokenizedBuffer::Lexer { std::optional kind; switch (word.front()) { case 'i': - kind = TokenKind::IntegerTypeLiteral(); + kind = TokenKind::IntegerTypeLiteral; break; case 'u': - kind = TokenKind::UnsignedIntegerTypeLiteral(); + kind = TokenKind::UnsignedIntegerTypeLiteral; break; case 'f': - kind = TokenKind::FloatingPointTypeLiteral(); + kind = TokenKind::FloatingPointTypeLiteral; break; default: return LexResult::NoMatch(); @@ -378,7 +378,7 @@ class TokenizedBuffer::Lexer { llvm::StringRef suffix = word.substr(1); if (!CanLexInteger(emitter_, suffix)) { return buffer_->AddToken( - {.kind = TokenKind::Error(), + {.kind = TokenKind::Error, .token_line = current_line_, .column = column, .error_length = static_cast(word.size())}); @@ -399,7 +399,7 @@ class TokenizedBuffer::Lexer { // Closes all open groups that cannot remain open across the symbol `K`. // Users may pass `Error` to close all open groups. auto CloseInvalidOpenGroups(TokenKind kind) -> void { - if (!kind.is_closing_symbol() && kind != TokenKind::Error()) { + if (!kind.is_closing_symbol() && kind != TokenKind::Error) { return; } @@ -471,17 +471,17 @@ class TokenizedBuffer::Lexer { // Check if the text matches a keyword token, and if so use that. TokenKind kind = llvm::StringSwitch(identifier_text) -#define CARBON_KEYWORD_TOKEN(Name, Spelling) .Case(Spelling, TokenKind::Name()) +#define CARBON_KEYWORD_TOKEN(Name, Spelling) .Case(Spelling, TokenKind::Name) #include "toolchain/lexer/token_kind.def" - .Default(TokenKind::Error()); - if (kind != TokenKind::Error()) { + .Default(TokenKind::Error); + if (kind != TokenKind::Error) { return buffer_->AddToken({.kind = kind, .token_line = current_line_, .column = identifier_column}); } // Otherwise we have a generic identifier. - return buffer_->AddToken({.kind = TokenKind::Identifier(), + return buffer_->AddToken({.kind = TokenKind::Identifier, .token_line = current_line_, .column = identifier_column, .id = GetOrCreateIdentifier(identifier_text)}); @@ -510,7 +510,7 @@ class TokenizedBuffer::Lexer { } auto token = buffer_->AddToken( - {.kind = TokenKind::Error(), + {.kind = TokenKind::Error, .token_line = current_line_, .column = current_column_, .error_length = static_cast(error_text.size())}); @@ -524,7 +524,7 @@ class TokenizedBuffer::Lexer { } auto AddEndOfFileToken() -> void { - buffer_->AddToken({.kind = TokenKind::EndOfFile(), + buffer_->AddToken({.kind = TokenKind::EndOfFile, .token_line = current_line_, .column = current_column_}); } @@ -576,7 +576,7 @@ auto TokenizedBuffer::Lex(SourceBuffer& source, DiagnosticConsumer& consumer) // The end-of-file token is always considered to be whitespace. lexer.NoteWhitespace(); - lexer.CloseInvalidOpenGroups(TokenKind::Error()); + lexer.CloseInvalidOpenGroups(TokenKind::Error); lexer.AddEndOfFileToken(); if (error_tracking_consumer.seen_error()) { @@ -609,7 +609,7 @@ auto TokenizedBuffer::GetTokenText(Token token) const -> llvm::StringRef { return fixed_spelling; } - if (token_info.kind == TokenKind::Error()) { + if (token_info.kind == TokenKind::Error) { const auto& line_info = GetLineInfo(token_info.token_line); int64_t token_start = line_info.start + token_info.column; return source_->text().substr(token_start, token_info.error_length); @@ -617,8 +617,8 @@ auto TokenizedBuffer::GetTokenText(Token token) const -> llvm::StringRef { // Refer back to the source text to preserve oddities like radix or digit // separators the author included. - if (token_info.kind == TokenKind::IntegerLiteral() || - token_info.kind == TokenKind::RealLiteral()) { + if (token_info.kind == TokenKind::IntegerLiteral || + token_info.kind == TokenKind::RealLiteral) { const auto& line_info = GetLineInfo(token_info.token_line); int64_t token_start = line_info.start + token_info.column; std::optional relexed_token = @@ -629,7 +629,7 @@ auto TokenizedBuffer::GetTokenText(Token token) const -> llvm::StringRef { // Refer back to the source text to find the original spelling, including // escape sequences etc. - if (token_info.kind == TokenKind::StringLiteral()) { + if (token_info.kind == TokenKind::StringLiteral) { const auto& line_info = GetLineInfo(token_info.token_line); int64_t token_start = line_info.start + token_info.column; std::optional relexed_token = @@ -648,18 +648,18 @@ auto TokenizedBuffer::GetTokenText(Token token) const -> llvm::StringRef { return llvm::StringRef(suffix.data() - 1, suffix.size() + 1); } - if (token_info.kind == TokenKind::EndOfFile()) { + if (token_info.kind == TokenKind::EndOfFile) { return llvm::StringRef(); } - CARBON_CHECK(token_info.kind == TokenKind::Identifier()) + CARBON_CHECK(token_info.kind == TokenKind::Identifier) << token_info.kind.name(); return GetIdentifierText(token_info.id); } auto TokenizedBuffer::GetIdentifier(Token token) const -> Identifier { const auto& token_info = GetTokenInfo(token); - CARBON_CHECK(token_info.kind == TokenKind::Identifier()) + CARBON_CHECK(token_info.kind == TokenKind::Identifier) << token_info.kind.name(); return token_info.id; } @@ -667,14 +667,14 @@ auto TokenizedBuffer::GetIdentifier(Token token) const -> Identifier { auto TokenizedBuffer::GetIntegerLiteral(Token token) const -> const llvm::APInt& { const auto& token_info = GetTokenInfo(token); - CARBON_CHECK(token_info.kind == TokenKind::IntegerLiteral()) + CARBON_CHECK(token_info.kind == TokenKind::IntegerLiteral) << token_info.kind.name(); return literal_int_storage_[token_info.literal_index]; } auto TokenizedBuffer::GetRealLiteral(Token token) const -> RealLiteralValue { const auto& token_info = GetTokenInfo(token); - CARBON_CHECK(token_info.kind == TokenKind::RealLiteral()) + CARBON_CHECK(token_info.kind == TokenKind::RealLiteral) << token_info.kind.name(); // Note that every real literal is at least three characters long, so we can @@ -690,7 +690,7 @@ auto TokenizedBuffer::GetRealLiteral(Token token) const -> RealLiteralValue { auto TokenizedBuffer::GetStringLiteral(Token token) const -> llvm::StringRef { const auto& token_info = GetTokenInfo(token); - CARBON_CHECK(token_info.kind == TokenKind::StringLiteral()) + CARBON_CHECK(token_info.kind == TokenKind::StringLiteral) << token_info.kind.name(); return literal_string_storage_[token_info.literal_index]; } @@ -826,18 +826,18 @@ auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream, Token token, token_text); switch (token_info.kind) { - case TokenKind::Identifier(): + case TokenKind::Identifier: output_stream << ", identifier: " << GetIdentifier(token).index; break; - case TokenKind::IntegerLiteral(): + case TokenKind::IntegerLiteral: output_stream << ", value: `"; GetIntegerLiteral(token).print(output_stream, /*isSigned=*/false); output_stream << "`"; break; - case TokenKind::RealLiteral(): + case TokenKind::RealLiteral: output_stream << ", value: `" << GetRealLiteral(token) << "`"; break; - case TokenKind::StringLiteral(): + case TokenKind::StringLiteral: output_stream << ", value: `" << GetStringLiteral(token) << "`"; break; default: diff --git a/toolchain/lexer/tokenized_buffer_test.cpp b/toolchain/lexer/tokenized_buffer_test.cpp index b7f0d34d14df..4d70b8e4c800 100644 --- a/toolchain/lexer/tokenized_buffer_test.cpp +++ b/toolchain/lexer/tokenized_buffer_test.cpp @@ -50,56 +50,41 @@ class LexerTest : public ::testing::Test { TEST_F(LexerTest, HandlesEmptyBuffer) { auto buffer = Lex(""); EXPECT_FALSE(buffer.has_errors()); - EXPECT_THAT( - buffer, - HasTokens(llvm::ArrayRef{{TokenKind::EndOfFile()}})); + EXPECT_THAT(buffer, + HasTokens(llvm::ArrayRef{{TokenKind::EndOfFile}})); } TEST_F(LexerTest, TracksLinesAndColumns) { auto buffer = Lex("\n ;;\n ;;;\n x\"foo\" '''baz\n a\n ''' y"); EXPECT_FALSE(buffer.has_errors()); - EXPECT_THAT(buffer, - HasTokens(llvm::ArrayRef{ - {.kind = TokenKind::Semi(), - .line = 2, - .column = 3, - .indent_column = 3}, - {.kind = TokenKind::Semi(), - .line = 2, - .column = 4, - .indent_column = 3}, - {.kind = TokenKind::Semi(), - .line = 3, - .column = 4, - .indent_column = 4}, - {.kind = TokenKind::Semi(), - .line = 3, - .column = 5, - .indent_column = 4}, - {.kind = TokenKind::Semi(), - .line = 3, - .column = 6, - .indent_column = 4}, - {.kind = TokenKind::Identifier(), - .line = 4, - .column = 4, - .indent_column = 4, - .text = "x"}, - {.kind = TokenKind::StringLiteral(), - .line = 4, - .column = 5, - .indent_column = 4}, - {.kind = TokenKind::StringLiteral(), - .line = 4, - .column = 11, - .indent_column = 4}, - {.kind = TokenKind::Identifier(), - .line = 6, - .column = 6, - .indent_column = 11, - .text = "y"}, - {.kind = TokenKind::EndOfFile(), .line = 6, .column = 7}, - })); + EXPECT_THAT( + buffer, + HasTokens(llvm::ArrayRef{ + {.kind = TokenKind::Semi, .line = 2, .column = 3, .indent_column = 3}, + {.kind = TokenKind::Semi, .line = 2, .column = 4, .indent_column = 3}, + {.kind = TokenKind::Semi, .line = 3, .column = 4, .indent_column = 4}, + {.kind = TokenKind::Semi, .line = 3, .column = 5, .indent_column = 4}, + {.kind = TokenKind::Semi, .line = 3, .column = 6, .indent_column = 4}, + {.kind = TokenKind::Identifier, + .line = 4, + .column = 4, + .indent_column = 4, + .text = "x"}, + {.kind = TokenKind::StringLiteral, + .line = 4, + .column = 5, + .indent_column = 4}, + {.kind = TokenKind::StringLiteral, + .line = 4, + .column = 11, + .indent_column = 4}, + {.kind = TokenKind::Identifier, + .line = 6, + .column = 6, + .indent_column = 11, + .text = "y"}, + {.kind = TokenKind::EndOfFile, .line = 6, .column = 7}, + })); } TEST_F(LexerTest, HandlesNumericLiteral) { @@ -107,51 +92,51 @@ TEST_F(LexerTest, HandlesNumericLiteral) { EXPECT_FALSE(buffer.has_errors()); ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {.kind = TokenKind::IntegerLiteral(), + {.kind = TokenKind::IntegerLiteral, .line = 1, .column = 1, .indent_column = 1, .text = "12"}, - {.kind = TokenKind::Minus(), + {.kind = TokenKind::Minus, .line = 1, .column = 3, .indent_column = 1}, - {.kind = TokenKind::IntegerLiteral(), + {.kind = TokenKind::IntegerLiteral, .line = 1, .column = 4, .indent_column = 1, .text = "578"}, - {.kind = TokenKind::IntegerLiteral(), + {.kind = TokenKind::IntegerLiteral, .line = 2, .column = 3, .indent_column = 3, .text = "1"}, - {.kind = TokenKind::IntegerLiteral(), + {.kind = TokenKind::IntegerLiteral, .line = 2, .column = 6, .indent_column = 3, .text = "2"}, - {.kind = TokenKind::IntegerLiteral(), + {.kind = TokenKind::IntegerLiteral, .line = 3, .column = 1, .indent_column = 1, .text = "0x12_3ABC"}, - {.kind = TokenKind::IntegerLiteral(), + {.kind = TokenKind::IntegerLiteral, .line = 4, .column = 1, .indent_column = 1, .text = "0b10_10_11"}, - {.kind = TokenKind::IntegerLiteral(), + {.kind = TokenKind::IntegerLiteral, .line = 5, .column = 1, .indent_column = 1, .text = "1_234_567"}, - {.kind = TokenKind::RealLiteral(), + {.kind = TokenKind::RealLiteral, .line = 6, .column = 1, .indent_column = 1, .text = "1.5e9"}, - {.kind = TokenKind::EndOfFile(), .line = 6, .column = 6}, + {.kind = TokenKind::EndOfFile, .line = 6, .column = 6}, })); auto token_12 = buffer.tokens().begin(); EXPECT_EQ(buffer.GetIntegerLiteral(*token_12), 12); @@ -179,32 +164,32 @@ TEST_F(LexerTest, HandlesInvalidNumericLiterals) { EXPECT_TRUE(buffer.has_errors()); ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {.kind = TokenKind::Error(), + {.kind = TokenKind::Error, .line = 1, .column = 1, .indent_column = 1, .text = "14x"}, - {.kind = TokenKind::IntegerLiteral(), + {.kind = TokenKind::IntegerLiteral, .line = 1, .column = 5, .indent_column = 1, .text = "15_49"}, - {.kind = TokenKind::Error(), + {.kind = TokenKind::Error, .line = 1, .column = 11, .indent_column = 1, .text = "0x3.5q"}, - {.kind = TokenKind::RealLiteral(), + {.kind = TokenKind::RealLiteral, .line = 1, .column = 18, .indent_column = 1, .text = "0x3_4.5_6"}, - {.kind = TokenKind::Error(), + {.kind = TokenKind::Error, .line = 1, .column = 28, .indent_column = 1, .text = "0ops"}, - {.kind = TokenKind::EndOfFile(), .line = 1, .column = 32}, + {.kind = TokenKind::EndOfFile, .line = 1, .column = 32}, })); } @@ -226,58 +211,57 @@ TEST_F(LexerTest, SplitsNumericLiteralsProperly) { )"; auto buffer = Lex(source_text); EXPECT_TRUE(buffer.has_errors()); - EXPECT_THAT(buffer, - HasTokens(llvm::ArrayRef{ - {.kind = TokenKind::IntegerLiteral(), .text = "1"}, - {.kind = TokenKind::Period()}, - // newline - {.kind = TokenKind::Period()}, - {.kind = TokenKind::IntegerLiteral(), .text = "2"}, - // newline - {.kind = TokenKind::IntegerLiteral(), .text = "3"}, - {.kind = TokenKind::Period()}, - {.kind = TokenKind::Plus()}, - {.kind = TokenKind::Identifier(), .text = "foo"}, - // newline - {.kind = TokenKind::RealLiteral(), .text = "4.0"}, - {.kind = TokenKind::Minus()}, - {.kind = TokenKind::Identifier(), .text = "bar"}, - // newline - {.kind = TokenKind::RealLiteral(), .text = "5.0e+123"}, - {.kind = TokenKind::Plus()}, - {.kind = TokenKind::IntegerLiteral(), .text = "456"}, - // newline - {.kind = TokenKind::Error(), .text = "6.0e+1e"}, - {.kind = TokenKind::Plus()}, - {.kind = TokenKind::IntegerLiteral(), .text = "2"}, - // newline - {.kind = TokenKind::Error(), .text = "1e7"}, - // newline - {.kind = TokenKind::IntegerLiteral(), .text = "8"}, - {.kind = TokenKind::Period()}, - {.kind = TokenKind::Period()}, - {.kind = TokenKind::IntegerLiteral(), .text = "10"}, - // newline - {.kind = TokenKind::RealLiteral(), .text = "9.0"}, - {.kind = TokenKind::Period()}, - {.kind = TokenKind::RealLiteral(), .text = "9.5"}, - // newline - {.kind = TokenKind::Error(), .text = "10.foo"}, - // newline - {.kind = TokenKind::RealLiteral(), .text = "11.0"}, - {.kind = TokenKind::Period()}, - {.kind = TokenKind::Identifier(), .text = "foo"}, - // newline - {.kind = TokenKind::Error(), .text = "12e"}, - {.kind = TokenKind::Plus()}, - {.kind = TokenKind::IntegerLiteral(), .text = "1"}, - // newline - {.kind = TokenKind::IntegerLiteral(), .text = "13"}, - {.kind = TokenKind::Period()}, - {.kind = TokenKind::Underscore()}, - // newline - {.kind = TokenKind::EndOfFile()}, - })); + EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ + {.kind = TokenKind::IntegerLiteral, .text = "1"}, + {.kind = TokenKind::Period}, + // newline + {.kind = TokenKind::Period}, + {.kind = TokenKind::IntegerLiteral, .text = "2"}, + // newline + {.kind = TokenKind::IntegerLiteral, .text = "3"}, + {.kind = TokenKind::Period}, + {.kind = TokenKind::Plus}, + {.kind = TokenKind::Identifier, .text = "foo"}, + // newline + {.kind = TokenKind::RealLiteral, .text = "4.0"}, + {.kind = TokenKind::Minus}, + {.kind = TokenKind::Identifier, .text = "bar"}, + // newline + {.kind = TokenKind::RealLiteral, .text = "5.0e+123"}, + {.kind = TokenKind::Plus}, + {.kind = TokenKind::IntegerLiteral, .text = "456"}, + // newline + {.kind = TokenKind::Error, .text = "6.0e+1e"}, + {.kind = TokenKind::Plus}, + {.kind = TokenKind::IntegerLiteral, .text = "2"}, + // newline + {.kind = TokenKind::Error, .text = "1e7"}, + // newline + {.kind = TokenKind::IntegerLiteral, .text = "8"}, + {.kind = TokenKind::Period}, + {.kind = TokenKind::Period}, + {.kind = TokenKind::IntegerLiteral, .text = "10"}, + // newline + {.kind = TokenKind::RealLiteral, .text = "9.0"}, + {.kind = TokenKind::Period}, + {.kind = TokenKind::RealLiteral, .text = "9.5"}, + // newline + {.kind = TokenKind::Error, .text = "10.foo"}, + // newline + {.kind = TokenKind::RealLiteral, .text = "11.0"}, + {.kind = TokenKind::Period}, + {.kind = TokenKind::Identifier, .text = "foo"}, + // newline + {.kind = TokenKind::Error, .text = "12e"}, + {.kind = TokenKind::Plus}, + {.kind = TokenKind::IntegerLiteral, .text = "1"}, + // newline + {.kind = TokenKind::IntegerLiteral, .text = "13"}, + {.kind = TokenKind::Period}, + {.kind = TokenKind::Underscore}, + // newline + {.kind = TokenKind::EndOfFile}, + })); } TEST_F(LexerTest, HandlesGarbageCharacters) { @@ -287,32 +271,29 @@ TEST_F(LexerTest, HandlesGarbageCharacters) { EXPECT_THAT( buffer, HasTokens(llvm::ArrayRef{ - {.kind = TokenKind::Error(), + {.kind = TokenKind::Error, .line = 1, .column = 1, // 💩 takes 4 bytes, and we count column as bytes offset. .text = llvm::StringRef("$$💩", 6)}, - {.kind = TokenKind::Minus(), .line = 1, .column = 7}, - {.kind = TokenKind::Error(), .line = 1, .column = 8, .text = "$"}, + {.kind = TokenKind::Minus, .line = 1, .column = 7}, + {.kind = TokenKind::Error, .line = 1, .column = 8, .text = "$"}, // newline - {.kind = TokenKind::Error(), + {.kind = TokenKind::Error, .line = 2, .column = 1, .text = llvm::StringRef("$\0$", 3)}, - {.kind = TokenKind::IntegerLiteral(), + {.kind = TokenKind::IntegerLiteral, .line = 2, .column = 4, .text = "12"}, - {.kind = TokenKind::Error(), .line = 2, .column = 6, .text = "$"}, + {.kind = TokenKind::Error, .line = 2, .column = 6, .text = "$"}, // newline - {.kind = TokenKind::Backslash(), - .line = 3, - .column = 1, - .text = "\\"}, - {.kind = TokenKind::Error(), .line = 3, .column = 2, .text = "\"\\"}, + {.kind = TokenKind::Backslash, .line = 3, .column = 1, .text = "\\"}, + {.kind = TokenKind::Error, .line = 3, .column = 2, .text = "\"\\"}, // newline - {.kind = TokenKind::Error(), .line = 4, .column = 1, .text = "\"x"}, - {.kind = TokenKind::EndOfFile(), .line = 4, .column = 3}, + {.kind = TokenKind::Error, .line = 4, .column = 1, .text = "\"x"}, + {.kind = TokenKind::EndOfFile, .line = 4, .column = 3}, })); } @@ -323,39 +304,39 @@ TEST_F(LexerTest, Symbols) { auto buffer = Lex("<<<"); EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::LessLess()}, - {TokenKind::Less()}, - {TokenKind::EndOfFile()}, + {TokenKind::LessLess}, + {TokenKind::Less}, + {TokenKind::EndOfFile}, })); buffer = Lex("<<=>>"); EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::LessLessEqual()}, - {TokenKind::GreaterGreater()}, - {TokenKind::EndOfFile()}, + {TokenKind::LessLessEqual}, + {TokenKind::GreaterGreater}, + {TokenKind::EndOfFile}, })); buffer = Lex("< <=> >"); EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::Less()}, - {TokenKind::LessEqualGreater()}, - {TokenKind::Greater()}, - {TokenKind::EndOfFile()}, + {TokenKind::Less}, + {TokenKind::LessEqualGreater}, + {TokenKind::Greater}, + {TokenKind::EndOfFile}, })); buffer = Lex("\\/?@&^!"); EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::Backslash()}, - {TokenKind::Slash()}, - {TokenKind::Question()}, - {TokenKind::At()}, - {TokenKind::Amp()}, - {TokenKind::Caret()}, - {TokenKind::Exclaim()}, - {TokenKind::EndOfFile()}, + {TokenKind::Backslash}, + {TokenKind::Slash}, + {TokenKind::Question}, + {TokenKind::At}, + {TokenKind::Amp}, + {TokenKind::Caret}, + {TokenKind::Exclaim}, + {TokenKind::EndOfFile}, })); } @@ -363,23 +344,23 @@ TEST_F(LexerTest, Parens) { auto buffer = Lex("()"); EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::OpenParen()}, - {TokenKind::CloseParen()}, - {TokenKind::EndOfFile()}, + {TokenKind::OpenParen}, + {TokenKind::CloseParen}, + {TokenKind::EndOfFile}, })); buffer = Lex("((()()))"); EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::OpenParen()}, - {TokenKind::OpenParen()}, - {TokenKind::OpenParen()}, - {TokenKind::CloseParen()}, - {TokenKind::OpenParen()}, - {TokenKind::CloseParen()}, - {TokenKind::CloseParen()}, - {TokenKind::CloseParen()}, - {TokenKind::EndOfFile()}, + {TokenKind::OpenParen}, + {TokenKind::OpenParen}, + {TokenKind::OpenParen}, + {TokenKind::CloseParen}, + {TokenKind::OpenParen}, + {TokenKind::CloseParen}, + {TokenKind::CloseParen}, + {TokenKind::CloseParen}, + {TokenKind::EndOfFile}, })); } @@ -387,23 +368,23 @@ TEST_F(LexerTest, CurlyBraces) { auto buffer = Lex("{}"); EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::OpenCurlyBrace()}, - {TokenKind::CloseCurlyBrace()}, - {TokenKind::EndOfFile()}, + {TokenKind::OpenCurlyBrace}, + {TokenKind::CloseCurlyBrace}, + {TokenKind::EndOfFile}, })); buffer = Lex("{{{}{}}}"); EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::OpenCurlyBrace()}, - {TokenKind::OpenCurlyBrace()}, - {TokenKind::OpenCurlyBrace()}, - {TokenKind::CloseCurlyBrace()}, - {TokenKind::OpenCurlyBrace()}, - {TokenKind::CloseCurlyBrace()}, - {TokenKind::CloseCurlyBrace()}, - {TokenKind::CloseCurlyBrace()}, - {TokenKind::EndOfFile()}, + {TokenKind::OpenCurlyBrace}, + {TokenKind::OpenCurlyBrace}, + {TokenKind::OpenCurlyBrace}, + {TokenKind::CloseCurlyBrace}, + {TokenKind::OpenCurlyBrace}, + {TokenKind::CloseCurlyBrace}, + {TokenKind::CloseCurlyBrace}, + {TokenKind::CloseCurlyBrace}, + {TokenKind::EndOfFile}, })); } @@ -425,7 +406,7 @@ TEST_F(LexerTest, MatchingGroups) { EXPECT_EQ(open_curly_token, buffer.GetMatchedOpeningToken(close_curly_token)); auto eof_token = *it++; - EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::EndOfFile()); + EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::EndOfFile); EXPECT_EQ(buffer.tokens().end(), it); } @@ -488,7 +469,7 @@ TEST_F(LexerTest, MatchingGroups) { buffer.GetMatchedOpeningToken(inner_close_paren_token)); auto eof_token = *it++; - EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::EndOfFile()); + EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::EndOfFile); EXPECT_EQ(buffer.tokens().end(), it); } } @@ -498,16 +479,16 @@ TEST_F(LexerTest, MismatchedGroups) { EXPECT_TRUE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::OpenCurlyBrace()}, - {.kind = TokenKind::CloseCurlyBrace(), .recovery = true}, - {TokenKind::EndOfFile()}, + {TokenKind::OpenCurlyBrace}, + {.kind = TokenKind::CloseCurlyBrace, .recovery = true}, + {TokenKind::EndOfFile}, })); buffer = Lex("}"); EXPECT_TRUE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {.kind = TokenKind::Error(), .text = "}"}, - {TokenKind::EndOfFile()}, + {.kind = TokenKind::Error, .text = "}"}, + {TokenKind::EndOfFile}, })); buffer = Lex("{(}"); @@ -515,11 +496,11 @@ TEST_F(LexerTest, MismatchedGroups) { EXPECT_THAT( buffer, HasTokens(llvm::ArrayRef{ - {.kind = TokenKind::OpenCurlyBrace(), .column = 1}, - {.kind = TokenKind::OpenParen(), .column = 2}, - {.kind = TokenKind::CloseParen(), .column = 3, .recovery = true}, - {.kind = TokenKind::CloseCurlyBrace(), .column = 3}, - {TokenKind::EndOfFile()}, + {.kind = TokenKind::OpenCurlyBrace, .column = 1}, + {.kind = TokenKind::OpenParen, .column = 2}, + {.kind = TokenKind::CloseParen, .column = 3, .recovery = true}, + {.kind = TokenKind::CloseCurlyBrace, .column = 3}, + {TokenKind::EndOfFile}, })); buffer = Lex(")({)"); @@ -527,12 +508,12 @@ TEST_F(LexerTest, MismatchedGroups) { EXPECT_THAT( buffer, HasTokens(llvm::ArrayRef{ - {.kind = TokenKind::Error(), .column = 1, .text = ")"}, - {.kind = TokenKind::OpenParen(), .column = 2}, - {.kind = TokenKind::OpenCurlyBrace(), .column = 3}, - {.kind = TokenKind::CloseCurlyBrace(), .column = 4, .recovery = true}, - {.kind = TokenKind::CloseParen(), .column = 4}, - {TokenKind::EndOfFile()}, + {.kind = TokenKind::Error, .column = 1, .text = ")"}, + {.kind = TokenKind::OpenParen, .column = 2}, + {.kind = TokenKind::OpenCurlyBrace, .column = 3}, + {.kind = TokenKind::CloseCurlyBrace, .column = 4, .recovery = true}, + {.kind = TokenKind::CloseParen, .column = 4}, + {TokenKind::EndOfFile}, })); } @@ -575,63 +556,55 @@ TEST_F(LexerTest, Keywords) { EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {.kind = TokenKind::Fn(), .column = 4, .indent_column = 4}, - {TokenKind::EndOfFile()}, + {.kind = TokenKind::Fn, .column = 4, .indent_column = 4}, + {TokenKind::EndOfFile}, })); buffer = Lex("and or not if else for return var break continue _"); EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {TokenKind::And()}, - {TokenKind::Or()}, - {TokenKind::Not()}, - {TokenKind::If()}, - {TokenKind::Else()}, - {TokenKind::For()}, - {TokenKind::Return()}, - {TokenKind::Var()}, - {TokenKind::Break()}, - {TokenKind::Continue()}, - {TokenKind::Underscore()}, - {TokenKind::EndOfFile()}, + {TokenKind::And}, + {TokenKind::Or}, + {TokenKind::Not}, + {TokenKind::If}, + {TokenKind::Else}, + {TokenKind::For}, + {TokenKind::Return}, + {TokenKind::Var}, + {TokenKind::Break}, + {TokenKind::Continue}, + {TokenKind::Underscore}, + {TokenKind::EndOfFile}, })); } TEST_F(LexerTest, Comments) { auto buffer = Lex(" ;\n // foo\n ;\n"); EXPECT_FALSE(buffer.has_errors()); - EXPECT_THAT(buffer, - HasTokens(llvm::ArrayRef{ - {.kind = TokenKind::Semi(), - .line = 1, - .column = 2, - .indent_column = 2}, - {.kind = TokenKind::Semi(), - .line = 3, - .column = 3, - .indent_column = 3}, - {.kind = TokenKind::EndOfFile(), .line = 3, .column = 4}, - })); + EXPECT_THAT( + buffer, + HasTokens(llvm::ArrayRef{ + {.kind = TokenKind::Semi, .line = 1, .column = 2, .indent_column = 2}, + {.kind = TokenKind::Semi, .line = 3, .column = 3, .indent_column = 3}, + {.kind = TokenKind::EndOfFile, .line = 3, .column = 4}, + })); buffer = Lex("// foo\n//\n// bar"); EXPECT_FALSE(buffer.has_errors()); - EXPECT_THAT( - buffer, - HasTokens(llvm::ArrayRef{{TokenKind::EndOfFile()}})); + EXPECT_THAT(buffer, + HasTokens(llvm::ArrayRef{{TokenKind::EndOfFile}})); // Make sure weird characters aren't a problem. buffer = Lex(" // foo#$!^?@-_💩🍫⃠ [̲̅$̲̅(̲̅ ͡° ͜ʖ ͡°̲̅)̲̅$̲̅]"); EXPECT_FALSE(buffer.has_errors()); - EXPECT_THAT( - buffer, - HasTokens(llvm::ArrayRef{{TokenKind::EndOfFile()}})); + EXPECT_THAT(buffer, + HasTokens(llvm::ArrayRef{{TokenKind::EndOfFile}})); // Make sure we can lex a comment at the end of the input. buffer = Lex("//"); EXPECT_FALSE(buffer.has_errors()); - EXPECT_THAT( - buffer, - HasTokens(llvm::ArrayRef{{TokenKind::EndOfFile()}})); + EXPECT_THAT(buffer, + HasTokens(llvm::ArrayRef{{TokenKind::EndOfFile}})); } TEST_F(LexerTest, InvalidComments) { @@ -651,35 +624,34 @@ TEST_F(LexerTest, Identifiers) { auto buffer = Lex(" foobar"); EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {.kind = TokenKind::Identifier(), + {.kind = TokenKind::Identifier, .column = 4, .indent_column = 4, .text = "foobar"}, - {TokenKind::EndOfFile()}, + {TokenKind::EndOfFile}, })); // Check different kinds of identifier character sequences. buffer = Lex("_foo_bar"); EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {.kind = TokenKind::Identifier(), .text = "_foo_bar"}, - {TokenKind::EndOfFile()}, + {.kind = TokenKind::Identifier, .text = "_foo_bar"}, + {TokenKind::EndOfFile}, })); buffer = Lex("foo2bar00"); EXPECT_FALSE(buffer.has_errors()); - EXPECT_THAT(buffer, - HasTokens(llvm::ArrayRef{ - {.kind = TokenKind::Identifier(), .text = "foo2bar00"}, - {TokenKind::EndOfFile()}, - })); + EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ + {.kind = TokenKind::Identifier, .text = "foo2bar00"}, + {TokenKind::EndOfFile}, + })); // Check that we can parse identifiers that start with a keyword. buffer = Lex("fnord"); EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {.kind = TokenKind::Identifier(), .text = "fnord"}, - {TokenKind::EndOfFile()}, + {.kind = TokenKind::Identifier, .text = "fnord"}, + {TokenKind::EndOfFile}, })); // Check multiple identifiers with indent and interning. @@ -687,33 +659,33 @@ TEST_F(LexerTest, Identifiers) { EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {.kind = TokenKind::Identifier(), + {.kind = TokenKind::Identifier, .line = 1, .column = 4, .indent_column = 4, .text = "foo"}, - {.kind = TokenKind::Semi()}, - {.kind = TokenKind::Identifier(), + {.kind = TokenKind::Semi}, + {.kind = TokenKind::Identifier, .line = 1, .column = 8, .indent_column = 4, .text = "bar"}, - {.kind = TokenKind::Identifier(), + {.kind = TokenKind::Identifier, .line = 2, .column = 1, .indent_column = 1, .text = "bar"}, - {.kind = TokenKind::Identifier(), + {.kind = TokenKind::Identifier, .line = 3, .column = 3, .indent_column = 3, .text = "foo"}, - {.kind = TokenKind::Identifier(), + {.kind = TokenKind::Identifier, .line = 3, .column = 7, .indent_column = 3, .text = "foo"}, - {.kind = TokenKind::EndOfFile(), .line = 3, .column = 10}, + {.kind = TokenKind::EndOfFile, .line = 3, .column = 10}, })); } @@ -739,32 +711,32 @@ TEST_F(LexerTest, StringLiterals) { EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {.kind = TokenKind::StringLiteral(), + {.kind = TokenKind::StringLiteral, .line = 2, .column = 5, .indent_column = 5, .string_contents = {"hello world\n"}}, - {.kind = TokenKind::StringLiteral(), + {.kind = TokenKind::StringLiteral, .line = 4, .column = 5, .indent_column = 5, .string_contents = {" test \xAB\n"}}, - {.kind = TokenKind::Identifier(), + {.kind = TokenKind::Identifier, .line = 7, .column = 10, .indent_column = 5, .text = "trailing"}, - {.kind = TokenKind::StringLiteral(), + {.kind = TokenKind::StringLiteral, .line = 9, .column = 7, .indent_column = 7, .string_contents = {"\""}}, - {.kind = TokenKind::StringLiteral(), + {.kind = TokenKind::StringLiteral, .line = 11, .column = 5, .indent_column = 5, .string_contents = llvm::StringLiteral::withInnerNUL("\0")}, - {.kind = TokenKind::StringLiteral(), + {.kind = TokenKind::StringLiteral, .line = 13, .column = 5, .indent_column = 5, @@ -772,22 +744,22 @@ TEST_F(LexerTest, StringLiterals) { // """x""" is three string literals, not one invalid // attempt at a block string literal. - {.kind = TokenKind::StringLiteral(), + {.kind = TokenKind::StringLiteral, .line = 15, .column = 5, .indent_column = 5, .string_contents = {""}}, - {.kind = TokenKind::StringLiteral(), + {.kind = TokenKind::StringLiteral, .line = 15, .column = 7, .indent_column = 5, .string_contents = {"x"}}, - {.kind = TokenKind::StringLiteral(), + {.kind = TokenKind::StringLiteral, .line = 15, .column = 10, .indent_column = 5, .string_contents = {""}}, - {.kind = TokenKind::EndOfFile(), .line = 16, .column = 3}, + {.kind = TokenKind::EndOfFile, .line = 16, .column = 3}, })); } @@ -818,7 +790,7 @@ TEST_F(LexerTest, InvalidStringLiterals) { // We should have formed at least one error token. bool found_error = false; for (TokenizedBuffer::Token token : buffer.tokens()) { - if (buffer.GetKind(token) == TokenKind::Error()) { + if (buffer.GetKind(token) == TokenKind::Error) { found_error = true; break; } @@ -839,81 +811,81 @@ TEST_F(LexerTest, TypeLiterals) { EXPECT_FALSE(buffer.has_errors()); ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef{ - {.kind = TokenKind::Identifier(), + {.kind = TokenKind::Identifier, .line = 2, .column = 5, .indent_column = 5, .text = {"i0"}}, - {.kind = TokenKind::IntegerTypeLiteral(), + {.kind = TokenKind::IntegerTypeLiteral, .line = 2, .column = 8, .indent_column = 5, .text = {"i1"}}, - {.kind = TokenKind::IntegerTypeLiteral(), + {.kind = TokenKind::IntegerTypeLiteral, .line = 2, .column = 11, .indent_column = 5, .text = {"i20"}}, - {.kind = TokenKind::IntegerTypeLiteral(), + {.kind = TokenKind::IntegerTypeLiteral, .line = 2, .column = 15, .indent_column = 5, .text = {"i999999999999"}}, - {.kind = TokenKind::Identifier(), + {.kind = TokenKind::Identifier, .line = 2, .column = 29, .indent_column = 5, .text = {"i0x1"}}, - {.kind = TokenKind::Identifier(), + {.kind = TokenKind::Identifier, .line = 3, .column = 5, .indent_column = 5, .text = {"u0"}}, - {.kind = TokenKind::UnsignedIntegerTypeLiteral(), + {.kind = TokenKind::UnsignedIntegerTypeLiteral, .line = 3, .column = 8, .indent_column = 5, .text = {"u1"}}, - {.kind = TokenKind::UnsignedIntegerTypeLiteral(), + {.kind = TokenKind::UnsignedIntegerTypeLiteral, .line = 3, .column = 11, .indent_column = 5, .text = {"u64"}}, - {.kind = TokenKind::Identifier(), + {.kind = TokenKind::Identifier, .line = 3, .column = 15, .indent_column = 5, .text = {"u64b"}}, - {.kind = TokenKind::FloatingPointTypeLiteral(), + {.kind = TokenKind::FloatingPointTypeLiteral, .line = 4, .column = 5, .indent_column = 5, .text = {"f32"}}, - {.kind = TokenKind::FloatingPointTypeLiteral(), + {.kind = TokenKind::FloatingPointTypeLiteral, .line = 4, .column = 9, .indent_column = 5, .text = {"f80"}}, - {.kind = TokenKind::FloatingPointTypeLiteral(), + {.kind = TokenKind::FloatingPointTypeLiteral, .line = 4, .column = 13, .indent_column = 5, .text = {"f1"}}, - {.kind = TokenKind::Identifier(), + {.kind = TokenKind::Identifier, .line = 4, .column = 16, .indent_column = 5, .text = {"fi"}}, - {.kind = TokenKind::Identifier(), + {.kind = TokenKind::Identifier, .line = 5, .column = 5, .indent_column = 5, .text = {"s1"}}, - {.kind = TokenKind::EndOfFile(), .line = 6, .column = 3}, + {.kind = TokenKind::EndOfFile, .line = 6, .column = 3}, })); auto token_i1 = buffer.tokens().begin() + 1; @@ -949,12 +921,12 @@ TEST_F(LexerTest, TypeLiteralTooManyDigits) { ASSERT_THAT( buffer, HasTokens(llvm::ArrayRef{ - {.kind = TokenKind::Error(), + {.kind = TokenKind::Error, .line = 1, .column = 1, .indent_column = 1, .text = {code}}, - {.kind = TokenKind::EndOfFile(), .line = 1, .column = Count + 2}, + {.kind = TokenKind::EndOfFile, .line = 1, .column = Count + 2}, })); } diff --git a/toolchain/lexer/tokenized_buffer_test_helpers.h b/toolchain/lexer/tokenized_buffer_test_helpers.h index 026afb759853..c4a570c77ecb 100644 --- a/toolchain/lexer/tokenized_buffer_test_helpers.h +++ b/toolchain/lexer/tokenized_buffer_test_helpers.h @@ -132,8 +132,8 @@ MATCHER_P(HasTokens, raw_all_expected, "") { } CARBON_CHECK(!expected.string_contents || - expected.kind == TokenKind::StringLiteral()); - if (expected.string_contents && actual_kind == TokenKind::StringLiteral()) { + expected.kind == TokenKind::StringLiteral); + if (expected.string_contents && actual_kind == TokenKind::StringLiteral) { llvm::StringRef actual_contents = buffer.GetStringLiteral(token); if (actual_contents != *expected.string_contents) { *result_listener << "\nToken " << index << " has contents `" diff --git a/toolchain/parser/parse_node_kind.h b/toolchain/parser/parse_node_kind.h index e3841f0ea610..a5537b3a33bb 100644 --- a/toolchain/parser/parse_node_kind.h +++ b/toolchain/parser/parse_node_kind.h @@ -9,7 +9,7 @@ namespace Carbon { -CARBON_DEFINE_RAW_ENUM_CLASS(ParseNodeKind, uint8_t){ +CARBON_DEFINE_RAW_ENUM_CLASS(ParseNodeKind, uint8_t) { #define CARBON_PARSE_NODE_KIND(Name) CARBON_RAW_ENUM_ENUMERATOR(Name) #include "toolchain/parser/parse_node_kind.def" }; diff --git a/toolchain/parser/parser.cpp b/toolchain/parser/parser.cpp index b6e23249211c..37229dbca5cf 100644 --- a/toolchain/parser/parser.cpp +++ b/toolchain/parser/parser.cpp @@ -95,7 +95,7 @@ Parser::Parser(ParseTree& tree, TokenizedBuffer& tokens, stack_context_(ParseContext::File) { CARBON_CHECK(position_ != end_) << "Empty TokenizedBuffer"; --end_; - CARBON_CHECK(tokens_->GetKind(*end_) == TokenKind::EndOfFile()) + CARBON_CHECK(tokens_->GetKind(*end_) == TokenKind::EndOfFile) << "TokenizedBuffer should end with EndOfFile, ended with " << tokens_->GetKind(*end_).name(); } @@ -121,7 +121,7 @@ auto Parser::AddNode(ParseNodeKind kind, TokenizedBuffer::Token token, auto Parser::ConsumeAndAddOpenParen(TokenizedBuffer::Token default_token, ParseNodeKind start_kind) -> void { - if (auto open_paren = ConsumeIf(TokenKind::OpenParen())) { + if (auto open_paren = ConsumeIf(TokenKind::OpenParen)) { AddLeafNode(start_kind, *open_paren, /*has_error=*/false); } else { emitter_->Emit(*position_, ExpectedParenAfter, @@ -136,9 +136,9 @@ auto Parser::ConsumeAndAddCloseParen(StateStackEntry state, // introducer. auto expected_paren = *(TokenizedBuffer::TokenIterator(state.token) + 1); - if (tokens_->GetKind(expected_paren) != TokenKind::OpenParen()) { + if (tokens_->GetKind(expected_paren) != TokenKind::OpenParen) { AddNode(close_kind, state.token, state.subtree_start, /*has_error=*/true); - } else if (auto close_token = ConsumeIf(TokenKind::CloseParen())) { + } else if (auto close_token = ConsumeIf(TokenKind::CloseParen)) { AddNode(close_kind, *close_token, state.subtree_start, state.has_error); } else { // TODO: Include the location of the matching open_paren in the diagnostic. @@ -187,7 +187,7 @@ auto Parser::FindNextOf(std::initializer_list desired_kinds) } // Step to the next token at the current bracketing level. - if (kind.is_closing_symbol() || kind == TokenKind::EndOfFile()) { + if (kind.is_closing_symbol() || kind == TokenKind::EndOfFile) { // There are no more tokens at this level. return std::nullopt; } else if (kind.is_opening_symbol()) { @@ -233,7 +233,7 @@ auto Parser::SkipPastLikelyEnd(TokenizedBuffer::Token skip_root) }; do { - if (PositionIs(TokenKind::CloseCurlyBrace())) { + if (PositionIs(TokenKind::CloseCurlyBrace)) { // Immediately bail out if we hit an unmatched close curly, this will // pop us up a level of the syntax grouping. return std::nullopt; @@ -241,7 +241,7 @@ auto Parser::SkipPastLikelyEnd(TokenizedBuffer::Token skip_root) // We assume that a semicolon is always intended to be the end of the // current construct. - if (auto semi = ConsumeIf(TokenKind::Semi())) { + if (auto semi = ConsumeIf(TokenKind::Semi)) { return semi; } @@ -269,7 +269,7 @@ auto Parser::HandleCodeBlockState() -> void { PopAndDiscardState(); PushState(ParserState::CodeBlockFinish); - if (ConsumeAndAddLeafNodeIf(TokenKind::OpenCurlyBrace(), + if (ConsumeAndAddLeafNodeIf(TokenKind::OpenCurlyBrace, ParseNodeKind::CodeBlockStart)) { PushState(ParserState::StatementScopeLoop); } else { @@ -287,27 +287,27 @@ auto Parser::HandleCodeBlockState() -> void { // Determines whether the given token is considered to be the start of an // operand according to the rules for infix operator parsing. static auto IsAssumedStartOfOperand(TokenKind kind) -> bool { - return kind.IsOneOf({TokenKind::OpenParen(), TokenKind::Identifier(), - TokenKind::IntegerLiteral(), TokenKind::RealLiteral(), - TokenKind::StringLiteral()}); + return kind.IsOneOf({TokenKind::OpenParen, TokenKind::Identifier, + TokenKind::IntegerLiteral, TokenKind::RealLiteral, + TokenKind::StringLiteral}); } // Determines whether the given token is considered to be the end of an // operand according to the rules for infix operator parsing. static auto IsAssumedEndOfOperand(TokenKind kind) -> bool { - return kind.IsOneOf({TokenKind::CloseParen(), TokenKind::CloseCurlyBrace(), - TokenKind::CloseSquareBracket(), TokenKind::Identifier(), - TokenKind::IntegerLiteral(), TokenKind::RealLiteral(), - TokenKind::StringLiteral()}); + return kind.IsOneOf({TokenKind::CloseParen, TokenKind::CloseCurlyBrace, + TokenKind::CloseSquareBracket, TokenKind::Identifier, + TokenKind::IntegerLiteral, TokenKind::RealLiteral, + TokenKind::StringLiteral}); } // Determines whether the given token could possibly be the start of an // operand. This is conservatively correct, and will never incorrectly return // `false`, but can incorrectly return `true`. static auto IsPossibleStartOfOperand(TokenKind kind) -> bool { - return !kind.IsOneOf({TokenKind::CloseParen(), TokenKind::CloseCurlyBrace(), - TokenKind::CloseSquareBracket(), TokenKind::Comma(), - TokenKind::Semi(), TokenKind::Colon()}); + return !kind.IsOneOf({TokenKind::CloseParen, TokenKind::CloseCurlyBrace, + TokenKind::CloseSquareBracket, TokenKind::Comma, + TokenKind::Semi, TokenKind::Colon}); } auto Parser::IsLexicallyValidInfixOperator() -> bool { @@ -405,7 +405,7 @@ auto Parser::DiagnoseOperatorFixity(OperatorFixity fixity) -> void { auto Parser::ConsumeListToken(ParseNodeKind comma_kind, TokenKind close_kind, bool already_has_error) -> ListTokenKind { - if (!PositionIs(TokenKind::Comma()) && !PositionIs(close_kind)) { + if (!PositionIs(TokenKind::Comma) && !PositionIs(close_kind)) { // Don't error a second time on the same element. if (!already_has_error) { CARBON_DIAGNOSTIC(UnexpectedTokenAfterListElement, Error, @@ -415,7 +415,7 @@ auto Parser::ConsumeListToken(ParseNodeKind comma_kind, TokenKind close_kind, } // Recover from the invalid token. - auto end_of_element = FindNextOf({TokenKind::Comma(), close_kind}); + auto end_of_element = FindNextOf({TokenKind::Comma, close_kind}); // The lexer guarantees that parentheses are balanced. CARBON_CHECK(end_of_element) << "missing matching `" << close_kind.opening_symbol() << "` for `" @@ -458,9 +458,9 @@ auto Parser::HandleBraceExpressionState() -> void { PushState(state); CARBON_CHECK(ConsumeAndAddLeafNodeIf( - TokenKind::OpenCurlyBrace(), + TokenKind::OpenCurlyBrace, ParseNodeKind::StructLiteralOrStructTypeLiteralStart)); - if (!PositionIs(TokenKind::CloseCurlyBrace())) { + if (!PositionIs(TokenKind::CloseCurlyBrace)) { PushState(ParserState::BraceExpressionParameterAsUnknown); } } @@ -506,7 +506,7 @@ auto Parser::HandleBraceExpressionParameterError(StateStackEntry state, auto Parser::HandleBraceExpressionParameter(BraceExpressionKind kind) -> void { auto state = PopState(); - if (!PositionIs(TokenKind::Period())) { + if (!PositionIs(TokenKind::Period)) { HandleBraceExpressionParameterError(state, kind); return; } @@ -536,10 +536,9 @@ auto Parser::HandleBraceExpressionParameterAfterDesignator( auto state = PopState(); if (state.has_error) { - auto recovery_pos = FindNextOf( - {TokenKind::Equal(), TokenKind::Colon(), TokenKind::Comma()}); - if (!recovery_pos || - tokens_->GetKind(*recovery_pos) == TokenKind::Comma()) { + auto recovery_pos = + FindNextOf({TokenKind::Equal, TokenKind::Colon, TokenKind::Comma}); + if (!recovery_pos || tokens_->GetKind(*recovery_pos) == TokenKind::Comma) { state.state = BraceExpressionKindToParserState( kind, ParserState::BraceExpressionParameterFinishAsType, ParserState::BraceExpressionParameterFinishAsValue, @@ -552,9 +551,9 @@ auto Parser::HandleBraceExpressionParameterAfterDesignator( // Work out the kind of this element. auto elem_kind = BraceExpressionKind::Unknown; - if (PositionIs(TokenKind::Colon())) { + if (PositionIs(TokenKind::Colon)) { elem_kind = BraceExpressionKind::Type; - } else if (PositionIs(TokenKind::Equal())) { + } else if (PositionIs(TokenKind::Equal)) { elem_kind = BraceExpressionKind::Value; } // Unknown kinds and changes between type and value are errors. @@ -618,7 +617,7 @@ auto Parser::HandleBraceExpressionParameterFinish(BraceExpressionKind kind) state.token, state.subtree_start, /*has_error=*/false); } - if (ConsumeListToken(ParseNodeKind::StructComma, TokenKind::CloseCurlyBrace(), + if (ConsumeListToken(ParseNodeKind::StructComma, TokenKind::CloseCurlyBrace, state.has_error) == ListTokenKind::Comma) { PushState(BraceExpressionKindToParserState( kind, ParserState::BraceExpressionParameterAsType, @@ -667,7 +666,7 @@ auto Parser::HandleCallExpressionState() -> void { AddNode(ParseNodeKind::CallExpressionStart, Consume(), state.subtree_start, state.has_error); - if (!PositionIs(TokenKind::CloseParen())) { + if (!PositionIs(TokenKind::CloseParen)) { PushState(ParserState::CallExpressionParameterFinish); PushState(ParserState::Expression); } @@ -681,7 +680,7 @@ auto Parser::HandleCallExpressionParameterFinishState() -> void { } if (ConsumeListToken(ParseNodeKind::CallExpressionComma, - TokenKind::CloseParen(), + TokenKind::CloseParen, state.has_error) == ListTokenKind::Comma) { PushState(ParserState::CallExpressionParameterFinish); PushState(ParserState::Expression); @@ -699,7 +698,7 @@ auto Parser::HandleCodeBlockFinishState() -> void { auto state = PopState(); // If the block started with an open curly, this is a close curly. - if (tokens_->GetKind(state.token) == TokenKind::OpenCurlyBrace()) { + if (tokens_->GetKind(state.token) == TokenKind::OpenCurlyBrace) { AddNode(ParseNodeKind::CodeBlock, Consume(), state.subtree_start, state.has_error); } else { @@ -712,28 +711,28 @@ auto Parser::HandleDeclarationLoopState() -> void { // This maintains the current state unless we're at the end of the file. switch (PositionKind()) { - case TokenKind::EndOfFile(): { + case TokenKind::EndOfFile: { PopAndDiscardState(); break; } - case TokenKind::Fn(): { + case TokenKind::Fn: { PushState(ParserState::FunctionIntroducer); AddLeafNode(ParseNodeKind::FunctionIntroducer, Consume()); break; } - case TokenKind::Package(): { + case TokenKind::Package: { PushState(ParserState::Package); break; } - case TokenKind::Semi(): { + case TokenKind::Semi: { AddLeafNode(ParseNodeKind::EmptyDeclaration, Consume()); break; } - case TokenKind::Var(): { + case TokenKind::Var: { PushState(ParserState::VarAsSemicolon); break; } - case TokenKind::Interface(): { + case TokenKind::Interface: { PushState(ParserState::InterfaceIntroducer); ++position_; break; @@ -754,7 +753,7 @@ auto Parser::HandleDeclarationLoopState() -> void { auto Parser::HandleDeducedParameterListFinishState() -> void { auto state = PopState(); - CARBON_CHECK(tokens_->GetKind(*position_) == TokenKind::CloseSquareBracket()) + CARBON_CHECK(tokens_->GetKind(*position_) == TokenKind::CloseSquareBracket) << "Expected current token to be: `]`, found: " << tokens_->GetKind(state.token); AddNode(ParseNodeKind::DeducedParameterList, Consume(), state.subtree_start, @@ -765,9 +764,9 @@ auto Parser::HandleDesignator(bool as_struct) -> void { auto state = PopState(); // `.` identifier - auto dot = ConsumeChecked(TokenKind::Period()); + auto dot = ConsumeChecked(TokenKind::Period); - if (!ConsumeAndAddLeafNodeIf(TokenKind::Identifier(), + if (!ConsumeAndAddLeafNodeIf(TokenKind::Identifier, ParseNodeKind::DesignatedName)) { CARBON_DIAGNOSTIC(ExpectedIdentifierAfterDot, Error, "Expected identifier after `.`."); @@ -837,32 +836,32 @@ auto Parser::HandleExpressionInPostfixState() -> void { // expression tree, such as an identifier or literal, or a parenthesized // expression. switch (PositionKind()) { - case TokenKind::Identifier(): { + case TokenKind::Identifier: { AddLeafNode(ParseNodeKind::NameReference, Consume()); PushState(state); break; } - case TokenKind::IntegerLiteral(): - case TokenKind::RealLiteral(): - case TokenKind::StringLiteral(): - case TokenKind::IntegerTypeLiteral(): - case TokenKind::UnsignedIntegerTypeLiteral(): - case TokenKind::FloatingPointTypeLiteral(): { + case TokenKind::IntegerLiteral: + case TokenKind::RealLiteral: + case TokenKind::StringLiteral: + case TokenKind::IntegerTypeLiteral: + case TokenKind::UnsignedIntegerTypeLiteral: + case TokenKind::FloatingPointTypeLiteral: { AddLeafNode(ParseNodeKind::Literal, Consume()); PushState(state); break; } - case TokenKind::OpenCurlyBrace(): { + case TokenKind::OpenCurlyBrace: { PushState(state); PushState(ParserState::BraceExpression); break; } - case TokenKind::OpenParen(): { + case TokenKind::OpenParen: { PushState(state); PushState(ParserState::ParenExpression); break; } - case TokenKind::SelfType(): { + case TokenKind::SelfType: { AddLeafNode(ParseNodeKind::SelfType, Consume()); PushState(state); break; @@ -884,13 +883,13 @@ auto Parser::HandleExpressionInPostfixLoopState() -> void { state.token = *position_; switch (PositionKind()) { - case TokenKind::Period(): { + case TokenKind::Period: { PushState(state); state.state = ParserState::DesignatorAsExpression; PushState(state); break; } - case TokenKind::OpenParen(): { + case TokenKind::OpenParen: { PushState(state); state.state = ParserState::CallExpression; PushState(state); @@ -982,7 +981,7 @@ auto Parser::HandleExpressionLoopForPrefixState() -> void { auto Parser::HandleExpressionStatementFinishState() -> void { auto state = PopState(); - if (auto semi = ConsumeIf(TokenKind::Semi())) { + if (auto semi = ConsumeIf(TokenKind::Semi)) { AddNode(ParseNodeKind::ExpressionStatement, *semi, state.subtree_start, state.has_error); return; @@ -1018,7 +1017,7 @@ auto Parser::HandleFunctionError(StateStackEntry state, auto Parser::HandleFunctionIntroducerState() -> void { auto state = PopState(); - if (!ConsumeAndAddLeafNodeIf(TokenKind::Identifier(), + if (!ConsumeAndAddLeafNodeIf(TokenKind::Identifier, ParseNodeKind::DeclaredName)) { CARBON_DIAGNOSTIC(ExpectedFunctionName, Error, "Expected function name after `fn` keyword."); @@ -1034,12 +1033,12 @@ auto Parser::HandleFunctionIntroducerState() -> void { PushState(state); // If there are deduced params handle them next. - if (PositionIs(TokenKind::OpenSquareBracket())) { + if (PositionIs(TokenKind::OpenSquareBracket)) { PushState(ParserState::DeducedParameterListFinish); // This is for sure a `[`, we can safely create the corresponding node. AddLeafNode(ParseNodeKind::DeducedParameterListStart, Consume()); - if (PositionIs(TokenKind::CloseSquareBracket())) { + if (PositionIs(TokenKind::CloseSquareBracket)) { return; } @@ -1055,7 +1054,7 @@ auto Parser::HandleFunctionIntroducerState() -> void { auto Parser::HandleFunctionAfterDeducedParameterListState() -> void { auto state = PopState(); - if (!PositionIs(TokenKind::OpenParen())) { + if (!PositionIs(TokenKind::OpenParen)) { CARBON_DIAGNOSTIC(ExpectedFunctionParams, Error, "Expected `(` after function name."); emitter_->Emit(*position_, ExpectedFunctionParams); @@ -1070,7 +1069,7 @@ auto Parser::HandleFunctionAfterDeducedParameterListState() -> void { PushState(ParserState::FunctionParameterListFinish); AddLeafNode(ParseNodeKind::ParameterListStart, Consume()); - if (!PositionIs(TokenKind::CloseParen())) { + if (!PositionIs(TokenKind::CloseParen)) { PushState(ParserState::FunctionParameter); } } @@ -1089,8 +1088,7 @@ auto Parser::HandleFunctionParameterFinishState() -> void { ReturnErrorOnState(); } - if (ConsumeListToken(ParseNodeKind::ParameterListComma, - TokenKind::CloseParen(), + if (ConsumeListToken(ParseNodeKind::ParameterListComma, TokenKind::CloseParen, state.has_error) == ListTokenKind::Comma) { PushState(ParserState::PatternAsFunctionParameter); } @@ -1099,7 +1097,7 @@ auto Parser::HandleFunctionParameterFinishState() -> void { auto Parser::HandleFunctionParameterListFinishState() -> void { auto state = PopState(); - AddNode(ParseNodeKind::ParameterList, ConsumeChecked(TokenKind::CloseParen()), + AddNode(ParseNodeKind::ParameterList, ConsumeChecked(TokenKind::CloseParen), state.subtree_start, state.has_error); } @@ -1112,7 +1110,7 @@ auto Parser::HandleFunctionAfterParameterListState() -> void { // If there is a return type, parse the expression before adding the return // type nod.e - if (PositionIs(TokenKind::MinusGreater())) { + if (PositionIs(TokenKind::MinusGreater)) { PushState(ParserState::FunctionReturnTypeFinish); ++position_; PushStateForExpression(PrecedenceGroup::ForType()); @@ -1130,12 +1128,12 @@ auto Parser::HandleFunctionSignatureFinishState() -> void { auto state = PopState(); switch (PositionKind()) { - case TokenKind::Semi(): { + case TokenKind::Semi: { AddNode(ParseNodeKind::FunctionDeclaration, Consume(), state.subtree_start, state.has_error); break; } - case TokenKind::OpenCurlyBrace(): { + case TokenKind::OpenCurlyBrace: { if (stack_context_ == ParseContext::Interface) { CARBON_DIAGNOSTIC( MethodImplNotAllowed, Error, @@ -1180,7 +1178,7 @@ auto Parser::HandleInterfaceIntroducerState() -> void { << "TODO: Support nesting."; stack_context_ = ParseContext::Interface; - if (!ConsumeAndAddLeafNodeIf(TokenKind::Identifier(), + if (!ConsumeAndAddLeafNodeIf(TokenKind::Identifier, ParseNodeKind::DeclaredName)) { CARBON_DIAGNOSTIC(ExpectedInterfaceName, Error, "Expected interface name after `interface` keyword."); @@ -1196,7 +1194,7 @@ auto Parser::HandleInterfaceIntroducerState() -> void { bool parse_body = true; - if (!PositionIs(TokenKind::OpenCurlyBrace())) { + if (!PositionIs(TokenKind::OpenCurlyBrace)) { CARBON_DIAGNOSTIC(ExpectedInterfaceOpenCurlyBrace, Error, "Expected `{{` to start interface definition."); emitter_->Emit(*position_, ExpectedInterfaceOpenCurlyBrace); @@ -1220,7 +1218,7 @@ auto Parser::HandleInterfaceDefinitionLoopState() -> void { // definition. switch (PositionKind()) { - case TokenKind::CloseCurlyBrace(): { + case TokenKind::CloseCurlyBrace: { auto state = PopState(); AddNode(ParseNodeKind::InterfaceBody, Consume(), state.subtree_start, @@ -1228,7 +1226,7 @@ auto Parser::HandleInterfaceDefinitionLoopState() -> void { break; } - case TokenKind::Fn(): { + case TokenKind::Fn: { PushState(ParserState::FunctionIntroducer); AddLeafNode(ParseNodeKind::FunctionIntroducer, Consume()); break; @@ -1265,7 +1263,7 @@ auto Parser::HandlePackageState() -> void { /*has_error=*/true); }; - if (!ConsumeAndAddLeafNodeIf(TokenKind::Identifier(), + if (!ConsumeAndAddLeafNodeIf(TokenKind::Identifier, ParseNodeKind::DeclaredName)) { CARBON_DIAGNOSTIC(ExpectedIdentifierAfterPackage, Error, "Expected identifier after `package`."); @@ -1275,10 +1273,10 @@ auto Parser::HandlePackageState() -> void { } bool library_parsed = false; - if (auto library_token = ConsumeIf(TokenKind::Library())) { + if (auto library_token = ConsumeIf(TokenKind::Library)) { auto library_start = tree_->size(); - if (!ConsumeAndAddLeafNodeIf(TokenKind::StringLiteral(), + if (!ConsumeAndAddLeafNodeIf(TokenKind::StringLiteral, ParseNodeKind::Literal)) { CARBON_DIAGNOSTIC( ExpectedLibraryName, Error, @@ -1294,16 +1292,16 @@ auto Parser::HandlePackageState() -> void { } switch (auto api_or_impl_token = tokens_->GetKind(*(position_))) { - case TokenKind::Api(): { + case TokenKind::Api: { AddLeafNode(ParseNodeKind::PackageApi, Consume()); break; } - case TokenKind::Impl(): { + case TokenKind::Impl: { AddLeafNode(ParseNodeKind::PackageImpl, Consume()); break; } default: { - if (!library_parsed && api_or_impl_token == TokenKind::StringLiteral()) { + if (!library_parsed && api_or_impl_token == TokenKind::StringLiteral) { // If we come acroess a string literal and we didn't parse `library // "..."` yet, then most probably the user forgot to add `library` // before the library name. @@ -1320,7 +1318,7 @@ auto Parser::HandlePackageState() -> void { } } - if (!PositionIs(TokenKind::Semi())) { + if (!PositionIs(TokenKind::Semi)) { CARBON_DIAGNOSTIC(ExpectedSemiToEndPackageDirective, Error, "Expected `;` to end package directive."); emitter_->Emit(*position_, ExpectedSemiToEndPackageDirective); @@ -1370,9 +1368,9 @@ auto Parser::HandleParenExpressionState() -> void { // Advance past the open paren. AddLeafNode(ParseNodeKind::ParenExpressionOrTupleLiteralStart, - ConsumeChecked(TokenKind::OpenParen())); + ConsumeChecked(TokenKind::OpenParen)); - if (PositionIs(TokenKind::CloseParen())) { + if (PositionIs(TokenKind::CloseParen)) { state.state = ParserState::ParenExpressionFinishAsTuple; PushState(state); } else { @@ -1386,9 +1384,8 @@ auto Parser::HandleParenExpressionState() -> void { auto Parser::HandleParenExpressionParameterFinish(bool as_tuple) -> void { auto state = PopState(); - auto list_token_kind = - ConsumeListToken(ParseNodeKind::TupleLiteralComma, - TokenKind::CloseParen(), state.has_error); + auto list_token_kind = ConsumeListToken( + ParseNodeKind::TupleLiteralComma, TokenKind::CloseParen, state.has_error); if (list_token_kind == ListTokenKind::Close) { return; } @@ -1442,8 +1439,8 @@ auto Parser::HandlePattern(PatternKind pattern_kind) -> void { state.state = ParserState::PatternFinish; // Handle an invalid pattern introducer for parameters and variables. - if (!PositionIs(TokenKind::Identifier()) || - tokens_->GetKind(*(position_ + 1)) != TokenKind::Colon()) { + if (!PositionIs(TokenKind::Identifier) || + tokens_->GetKind(*(position_ + 1)) != TokenKind::Colon) { switch (pattern_kind) { case PatternKind::Parameter: { CARBON_DIAGNOSTIC(ExpectedParameterName, Error, @@ -1514,8 +1511,8 @@ auto Parser::HandleSelfPatternState() -> void { // self `:` type auto possible_self_param = - (PositionIs(TokenKind::SelfParameter()) && - tokens_->GetKind(*(position_ + 1)) == TokenKind::Colon()); + (PositionIs(TokenKind::SelfParameter) && + tokens_->GetKind(*(position_ + 1)) == TokenKind::Colon); if (possible_self_param) { // Ensure the finish state always follows. @@ -1533,9 +1530,9 @@ auto Parser::HandleSelfPatternState() -> void { // addr self `:` type auto possible_addr_self_param = - (PositionIs(TokenKind::Addr()) && - tokens_->GetKind(*(position_ + 1)) == TokenKind::SelfParameter() && - tokens_->GetKind(*(position_ + 2)) == TokenKind::Colon()); + (PositionIs(TokenKind::Addr) && + tokens_->GetKind(*(position_ + 1)) == TokenKind::SelfParameter && + tokens_->GetKind(*(position_ + 2)) == TokenKind::Colon); if (possible_addr_self_param) { // Ensure the finish state always follows. @@ -1560,7 +1557,7 @@ auto Parser::HandleSelfPatternState() -> void { // Try to recover by skipping to the next `]`. if (auto next_close_square_bracket = - FindNextOf({TokenKind::CloseSquareBracket()}); + FindNextOf({TokenKind::CloseSquareBracket}); next_close_square_bracket) { SkipTo(*next_close_square_bracket); } @@ -1572,35 +1569,35 @@ auto Parser::HandleStatementState() -> void { PopAndDiscardState(); switch (PositionKind()) { - case TokenKind::Break(): { + case TokenKind::Break: { PushState(ParserState::StatementBreakFinish); AddLeafNode(ParseNodeKind::BreakStatementStart, Consume()); break; } - case TokenKind::Continue(): { + case TokenKind::Continue: { PushState(ParserState::StatementContinueFinish); AddLeafNode(ParseNodeKind::ContinueStatementStart, Consume()); break; } - case TokenKind::For(): { + case TokenKind::For: { PushState(ParserState::StatementForFinish); PushState(ParserState::StatementForHeader); ++position_; break; } - case TokenKind::If(): { + case TokenKind::If: { PushState(ParserState::StatementIf); break; } - case TokenKind::Return(): { + case TokenKind::Return: { PushState(ParserState::StatementReturn); break; } - case TokenKind::Var(): { + case TokenKind::Var: { PushState(ParserState::VarAsSemicolon); break; } - case TokenKind::While(): { + case TokenKind::While: { PushState(ParserState::StatementWhile); break; } @@ -1627,7 +1624,7 @@ auto Parser::HandleStatementForHeaderState() -> void { state.state = ParserState::StatementForHeaderIn; - if (PositionIs(TokenKind::Var())) { + if (PositionIs(TokenKind::Var)) { PushState(state); PushState(ParserState::VarAsFor); } else { @@ -1635,7 +1632,7 @@ auto Parser::HandleStatementForHeaderState() -> void { "Expected `var` declaration."); emitter_->Emit(*position_, ExpectedVariableDeclaration); - if (auto next_in = FindNextOf({TokenKind::In()})) { + if (auto next_in = FindNextOf({TokenKind::In})) { SkipTo(*next_in); ++position_; } @@ -1686,13 +1683,13 @@ auto Parser::HandleStatementIfConditionFinishState() -> void { auto Parser::HandleStatementIfThenBlockFinishState() -> void { auto state = PopState(); - if (ConsumeAndAddLeafNodeIf(TokenKind::Else(), + if (ConsumeAndAddLeafNodeIf(TokenKind::Else, ParseNodeKind::IfStatementElse)) { state.state = ParserState::StatementIfElseBlockFinish; PushState(state); // `else if` is permitted as a special case. - PushState(PositionIs(TokenKind::If()) ? ParserState::StatementIf - : ParserState::CodeBlock); + PushState(PositionIs(TokenKind::If) ? ParserState::StatementIf + : ParserState::CodeBlock); } else { AddNode(ParseNodeKind::IfStatement, state.token, state.subtree_start, state.has_error); @@ -1708,7 +1705,7 @@ auto Parser::HandleStatementIfElseBlockFinishState() -> void { auto Parser::HandleStatementKeywordFinish(ParseNodeKind node_kind) -> void { auto state = PopState(); - auto semi = ConsumeIf(TokenKind::Semi()); + auto semi = ConsumeIf(TokenKind::Semi); if (!semi) { CARBON_DIAGNOSTIC(ExpectedSemiAfter, Error, "Expected `;` after `{0}`.", TokenKind); @@ -1731,7 +1728,7 @@ auto Parser::HandleStatementReturnState() -> void { PushState(state); AddLeafNode(ParseNodeKind::ReturnStatementStart, Consume()); - if (!PositionIs(TokenKind::Semi())) { + if (!PositionIs(TokenKind::Semi)) { PushState(ParserState::Expression); } } @@ -1744,7 +1741,7 @@ auto Parser::HandleStatementScopeLoopState() -> void { // This maintains the current state until we're at the end of the scope. auto token_kind = PositionKind(); - if (token_kind == TokenKind::CloseCurlyBrace()) { + if (token_kind == TokenKind::CloseCurlyBrace) { auto state = PopState(); if (state.has_error) { ReturnErrorOnState(); @@ -1802,13 +1799,12 @@ auto Parser::HandleVarAfterPatternState() -> void { auto state = PopState(); if (state.has_error) { - if (auto after_pattern = - FindNextOf({TokenKind::Equal(), TokenKind::Semi()})) { + if (auto after_pattern = FindNextOf({TokenKind::Equal, TokenKind::Semi})) { SkipTo(*after_pattern); } } - if (auto equals = ConsumeIf(TokenKind::Equal())) { + if (auto equals = ConsumeIf(TokenKind::Equal)) { AddLeafNode(ParseNodeKind::VariableInitializer, *equals); PushState(ParserState::Expression); } @@ -1818,7 +1814,7 @@ auto Parser::HandleVarFinishAsSemicolonState() -> void { auto state = PopState(); auto end_token = state.token; - if (PositionIs(TokenKind::Semi())) { + if (PositionIs(TokenKind::Semi)) { end_token = Consume(); } else { emitter_->Emit(*position_, ExpectedSemiAfterExpression); @@ -1835,9 +1831,9 @@ auto Parser::HandleVarFinishAsForState() -> void { auto state = PopState(); auto end_token = state.token; - if (PositionIs(TokenKind::In())) { + if (PositionIs(TokenKind::In)) { end_token = Consume(); - } else if (PositionIs(TokenKind::Colon())) { + } else if (PositionIs(TokenKind::Colon)) { CARBON_DIAGNOSTIC(ExpectedInNotColon, Error, "`:` should be replaced by `in`."); emitter_->Emit(*position_, ExpectedInNotColon); diff --git a/toolchain/parser/precedence.cpp b/toolchain/parser/precedence.cpp index 71a0a4162b3c..543512b1d393 100644 --- a/toolchain/parser/precedence.cpp +++ b/toolchain/parser/precedence.cpp @@ -196,18 +196,18 @@ auto PrecedenceGroup::ForType() -> PrecedenceGroup { auto PrecedenceGroup::ForLeading(TokenKind kind) -> std::optional { switch (kind) { - case TokenKind::Star(): + case TokenKind::Star: return PrecedenceGroup(TermPrefix); - case TokenKind::Not(): + case TokenKind::Not: return PrecedenceGroup(LogicalPrefix); - case TokenKind::Minus(): - case TokenKind::MinusMinus(): - case TokenKind::PlusPlus(): + case TokenKind::Minus: + case TokenKind::MinusMinus: + case TokenKind::PlusPlus: return PrecedenceGroup(NumericPrefix); - case TokenKind::Tilde(): + case TokenKind::Tilde: return PrecedenceGroup(BitwisePrefix); default: @@ -219,92 +219,92 @@ auto PrecedenceGroup::ForTrailing(TokenKind kind, bool infix) -> std::optional { switch (kind) { // Assignment operators. - case TokenKind::Equal(): + case TokenKind::Equal: return Trailing{.level = SimpleAssignment, .is_binary = true}; - case TokenKind::PlusEqual(): - case TokenKind::MinusEqual(): - case TokenKind::StarEqual(): - case TokenKind::SlashEqual(): - case TokenKind::PercentEqual(): - case TokenKind::AmpEqual(): - case TokenKind::PipeEqual(): - case TokenKind::GreaterGreaterEqual(): - case TokenKind::LessLessEqual(): + case TokenKind::PlusEqual: + case TokenKind::MinusEqual: + case TokenKind::StarEqual: + case TokenKind::SlashEqual: + case TokenKind::PercentEqual: + case TokenKind::AmpEqual: + case TokenKind::PipeEqual: + case TokenKind::GreaterGreaterEqual: + case TokenKind::LessLessEqual: return Trailing{.level = CompoundAssignment, .is_binary = true}; // Logical operators. - case TokenKind::And(): + case TokenKind::And: return Trailing{.level = LogicalAnd, .is_binary = true}; - case TokenKind::Or(): + case TokenKind::Or: return Trailing{.level = LogicalOr, .is_binary = true}; // Bitwise operators. - case TokenKind::Amp(): + case TokenKind::Amp: return Trailing{.level = BitwiseAnd, .is_binary = true}; - case TokenKind::Pipe(): + case TokenKind::Pipe: return Trailing{.level = BitwiseOr, .is_binary = true}; - case TokenKind::Xor(): + case TokenKind::Xor: return Trailing{.level = BitwiseXor, .is_binary = true}; - case TokenKind::GreaterGreater(): - case TokenKind::LessLess(): + case TokenKind::GreaterGreater: + case TokenKind::LessLess: return Trailing{.level = BitShift, .is_binary = true}; // Relational operators. - case TokenKind::EqualEqual(): - case TokenKind::ExclaimEqual(): - case TokenKind::Less(): - case TokenKind::LessEqual(): - case TokenKind::Greater(): - case TokenKind::GreaterEqual(): - case TokenKind::LessEqualGreater(): + case TokenKind::EqualEqual: + case TokenKind::ExclaimEqual: + case TokenKind::Less: + case TokenKind::LessEqual: + case TokenKind::Greater: + case TokenKind::GreaterEqual: + case TokenKind::LessEqualGreater: return Trailing{.level = Relational, .is_binary = true}; // Additive operators. - case TokenKind::Plus(): - case TokenKind::Minus(): + case TokenKind::Plus: + case TokenKind::Minus: return Trailing{.level = Additive, .is_binary = true}; // Multiplicative operators. - case TokenKind::Slash(): + case TokenKind::Slash: return Trailing{.level = Multiplicative, .is_binary = true}; - case TokenKind::Percent(): + case TokenKind::Percent: return Trailing{.level = Modulo, .is_binary = true}; // `*` could be multiplication or pointer type formation. - case TokenKind::Star(): + case TokenKind::Star: return infix ? Trailing{.level = Multiplicative, .is_binary = true} : Trailing{.level = TypePostfix, .is_binary = false}; // Postfix operators. - case TokenKind::MinusMinus(): - case TokenKind::PlusPlus(): + case TokenKind::MinusMinus: + case TokenKind::PlusPlus: return Trailing{.level = NumericPostfix, .is_binary = false}; // Prefix-only operators. - case TokenKind::Tilde(): - case TokenKind::Not(): + case TokenKind::Tilde: + case TokenKind::Not: break; // Symbolic tokens that might be operators eventually. - case TokenKind::Backslash(): - case TokenKind::Caret(): - case TokenKind::CaretEqual(): - case TokenKind::Comma(): - case TokenKind::TildeEqual(): - case TokenKind::Exclaim(): - case TokenKind::LessGreater(): - case TokenKind::Question(): - case TokenKind::Colon(): + case TokenKind::Backslash: + case TokenKind::Caret: + case TokenKind::CaretEqual: + case TokenKind::Comma: + case TokenKind::TildeEqual: + case TokenKind::Exclaim: + case TokenKind::LessGreater: + case TokenKind::Question: + case TokenKind::Colon: break; // Symbolic tokens that are intentionally not operators. - case TokenKind::At(): - case TokenKind::LessMinus(): - case TokenKind::MinusGreater(): - case TokenKind::EqualGreater(): - case TokenKind::ColonEqual(): - case TokenKind::Period(): - case TokenKind::Semi(): + case TokenKind::At: + case TokenKind::LessMinus: + case TokenKind::MinusGreater: + case TokenKind::EqualGreater: + case TokenKind::ColonEqual: + case TokenKind::Period: + case TokenKind::Semi: break; default: diff --git a/toolchain/parser/precedence_test.cpp b/toolchain/parser/precedence_test.cpp index df10a5a2ccc6..60a5d62b511b 100644 --- a/toolchain/parser/precedence_test.cpp +++ b/toolchain/parser/precedence_test.cpp @@ -15,140 +15,133 @@ namespace { using ::testing::Eq; TEST(PrecedenceTest, OperatorsAreRecognized) { - EXPECT_TRUE(PrecedenceGroup::ForLeading(TokenKind::Minus()).has_value()); - EXPECT_TRUE(PrecedenceGroup::ForLeading(TokenKind::Tilde()).has_value()); - EXPECT_FALSE(PrecedenceGroup::ForLeading(TokenKind::Slash()).has_value()); - EXPECT_FALSE( - PrecedenceGroup::ForLeading(TokenKind::Identifier()).has_value()); + EXPECT_TRUE(PrecedenceGroup::ForLeading(TokenKind::Minus).has_value()); + EXPECT_TRUE(PrecedenceGroup::ForLeading(TokenKind::Tilde).has_value()); + EXPECT_FALSE(PrecedenceGroup::ForLeading(TokenKind::Slash).has_value()); + EXPECT_FALSE(PrecedenceGroup::ForLeading(TokenKind::Identifier).has_value()); EXPECT_TRUE( - PrecedenceGroup::ForTrailing(TokenKind::Minus(), false).has_value()); + PrecedenceGroup::ForTrailing(TokenKind::Minus, false).has_value()); EXPECT_FALSE( - PrecedenceGroup::ForTrailing(TokenKind::Tilde(), false).has_value()); - EXPECT_TRUE( - PrecedenceGroup::ForTrailing(TokenKind::Slash(), true).has_value()); + PrecedenceGroup::ForTrailing(TokenKind::Tilde, false).has_value()); + EXPECT_TRUE(PrecedenceGroup::ForTrailing(TokenKind::Slash, true).has_value()); EXPECT_FALSE( - PrecedenceGroup::ForTrailing(TokenKind::Identifier(), false).has_value()); + PrecedenceGroup::ForTrailing(TokenKind::Identifier, false).has_value()); - EXPECT_TRUE( - PrecedenceGroup::ForTrailing(TokenKind::Minus(), true)->is_binary); + EXPECT_TRUE(PrecedenceGroup::ForTrailing(TokenKind::Minus, true)->is_binary); EXPECT_FALSE( - PrecedenceGroup::ForTrailing(TokenKind::MinusMinus(), false)->is_binary); + PrecedenceGroup::ForTrailing(TokenKind::MinusMinus, false)->is_binary); } TEST(PrecedenceTest, InfixVsPostfix) { // A trailing `-` is always infix; a trailing `--` is always postfix. - EXPECT_TRUE( - PrecedenceGroup::ForTrailing(TokenKind::Minus(), false)->is_binary); + EXPECT_TRUE(PrecedenceGroup::ForTrailing(TokenKind::Minus, false)->is_binary); EXPECT_FALSE( - PrecedenceGroup::ForTrailing(TokenKind::MinusMinus(), true)->is_binary); + PrecedenceGroup::ForTrailing(TokenKind::MinusMinus, true)->is_binary); // A trailing `*` is interpreted based on context. - EXPECT_TRUE(PrecedenceGroup::ForTrailing(TokenKind::Star(), true)->is_binary); - EXPECT_FALSE( - PrecedenceGroup::ForTrailing(TokenKind::Star(), false)->is_binary); + EXPECT_TRUE(PrecedenceGroup::ForTrailing(TokenKind::Star, true)->is_binary); + EXPECT_FALSE(PrecedenceGroup::ForTrailing(TokenKind::Star, false)->is_binary); // Infix `*` can appear in type contexts; binary `*` cannot. EXPECT_THAT(PrecedenceGroup::GetPriority( - PrecedenceGroup::ForTrailing(TokenKind::Star(), true)->level, + PrecedenceGroup::ForTrailing(TokenKind::Star, true)->level, PrecedenceGroup::ForType()), Eq(OperatorPriority::Ambiguous)); EXPECT_THAT(PrecedenceGroup::GetPriority( - PrecedenceGroup::ForTrailing(TokenKind::Star(), false)->level, + PrecedenceGroup::ForTrailing(TokenKind::Star, false)->level, PrecedenceGroup::ForType()), Eq(OperatorPriority::LeftFirst)); // Binary `*` can appear in `+` contexts; binary `*` cannot. EXPECT_THAT(PrecedenceGroup::GetPriority( - PrecedenceGroup::ForTrailing(TokenKind::Star(), true)->level, - PrecedenceGroup::ForTrailing(TokenKind::Plus(), true)->level), + PrecedenceGroup::ForTrailing(TokenKind::Star, true)->level, + PrecedenceGroup::ForTrailing(TokenKind::Plus, true)->level), Eq(OperatorPriority::LeftFirst)); EXPECT_THAT(PrecedenceGroup::GetPriority( - PrecedenceGroup::ForTrailing(TokenKind::Star(), false)->level, - PrecedenceGroup::ForTrailing(TokenKind::Plus(), true)->level), + PrecedenceGroup::ForTrailing(TokenKind::Star, false)->level, + PrecedenceGroup::ForTrailing(TokenKind::Plus, true)->level), Eq(OperatorPriority::Ambiguous)); } TEST(PrecedenceTest, Associativity) { - EXPECT_THAT( - PrecedenceGroup::ForLeading(TokenKind::Minus())->GetAssociativity(), - Eq(Associativity::RightToLeft)); - EXPECT_THAT(PrecedenceGroup::ForTrailing(TokenKind::PlusPlus(), false) + EXPECT_THAT(PrecedenceGroup::ForLeading(TokenKind::Minus)->GetAssociativity(), + Eq(Associativity::RightToLeft)); + EXPECT_THAT(PrecedenceGroup::ForTrailing(TokenKind::PlusPlus, false) ->level.GetAssociativity(), Eq(Associativity::LeftToRight)); - EXPECT_THAT(PrecedenceGroup::ForTrailing(TokenKind::Plus(), true) + EXPECT_THAT(PrecedenceGroup::ForTrailing(TokenKind::Plus, true) ->level.GetAssociativity(), Eq(Associativity::LeftToRight)); - EXPECT_THAT(PrecedenceGroup::ForTrailing(TokenKind::Equal(), true) + EXPECT_THAT(PrecedenceGroup::ForTrailing(TokenKind::Equal, true) ->level.GetAssociativity(), Eq(Associativity::RightToLeft)); - EXPECT_THAT(PrecedenceGroup::ForTrailing(TokenKind::PlusEqual(), true) + EXPECT_THAT(PrecedenceGroup::ForTrailing(TokenKind::PlusEqual, true) ->level.GetAssociativity(), Eq(Associativity::None)); } TEST(PrecedenceTest, DirectRelations) { EXPECT_THAT(PrecedenceGroup::GetPriority( - PrecedenceGroup::ForTrailing(TokenKind::Star(), true)->level, - PrecedenceGroup::ForTrailing(TokenKind::Plus(), true)->level), + PrecedenceGroup::ForTrailing(TokenKind::Star, true)->level, + PrecedenceGroup::ForTrailing(TokenKind::Plus, true)->level), Eq(OperatorPriority::LeftFirst)); EXPECT_THAT(PrecedenceGroup::GetPriority( - PrecedenceGroup::ForTrailing(TokenKind::Plus(), true)->level, - PrecedenceGroup::ForTrailing(TokenKind::Star(), true)->level), + PrecedenceGroup::ForTrailing(TokenKind::Plus, true)->level, + PrecedenceGroup::ForTrailing(TokenKind::Star, true)->level), Eq(OperatorPriority::RightFirst)); EXPECT_THAT(PrecedenceGroup::GetPriority( - PrecedenceGroup::ForTrailing(TokenKind::Amp(), true)->level, - PrecedenceGroup::ForTrailing(TokenKind::Less(), true)->level), + PrecedenceGroup::ForTrailing(TokenKind::Amp, true)->level, + PrecedenceGroup::ForTrailing(TokenKind::Less, true)->level), Eq(OperatorPriority::LeftFirst)); EXPECT_THAT(PrecedenceGroup::GetPriority( - PrecedenceGroup::ForTrailing(TokenKind::Less(), true)->level, - PrecedenceGroup::ForTrailing(TokenKind::Amp(), true)->level), + PrecedenceGroup::ForTrailing(TokenKind::Less, true)->level, + PrecedenceGroup::ForTrailing(TokenKind::Amp, true)->level), Eq(OperatorPriority::RightFirst)); } TEST(PrecedenceTest, IndirectRelations) { EXPECT_THAT(PrecedenceGroup::GetPriority( - PrecedenceGroup::ForTrailing(TokenKind::Star(), true)->level, - PrecedenceGroup::ForTrailing(TokenKind::Or(), true)->level), + PrecedenceGroup::ForTrailing(TokenKind::Star, true)->level, + PrecedenceGroup::ForTrailing(TokenKind::Or, true)->level), Eq(OperatorPriority::LeftFirst)); EXPECT_THAT(PrecedenceGroup::GetPriority( - PrecedenceGroup::ForTrailing(TokenKind::Or(), true)->level, - PrecedenceGroup::ForTrailing(TokenKind::Star(), true)->level), + PrecedenceGroup::ForTrailing(TokenKind::Or, true)->level, + PrecedenceGroup::ForTrailing(TokenKind::Star, true)->level), Eq(OperatorPriority::RightFirst)); - EXPECT_THAT( - PrecedenceGroup::GetPriority( - *PrecedenceGroup::ForLeading(TokenKind::Tilde()), - PrecedenceGroup::ForTrailing(TokenKind::Equal(), true)->level), - Eq(OperatorPriority::LeftFirst)); EXPECT_THAT(PrecedenceGroup::GetPriority( - PrecedenceGroup::ForTrailing(TokenKind::Equal(), true)->level, - *PrecedenceGroup::ForLeading(TokenKind::Tilde())), + *PrecedenceGroup::ForLeading(TokenKind::Tilde), + PrecedenceGroup::ForTrailing(TokenKind::Equal, true)->level), + Eq(OperatorPriority::LeftFirst)); + EXPECT_THAT(PrecedenceGroup::GetPriority( + PrecedenceGroup::ForTrailing(TokenKind::Equal, true)->level, + *PrecedenceGroup::ForLeading(TokenKind::Tilde)), Eq(OperatorPriority::RightFirst)); } TEST(PrecedenceTest, IncomparableOperators) { EXPECT_THAT(PrecedenceGroup::GetPriority( - *PrecedenceGroup::ForLeading(TokenKind::Tilde()), - *PrecedenceGroup::ForLeading(TokenKind::Not())), + *PrecedenceGroup::ForLeading(TokenKind::Tilde), + *PrecedenceGroup::ForLeading(TokenKind::Not)), Eq(OperatorPriority::Ambiguous)); EXPECT_THAT(PrecedenceGroup::GetPriority( - *PrecedenceGroup::ForLeading(TokenKind::Tilde()), - *PrecedenceGroup::ForLeading(TokenKind::Minus())), + *PrecedenceGroup::ForLeading(TokenKind::Tilde), + *PrecedenceGroup::ForLeading(TokenKind::Minus)), Eq(OperatorPriority::Ambiguous)); EXPECT_THAT(PrecedenceGroup::GetPriority( - *PrecedenceGroup::ForLeading(TokenKind::Minus()), - PrecedenceGroup::ForTrailing(TokenKind::Amp(), true)->level), + *PrecedenceGroup::ForLeading(TokenKind::Minus), + PrecedenceGroup::ForTrailing(TokenKind::Amp, true)->level), Eq(OperatorPriority::Ambiguous)); EXPECT_THAT( PrecedenceGroup::GetPriority( - PrecedenceGroup::ForTrailing(TokenKind::Equal(), true)->level, - PrecedenceGroup::ForTrailing(TokenKind::PipeEqual(), true)->level), + PrecedenceGroup::ForTrailing(TokenKind::Equal, true)->level, + PrecedenceGroup::ForTrailing(TokenKind::PipeEqual, true)->level), Eq(OperatorPriority::Ambiguous)); EXPECT_THAT(PrecedenceGroup::GetPriority( - PrecedenceGroup::ForTrailing(TokenKind::Plus(), true)->level, - PrecedenceGroup::ForTrailing(TokenKind::Amp(), true)->level), + PrecedenceGroup::ForTrailing(TokenKind::Plus, true)->level, + PrecedenceGroup::ForTrailing(TokenKind::Amp, true)->level), Eq(OperatorPriority::Ambiguous)); } diff --git a/toolchain/semantics/semantics_parse_tree_handler.cpp b/toolchain/semantics/semantics_parse_tree_handler.cpp index 475430ba19fe..bd0e40652d76 100644 --- a/toolchain/semantics/semantics_parse_tree_handler.cpp +++ b/toolchain/semantics/semantics_parse_tree_handler.cpp @@ -447,7 +447,7 @@ auto SemanticsParseTreeHandler::HandleInfixOperator(ParseTree::Node parse_node) // Figure out the operator for the token. auto token = parse_tree_->node_token(parse_node); switch (auto token_kind = tokens_->GetKind(token)) { - case TokenKind::Plus(): + case TokenKind::Plus: Push(parse_node, SemanticsNode::MakeBinaryOperatorAdd( parse_node, result_type, lhs_id, rhs_id)); break; @@ -475,18 +475,18 @@ auto SemanticsParseTreeHandler::HandleLiteral(ParseTree::Node parse_node) -> void { auto token = parse_tree_->node_token(parse_node); switch (auto token_kind = tokens_->GetKind(token)) { - case TokenKind::IntegerLiteral(): { + case TokenKind::IntegerLiteral: { auto id = semantics_->AddIntegerLiteral(tokens_->GetIntegerLiteral(token)); Push(parse_node, SemanticsNode::MakeIntegerLiteral(parse_node, id)); break; } - case TokenKind::RealLiteral(): { + case TokenKind::RealLiteral: { // TODO: Add storage of the Real literal. Push(parse_node, SemanticsNode::MakeRealLiteral(parse_node)); break; } - case TokenKind::IntegerTypeLiteral(): { + case TokenKind::IntegerTypeLiteral: { auto text = tokens_->GetTokenText(token); CARBON_CHECK(text == "i32") << "Currently only i32 is allowed"; Push(parse_node, SemanticsNodeId::MakeBuiltinReference(