diff --git a/toolchain/lexer/BUILD b/toolchain/lexer/BUILD index 1954862d5550..8ba3b57bf680 100644 --- a/toolchain/lexer/BUILD +++ b/toolchain/lexer/BUILD @@ -10,7 +10,7 @@ cc_library( name = "token_kind", srcs = ["token_kind.cpp"], hdrs = ["token_kind.h"], - textual_hdrs = ["token_registry.def"], + textual_hdrs = ["token_kind.def"], deps = [ "//common:check", "//common:ostream", diff --git a/toolchain/lexer/token_kind.cpp b/toolchain/lexer/token_kind.cpp index c1804fc125a4..df56bfb406f7 100644 --- a/toolchain/lexer/token_kind.cpp +++ b/toolchain/lexer/token_kind.cpp @@ -9,100 +9,100 @@ namespace Carbon { -auto TokenKind::Name() const -> llvm::StringRef { +auto TokenKind::name() const -> llvm::StringRef { static constexpr llvm::StringLiteral Names[] = { #define CARBON_TOKEN(TokenName) #TokenName, -#include "toolchain/lexer/token_registry.def" +#include "toolchain/lexer/token_kind.def" }; return Names[static_cast(kind_value_)]; } -auto TokenKind::IsSymbol() const -> bool { +auto TokenKind::is_symbol() const -> bool { static constexpr bool Table[] = { #define CARBON_TOKEN(TokenName) false, #define CARBON_SYMBOL_TOKEN(TokenName, Spelling) true, -#include "toolchain/lexer/token_registry.def" +#include "toolchain/lexer/token_kind.def" }; return Table[static_cast(kind_value_)]; } -auto TokenKind::IsGroupingSymbol() const -> bool { +auto TokenKind::is_grouping_symbol() const -> bool { static constexpr bool Table[] = { #define CARBON_TOKEN(TokenName) false, #define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \ true, #define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \ true, -#include "toolchain/lexer/token_registry.def" +#include "toolchain/lexer/token_kind.def" }; return Table[static_cast(kind_value_)]; } -auto TokenKind::IsOpeningSymbol() const -> bool { +auto TokenKind::is_opening_symbol() const -> bool { static constexpr bool Table[] = { #define CARBON_TOKEN(TokenName) false, #define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \ true, -#include "toolchain/lexer/token_registry.def" +#include "toolchain/lexer/token_kind.def" }; return Table[static_cast(kind_value_)]; } -auto TokenKind::GetClosingSymbol() const -> TokenKind { +auto TokenKind::closing_symbol() const -> TokenKind { static constexpr TokenKind Table[] = { #define CARBON_TOKEN(TokenName) Error(), #define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \ ClosingName(), -#include "toolchain/lexer/token_registry.def" +#include "toolchain/lexer/token_kind.def" }; auto result = Table[static_cast(kind_value_)]; CARBON_CHECK(result != Error()) << "Only opening symbols are valid!"; return result; } -auto TokenKind::IsClosingSymbol() const -> bool { +auto TokenKind::is_closing_symbol() const -> bool { static constexpr bool Table[] = { #define CARBON_TOKEN(TokenName) false, #define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \ true, -#include "toolchain/lexer/token_registry.def" +#include "toolchain/lexer/token_kind.def" }; return Table[static_cast(kind_value_)]; } -auto TokenKind::GetOpeningSymbol() const -> TokenKind { +auto TokenKind::opening_symbol() const -> TokenKind { static constexpr TokenKind Table[] = { #define CARBON_TOKEN(TokenName) Error(), #define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \ OpeningName(), -#include "toolchain/lexer/token_registry.def" +#include "toolchain/lexer/token_kind.def" }; auto result = Table[static_cast(kind_value_)]; CARBON_CHECK(result != Error()) << "Only closing symbols are valid!"; return result; } -auto TokenKind::IsKeyword() const -> bool { +auto TokenKind::is_keyword() const -> bool { static constexpr bool Table[] = { #define CARBON_TOKEN(TokenName) false, #define CARBON_KEYWORD_TOKEN(TokenName, Spelling) true, -#include "toolchain/lexer/token_registry.def" +#include "toolchain/lexer/token_kind.def" }; return Table[static_cast(kind_value_)]; } -auto TokenKind::IsSizedTypeLiteral() const -> bool { +auto TokenKind::is_sized_type_literal() const -> bool { return *this == TokenKind::IntegerTypeLiteral() || *this == TokenKind::UnsignedIntegerTypeLiteral() || *this == TokenKind::FloatingPointTypeLiteral(); } -auto TokenKind::GetFixedSpelling() const -> llvm::StringRef { +auto TokenKind::fixed_spelling() const -> llvm::StringRef { static constexpr llvm::StringLiteral Table[] = { #define CARBON_TOKEN(TokenName) "", #define CARBON_SYMBOL_TOKEN(TokenName, Spelling) Spelling, #define CARBON_KEYWORD_TOKEN(TokenName, Spelling) Spelling, -#include "toolchain/lexer/token_registry.def" +#include "toolchain/lexer/token_kind.def" }; return Table[static_cast(kind_value_)]; } diff --git a/toolchain/lexer/token_registry.def b/toolchain/lexer/token_kind.def similarity index 100% rename from toolchain/lexer/token_registry.def rename to toolchain/lexer/token_kind.def diff --git a/toolchain/lexer/token_kind.h b/toolchain/lexer/token_kind.h index 2ccc67708a87..3f1add35e12c 100644 --- a/toolchain/lexer/token_kind.h +++ b/toolchain/lexer/token_kind.h @@ -19,7 +19,7 @@ class TokenKind { // be used, for example in the conversion operator. enum class KindEnum : int8_t { #define CARBON_TOKEN(TokenName) TokenName, -#include "toolchain/lexer/token_registry.def" +#include "toolchain/lexer/token_kind.def" }; public: @@ -29,7 +29,7 @@ class TokenKind { static constexpr auto TokenName()->TokenKind { \ return TokenKind(KindEnum::TokenName); \ } -#include "toolchain/lexer/token_registry.def" +#include "toolchain/lexer/token_kind.def" // The default constructor is deleted as objects of this type should always be // constructed using the above factory functions for each unique kind. @@ -43,7 +43,7 @@ class TokenKind { } // Get a friendly name for the token for logging or debugging. - [[nodiscard]] auto Name() const -> llvm::StringRef; + [[nodiscard]] auto name() const -> llvm::StringRef; // Test whether this kind of token is a simple symbol sequence (punctuation, // not letters) that appears directly in the source text and can be @@ -51,37 +51,37 @@ class TokenKind { // inside of other tokens, outside of the contents of other tokens they // don't require any specific characters before or after to distinguish them // in the source. Returns false otherwise. - [[nodiscard]] auto IsSymbol() const -> bool; + [[nodiscard]] auto is_symbol() const -> bool; // Test whether this kind of token is a grouping symbol (part of an opening // and closing pair that must always be matched in the token stream). - [[nodiscard]] auto IsGroupingSymbol() const -> bool; + [[nodiscard]] auto is_grouping_symbol() const -> bool; // Test whether this kind of token is an opening symbol for a group. - [[nodiscard]] auto IsOpeningSymbol() const -> bool; + [[nodiscard]] auto is_opening_symbol() const -> bool; // Returns the associated closing symbol for an opening symbol. // // The token kind must be an opening symbol. - [[nodiscard]] auto GetClosingSymbol() const -> TokenKind; + [[nodiscard]] auto closing_symbol() const -> TokenKind; // Test whether this kind of token is a closing symbol for a group. - [[nodiscard]] auto IsClosingSymbol() const -> bool; + [[nodiscard]] auto is_closing_symbol() const -> bool; // Returns the associated opening symbol for a closing symbol. // // The token kind must be a closing symbol. - [[nodiscard]] auto GetOpeningSymbol() const -> TokenKind; + [[nodiscard]] auto opening_symbol() const -> TokenKind; // Test whether this kind of token is a keyword. - [[nodiscard]] auto IsKeyword() const -> bool; + [[nodiscard]] auto is_keyword() const -> bool; // Test whether this kind of token is a sized type literal. - [[nodiscard]] auto IsSizedTypeLiteral() const -> bool; + [[nodiscard]] auto is_sized_type_literal() const -> bool; // If this token kind has a fixed spelling when in source code, returns it. // Otherwise returns an empty string. - [[nodiscard]] auto GetFixedSpelling() const -> llvm::StringRef; + [[nodiscard]] auto fixed_spelling() const -> llvm::StringRef; // Test whether this token kind is in the provided list. [[nodiscard]] auto IsOneOf(std::initializer_list kinds) const @@ -101,7 +101,7 @@ class TokenKind { constexpr operator KindEnum() const { return kind_value_; } // Prints the TokenKind, typically for diagnostics. - void Print(llvm::raw_ostream& out) const { out << GetFixedSpelling(); } + void Print(llvm::raw_ostream& out) const { out << fixed_spelling(); } private: constexpr explicit TokenKind(KindEnum kind_value) : kind_value_(kind_value) {} diff --git a/toolchain/lexer/token_kind_test.cpp b/toolchain/lexer/token_kind_test.cpp index 74dcc9dcf88d..62b396aaca7f 100644 --- a/toolchain/lexer/token_kind_test.cpp +++ b/toolchain/lexer/token_kind_test.cpp @@ -24,59 +24,59 @@ constexpr llvm::StringLiteral SymbolRegex = // We restrict keywords to be lowercase ASCII letters and underscores. constexpr llvm::StringLiteral KeywordRegex = "[a-z_]+"; -#define CARBON_TOKEN(TokenName) \ - TEST(TokenKindTest, TokenName) { \ - EXPECT_EQ(#TokenName, TokenKind::TokenName().Name()); \ - EXPECT_FALSE(TokenKind::TokenName().IsSymbol()); \ - EXPECT_FALSE(TokenKind::TokenName().IsKeyword()); \ - EXPECT_EQ("", TokenKind::TokenName().GetFixedSpelling()); \ +#define CARBON_TOKEN(TokenName) \ + TEST(TokenKindTest, TokenName) { \ + EXPECT_EQ(#TokenName, TokenKind::TokenName().name()); \ + EXPECT_FALSE(TokenKind::TokenName().is_symbol()); \ + EXPECT_FALSE(TokenKind::TokenName().is_keyword()); \ + EXPECT_EQ("", TokenKind::TokenName().fixed_spelling()); \ } -#define CARBON_SYMBOL_TOKEN(TokenName, Spelling) \ - TEST(TokenKindTest, TokenName) { \ - EXPECT_EQ(#TokenName, TokenKind::TokenName().Name()); \ - EXPECT_TRUE(TokenKind::TokenName().IsSymbol()); \ - EXPECT_FALSE(TokenKind::TokenName().IsGroupingSymbol()); \ - EXPECT_FALSE(TokenKind::TokenName().IsOpeningSymbol()); \ - EXPECT_FALSE(TokenKind::TokenName().IsClosingSymbol()); \ - EXPECT_FALSE(TokenKind::TokenName().IsKeyword()); \ - EXPECT_EQ(Spelling, TokenKind::TokenName().GetFixedSpelling()); \ - EXPECT_THAT(Spelling, MatchesRegex(SymbolRegex.str())); \ +#define CARBON_SYMBOL_TOKEN(TokenName, Spelling) \ + TEST(TokenKindTest, TokenName) { \ + EXPECT_EQ(#TokenName, TokenKind::TokenName().name()); \ + EXPECT_TRUE(TokenKind::TokenName().is_symbol()); \ + EXPECT_FALSE(TokenKind::TokenName().is_grouping_symbol()); \ + EXPECT_FALSE(TokenKind::TokenName().is_opening_symbol()); \ + EXPECT_FALSE(TokenKind::TokenName().is_closing_symbol()); \ + EXPECT_FALSE(TokenKind::TokenName().is_keyword()); \ + EXPECT_EQ(Spelling, TokenKind::TokenName().fixed_spelling()); \ + EXPECT_THAT(Spelling, MatchesRegex(SymbolRegex.str())); \ } #define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \ TEST(TokenKindTest, TokenName) { \ - EXPECT_EQ(#TokenName, TokenKind::TokenName().Name()); \ - EXPECT_TRUE(TokenKind::TokenName().IsSymbol()); \ - EXPECT_TRUE(TokenKind::TokenName().IsGroupingSymbol()); \ - EXPECT_TRUE(TokenKind::TokenName().IsOpeningSymbol()); \ + EXPECT_EQ(#TokenName, TokenKind::TokenName().name()); \ + EXPECT_TRUE(TokenKind::TokenName().is_symbol()); \ + EXPECT_TRUE(TokenKind::TokenName().is_grouping_symbol()); \ + EXPECT_TRUE(TokenKind::TokenName().is_opening_symbol()); \ EXPECT_EQ(TokenKind::ClosingName(), \ - TokenKind::TokenName().GetClosingSymbol()); \ - EXPECT_FALSE(TokenKind::TokenName().IsClosingSymbol()); \ - EXPECT_FALSE(TokenKind::TokenName().IsKeyword()); \ - EXPECT_EQ(Spelling, TokenKind::TokenName().GetFixedSpelling()); \ + TokenKind::TokenName().closing_symbol()); \ + EXPECT_FALSE(TokenKind::TokenName().is_closing_symbol()); \ + EXPECT_FALSE(TokenKind::TokenName().is_keyword()); \ + EXPECT_EQ(Spelling, TokenKind::TokenName().fixed_spelling()); \ EXPECT_THAT(Spelling, MatchesRegex(SymbolRegex.str())); \ } #define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \ TEST(TokenKindTest, TokenName) { \ - EXPECT_EQ(#TokenName, TokenKind::TokenName().Name()); \ - EXPECT_TRUE(TokenKind::TokenName().IsSymbol()); \ - EXPECT_TRUE(TokenKind::TokenName().IsGroupingSymbol()); \ - EXPECT_FALSE(TokenKind::TokenName().IsOpeningSymbol()); \ - EXPECT_TRUE(TokenKind::TokenName().IsClosingSymbol()); \ + EXPECT_EQ(#TokenName, TokenKind::TokenName().name()); \ + EXPECT_TRUE(TokenKind::TokenName().is_symbol()); \ + EXPECT_TRUE(TokenKind::TokenName().is_grouping_symbol()); \ + EXPECT_FALSE(TokenKind::TokenName().is_opening_symbol()); \ + EXPECT_TRUE(TokenKind::TokenName().is_closing_symbol()); \ EXPECT_EQ(TokenKind::OpeningName(), \ - TokenKind::TokenName().GetOpeningSymbol()); \ - EXPECT_FALSE(TokenKind::TokenName().IsKeyword()); \ - EXPECT_EQ(Spelling, TokenKind::TokenName().GetFixedSpelling()); \ + TokenKind::TokenName().opening_symbol()); \ + EXPECT_FALSE(TokenKind::TokenName().is_keyword()); \ + EXPECT_EQ(Spelling, TokenKind::TokenName().fixed_spelling()); \ EXPECT_THAT(Spelling, MatchesRegex(SymbolRegex.str())); \ } -#define CARBON_KEYWORD_TOKEN(TokenName, Spelling) \ - TEST(TokenKindTest, TokenName) { \ - EXPECT_EQ(#TokenName, TokenKind::TokenName().Name()); \ - EXPECT_FALSE(TokenKind::TokenName().IsSymbol()); \ - EXPECT_TRUE(TokenKind::TokenName().IsKeyword()); \ - EXPECT_EQ(Spelling, TokenKind::TokenName().GetFixedSpelling()); \ - EXPECT_THAT(Spelling, MatchesRegex(KeywordRegex.str())); \ +#define CARBON_KEYWORD_TOKEN(TokenName, Spelling) \ + TEST(TokenKindTest, TokenName) { \ + EXPECT_EQ(#TokenName, TokenKind::TokenName().name()); \ + EXPECT_FALSE(TokenKind::TokenName().is_symbol()); \ + EXPECT_TRUE(TokenKind::TokenName().is_keyword()); \ + EXPECT_EQ(Spelling, TokenKind::TokenName().fixed_spelling()); \ + EXPECT_THAT(Spelling, MatchesRegex(KeywordRegex.str())); \ } -#include "toolchain/lexer/token_registry.def" +#include "toolchain/lexer/token_kind.def" // Verify that the symbol tokens are sorted from longest to shortest. This is // important to ensure that simply in-order testing will identify tokens @@ -87,7 +87,7 @@ TEST(TokenKindTest, SymbolsInDescendingLength) { EXPECT_LE(llvm::StringRef(Spelling).size(), previous_length) \ << "Symbol token not in descending length order: " << #TokenName; \ previous_length = llvm::StringRef(Spelling).size(); -#include "toolchain/lexer/token_registry.def" +#include "toolchain/lexer/token_kind.def" EXPECT_GT(previous_length, 0); } diff --git a/toolchain/lexer/tokenized_buffer.cpp b/toolchain/lexer/tokenized_buffer.cpp index 176955ab80e1..4fc7b3d15882 100644 --- a/toolchain/lexer/tokenized_buffer.cpp +++ b/toolchain/lexer/tokenized_buffer.cpp @@ -293,7 +293,7 @@ class TokenizedBuffer::Lexer { TokenKind kind = llvm::StringSwitch(source_text) #define CARBON_SYMBOL_TOKEN(Name, Spelling) \ .StartsWith(Spelling, TokenKind::Name()) -#include "toolchain/lexer/token_registry.def" +#include "toolchain/lexer/token_kind.def" .Default(TokenKind::Error()); if (kind == TokenKind::Error()) { return LexResult::NoMatch(); @@ -309,17 +309,17 @@ class TokenizedBuffer::Lexer { const char* location = source_text.begin(); Token token = buffer_->AddToken( {.kind = kind, .token_line = current_line_, .column = current_column_}); - current_column_ += kind.GetFixedSpelling().size(); - source_text = source_text.drop_front(kind.GetFixedSpelling().size()); + current_column_ += kind.fixed_spelling().size(); + source_text = source_text.drop_front(kind.fixed_spelling().size()); // Opening symbols just need to be pushed onto our queue of opening groups. - if (kind.IsOpeningSymbol()) { + if (kind.is_opening_symbol()) { open_groups_.push_back(token); return token; } // Only closing symbols need further special handling. - if (!kind.IsClosingSymbol()) { + if (!kind.is_closing_symbol()) { return token; } @@ -329,7 +329,7 @@ class TokenizedBuffer::Lexer { // a closing symbol. if (open_groups_.empty()) { closing_token_info.kind = TokenKind::Error(); - closing_token_info.error_length = kind.GetFixedSpelling().size(); + closing_token_info.error_length = kind.fixed_spelling().size(); CARBON_DIAGNOSTIC( UnmatchedClosing, Error, @@ -399,14 +399,14 @@ class TokenizedBuffer::Lexer { // Closes all open groups that cannot remain open across the symbol `K`. // Users may pass `Error` to close all open groups. auto CloseInvalidOpenGroups(TokenKind kind) -> void { - if (!kind.IsClosingSymbol() && kind != TokenKind::Error()) { + if (!kind.is_closing_symbol() && kind != TokenKind::Error()) { return; } while (!open_groups_.empty()) { Token opening_token = open_groups_.back(); TokenKind opening_kind = buffer_->GetTokenInfo(opening_token).kind; - if (kind == opening_kind.GetClosingSymbol()) { + if (kind == opening_kind.closing_symbol()) { return; } @@ -423,7 +423,7 @@ class TokenizedBuffer::Lexer { // TODO: do a smarter backwards scan for where to put the closing // token. Token closing_token = buffer_->AddToken( - {.kind = opening_kind.GetClosingSymbol(), + {.kind = opening_kind.closing_symbol(), .has_trailing_space = buffer_->HasTrailingWhitespace(prev_token), .is_recovery = true, .token_line = current_line_, @@ -472,7 +472,7 @@ class TokenizedBuffer::Lexer { // Check if the text matches a keyword token, and if so use that. TokenKind kind = llvm::StringSwitch(identifier_text) #define CARBON_KEYWORD_TOKEN(Name, Spelling) .Case(Spelling, TokenKind::Name()) -#include "toolchain/lexer/token_registry.def" +#include "toolchain/lexer/token_kind.def" .Default(TokenKind::Error()); if (kind != TokenKind::Error()) { return buffer_->AddToken({.kind = kind, @@ -500,7 +500,7 @@ class TokenizedBuffer::Lexer { } return llvm::StringSwitch(llvm::StringRef(&c, 1)) #define CARBON_SYMBOL_TOKEN(Name, Spelling) .StartsWith(Spelling, false) -#include "toolchain/lexer/token_registry.def" +#include "toolchain/lexer/token_kind.def" .Default(true); }); if (error_text.empty()) { @@ -604,7 +604,7 @@ auto TokenizedBuffer::GetColumnNumber(Token token) const -> int { auto TokenizedBuffer::GetTokenText(Token token) const -> llvm::StringRef { const auto& token_info = GetTokenInfo(token); - llvm::StringRef fixed_spelling = token_info.kind.GetFixedSpelling(); + llvm::StringRef fixed_spelling = token_info.kind.fixed_spelling(); if (!fixed_spelling.empty()) { return fixed_spelling; } @@ -640,7 +640,7 @@ auto TokenizedBuffer::GetTokenText(Token token) const -> llvm::StringRef { // Refer back to the source text to avoid needing to reconstruct the // spelling from the size. - if (token_info.kind.IsSizedTypeLiteral()) { + if (token_info.kind.is_sized_type_literal()) { const auto& line_info = GetLineInfo(token_info.token_line); int64_t token_start = line_info.start + token_info.column; llvm::StringRef suffix = @@ -653,14 +653,14 @@ auto TokenizedBuffer::GetTokenText(Token token) const -> llvm::StringRef { } CARBON_CHECK(token_info.kind == TokenKind::Identifier()) - << token_info.kind.Name(); + << token_info.kind.name(); return GetIdentifierText(token_info.id); } auto TokenizedBuffer::GetIdentifier(Token token) const -> Identifier { const auto& token_info = GetTokenInfo(token); CARBON_CHECK(token_info.kind == TokenKind::Identifier()) - << token_info.kind.Name(); + << token_info.kind.name(); return token_info.id; } @@ -668,14 +668,14 @@ auto TokenizedBuffer::GetIntegerLiteral(Token token) const -> const llvm::APInt& { const auto& token_info = GetTokenInfo(token); CARBON_CHECK(token_info.kind == TokenKind::IntegerLiteral()) - << token_info.kind.Name(); + << token_info.kind.name(); return literal_int_storage_[token_info.literal_index]; } auto TokenizedBuffer::GetRealLiteral(Token token) const -> RealLiteralValue { const auto& token_info = GetTokenInfo(token); CARBON_CHECK(token_info.kind == TokenKind::RealLiteral()) - << token_info.kind.Name(); + << token_info.kind.name(); // Note that every real literal is at least three characters long, so we can // safely look at the second character to determine whether we have a @@ -691,30 +691,31 @@ auto TokenizedBuffer::GetRealLiteral(Token token) const -> RealLiteralValue { auto TokenizedBuffer::GetStringLiteral(Token token) const -> llvm::StringRef { const auto& token_info = GetTokenInfo(token); CARBON_CHECK(token_info.kind == TokenKind::StringLiteral()) - << token_info.kind.Name(); + << token_info.kind.name(); return literal_string_storage_[token_info.literal_index]; } auto TokenizedBuffer::GetTypeLiteralSize(Token token) const -> const llvm::APInt& { const auto& token_info = GetTokenInfo(token); - CARBON_CHECK(token_info.kind.IsSizedTypeLiteral()) << token_info.kind.Name(); + CARBON_CHECK(token_info.kind.is_sized_type_literal()) + << token_info.kind.name(); return literal_int_storage_[token_info.literal_index]; } auto TokenizedBuffer::GetMatchedClosingToken(Token opening_token) const -> Token { const auto& opening_token_info = GetTokenInfo(opening_token); - CARBON_CHECK(opening_token_info.kind.IsOpeningSymbol()) - << opening_token_info.kind.Name(); + CARBON_CHECK(opening_token_info.kind.is_opening_symbol()) + << opening_token_info.kind.name(); return opening_token_info.closing_token; } auto TokenizedBuffer::GetMatchedOpeningToken(Token closing_token) const -> Token { const auto& closing_token_info = GetTokenInfo(closing_token); - CARBON_CHECK(closing_token_info.kind.IsClosingSymbol()) - << closing_token_info.kind.Name(); + CARBON_CHECK(closing_token_info.kind.is_closing_symbol()) + << closing_token_info.kind.name(); return closing_token_info.opening_token; } @@ -769,7 +770,7 @@ static auto ComputeDecimalPrintedWidth(int number) -> int { auto TokenizedBuffer::GetTokenPrintWidths(Token token) const -> PrintWidths { PrintWidths widths = {}; widths.index = ComputeDecimalPrintedWidth(token_infos_.size()); - widths.kind = GetKind(token).Name().size(); + widths.kind = GetKind(token).name().size(); widths.line = ComputeDecimalPrintedWidth(GetLineNumber(token)); widths.column = ComputeDecimalPrintedWidth(GetColumnNumber(token)); widths.indent = @@ -816,7 +817,7 @@ auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream, Token token, "spelling: '{5}'", llvm::format_decimal(token_index, widths.index), llvm::right_justify( - (llvm::Twine("'") + token_info.kind.Name() + "'").str(), + (llvm::Twine("'") + token_info.kind.name() + "'").str(), widths.kind + 2), llvm::format_decimal(GetLineNumber(token_info.token_line), widths.line), llvm::format_decimal(GetColumnNumber(token), widths.column), @@ -840,10 +841,10 @@ auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream, Token token, output_stream << ", value: `" << GetStringLiteral(token) << "`"; break; default: - if (token_info.kind.IsOpeningSymbol()) { + if (token_info.kind.is_opening_symbol()) { output_stream << ", closing_token: " << GetMatchedClosingToken(token).index; - } else if (token_info.kind.IsClosingSymbol()) { + } else if (token_info.kind.is_closing_symbol()) { output_stream << ", opening_token: " << GetMatchedOpeningToken(token).index; } diff --git a/toolchain/lexer/tokenized_buffer_test_helpers.h b/toolchain/lexer/tokenized_buffer_test_helpers.h index 7e2b50407f82..026afb759853 100644 --- a/toolchain/lexer/tokenized_buffer_test_helpers.h +++ b/toolchain/lexer/tokenized_buffer_test_helpers.h @@ -28,7 +28,7 @@ namespace Testing { struct ExpectedToken { friend auto operator<<(std::ostream& output, const ExpectedToken& expected) -> std::ostream& { - output << "\ntoken: { kind: '" << expected.kind.Name().str() << "'"; + output << "\ntoken: { kind: '" << expected.kind.name().str() << "'"; if (expected.line != -1) { output << ", line: " << expected.line; } @@ -83,8 +83,8 @@ MATCHER_P(HasTokens, raw_all_expected, "") { TokenKind actual_kind = buffer.GetKind(token); if (actual_kind != expected.kind) { *result_listener << "\nToken " << index << " is a " - << actual_kind.Name().str() << ", expected a " - << expected.kind.Name().str() << "."; + << actual_kind.name().str() << ", expected a " + << expected.kind.name().str() << "."; matches = false; } diff --git a/toolchain/parser/parser.cpp b/toolchain/parser/parser.cpp index 2f6849b0ff76..b2aa5f7beb16 100644 --- a/toolchain/parser/parser.cpp +++ b/toolchain/parser/parser.cpp @@ -75,7 +75,7 @@ class Parser::PrettyStackTraceParseState : public llvm::PrettyStackTraceEntry { output << " @ " << parser_->tokens_->GetLineNumber(line) << ":" << parser_->tokens_->GetColumnNumber(token) << ":" << " token " << token << " : " - << parser_->tokens_->GetKind(token).Name() << "\n"; + << parser_->tokens_->GetKind(token).name() << "\n"; } const Parser* parser_; @@ -93,7 +93,7 @@ Parser::Parser(ParseTree& tree, TokenizedBuffer& tokens, --end_; CARBON_CHECK(tokens_->GetKind(*end_) == TokenKind::EndOfFile()) << "TokenizedBuffer should end with EndOfFile, ended with " - << tokens_->GetKind(*end_).Name(); + << tokens_->GetKind(*end_).name(); } auto Parser::AddLeafNode(ParseNodeKind kind, TokenizedBuffer::Token token, @@ -160,7 +160,7 @@ auto Parser::ConsumeAndAddLeafNodeIf(TokenKind token_kind, auto Parser::ConsumeChecked(TokenKind kind) -> TokenizedBuffer::Token { CARBON_CHECK(PositionIs(kind)) - << "Required " << kind.Name() << ", found " << PositionKind().Name(); + << "Required " << kind.name() << ", found " << PositionKind().name(); return Consume(); } @@ -183,10 +183,10 @@ auto Parser::FindNextOf(std::initializer_list desired_kinds) } // Step to the next token at the current bracketing level. - if (kind.IsClosingSymbol() || kind == TokenKind::EndOfFile()) { + if (kind.is_closing_symbol() || kind == TokenKind::EndOfFile()) { // There are no more tokens at this level. return std::nullopt; - } else if (kind.IsOpeningSymbol()) { + } else if (kind.is_opening_symbol()) { new_position = TokenizedBuffer::TokenIterator( tokens_->GetMatchedClosingToken(token)); // Advance past the closing token. @@ -198,7 +198,7 @@ auto Parser::FindNextOf(std::initializer_list desired_kinds) } auto Parser::SkipMatchingGroup() -> bool { - if (!PositionKind().IsOpeningSymbol()) { + if (!PositionKind().is_opening_symbol()) { return false; } @@ -377,7 +377,7 @@ auto Parser::DiagnoseOperatorFixity(OperatorFixity fixity) -> void { // Whitespace is not permitted between a symbolic pre/postfix operator and // its operand. - if (PositionKind().IsSymbol() && + if (PositionKind().is_symbol() && (prefix ? tokens_->HasTrailingWhitespace(*position_) : tokens_->HasLeadingWhitespace(*position_))) { CARBON_DIAGNOSTIC(UnaryOperatorHasWhitespace, Error, @@ -414,7 +414,7 @@ auto Parser::ConsumeListToken(ParseNodeKind comma_kind, TokenKind close_kind, auto end_of_element = FindNextOf({TokenKind::Comma(), close_kind}); // The lexer guarantees that parentheses are balanced. CARBON_CHECK(end_of_element) - << "missing matching `" << close_kind.GetOpeningSymbol() << "` for `" + << "missing matching `" << close_kind.opening_symbol() << "` for `" << close_kind << "`"; SkipTo(*end_of_element); @@ -764,7 +764,7 @@ auto Parser::HandleDesignator(bool as_struct) -> void { emitter_->Emit(*position_, ExpectedIdentifierAfterDot); // If we see a keyword, assume it was intended to be the designated name. // TODO: Should keywords be valid in designators? - if (PositionKind().IsKeyword()) { + if (PositionKind().is_keyword()) { AddLeafNode(ParseNodeKind::DesignatedName(), Consume(), /*has_error=*/true); } else { diff --git a/toolchain/semantics/semantics_parse_tree_handler.cpp b/toolchain/semantics/semantics_parse_tree_handler.cpp index ba9cfe51ac96..ba8142b2e02a 100644 --- a/toolchain/semantics/semantics_parse_tree_handler.cpp +++ b/toolchain/semantics/semantics_parse_tree_handler.cpp @@ -345,7 +345,7 @@ auto SemanticsParseTreeHandler::HandleInfixOperator(ParseTree::Node parse_node) parse_node, result_type, lhs_id, rhs_id)); break; default: - CARBON_FATAL() << "Unrecognized token kind: " << token_kind.Name(); + CARBON_FATAL() << "Unrecognized token kind: " << token_kind.name(); } } @@ -387,7 +387,7 @@ auto SemanticsParseTreeHandler::HandleLiteral(ParseTree::Node parse_node) break; } default: - CARBON_FATAL() << "Unhandled kind: " << token_kind.Name(); + CARBON_FATAL() << "Unhandled kind: " << token_kind.name(); } }