From eae630a3dbd015cb90ac72a6612dc7a3508539ea Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Wed, 29 Nov 2023 12:33:58 -0800 Subject: [PATCH] Rename Lex::{Token,Line} -> Lex::{Token,Line}Index. (#3433) As discussed [on discord](https://discord.com/channels/655572317891461132/655578254970716160/1178878128714678282) and today's toolchain discussion. --- toolchain/lex/lex.cpp | 44 +++---- toolchain/lex/token_kind.def | 2 +- toolchain/lex/tokenized_buffer.cpp | 80 ++++++------ toolchain/lex/tokenized_buffer.h | 121 +++++++++--------- toolchain/lex/tokenized_buffer_fuzzer.cpp | 2 +- toolchain/lex/tokenized_buffer_test.cpp | 4 +- toolchain/parse/context.cpp | 32 ++--- toolchain/parse/context.h | 45 ++++--- .../parse/handle_decl_name_and_params.cpp | 2 +- toolchain/parse/handle_import_and_package.cpp | 4 +- toolchain/parse/handle_paren_condition.cpp | 2 +- toolchain/parse/handle_statement.cpp | 2 +- toolchain/parse/handle_var.cpp | 3 +- toolchain/parse/node_kind.def | 2 +- toolchain/parse/tree.cpp | 2 +- toolchain/parse/tree.h | 6 +- 16 files changed, 182 insertions(+), 171 deletions(-) diff --git a/toolchain/lex/lex.cpp b/toolchain/lex/lex.cpp index 00ab17af51e4..49b9d6858bf9 100644 --- a/toolchain/lex/lex.cpp +++ b/toolchain/lex/lex.cpp @@ -58,7 +58,7 @@ class [[clang::internal_linkage]] Lexer { // Consumes (and discard) a valid token to construct a result // indicating a token has been produced. Relies on implicit conversions. // NOLINTNEXTLINE(google-explicit-constructor) - LexResult(Token /*discarded_token*/) : LexResult(true) {} + LexResult(TokenIndex /*discarded_token*/) : LexResult(true) {} // Returns a result indicating no token was produced. static auto NoMatch() -> LexResult { return LexResult(false); } @@ -89,7 +89,7 @@ class [[clang::internal_linkage]] Lexer { // But because it can, the compiler will flatten this otherwise. [[gnu::noinline]] auto CreateLines(llvm::StringRef source_text) -> void; - auto current_line() -> Line { return Line(line_index_); } + auto current_line() -> LineIndex { return LineIndex(line_index_); } auto current_line_info() -> TokenizedBuffer::LineInfo* { return &buffer_.line_infos_[line_index_]; @@ -125,7 +125,7 @@ class [[clang::internal_linkage]] Lexer { -> LexResult; auto LexOneCharSymbolToken(llvm::StringRef source_text, TokenKind kind, - ssize_t& position) -> Token; + ssize_t& position) -> TokenIndex; auto LexOpeningSymbolToken(llvm::StringRef source_text, TokenKind kind, ssize_t& position) -> LexResult; @@ -170,7 +170,7 @@ class [[clang::internal_linkage]] Lexer { ssize_t line_index_; - llvm::SmallVector open_groups_; + llvm::SmallVector open_groups_; ErrorTrackingDiagnosticConsumer consumer_; @@ -937,7 +937,7 @@ auto Lexer::LexStringLiteral(llvm::StringRef source_text, ssize_t& position) return LexError(source_text, position); } - Line string_line = current_line(); + LineIndex string_line = current_line(); int string_column = ComputeColumn(position); ssize_t literal_size = literal->text().size(); position += literal_size; @@ -975,7 +975,7 @@ auto Lexer::LexStringLiteral(llvm::StringRef source_text, ssize_t& position) } auto Lexer::LexOneCharSymbolToken(llvm::StringRef source_text, TokenKind kind, - ssize_t& position) -> Token { + ssize_t& position) -> TokenIndex { // Verify in a debug build that the incoming token kind is correct. CARBON_DCHECK(kind != TokenKind::Error); CARBON_DCHECK(kind.fixed_spelling().size() == 1); @@ -984,16 +984,16 @@ auto Lexer::LexOneCharSymbolToken(llvm::StringRef source_text, TokenKind kind, << "' instead of the spelling '" << kind.fixed_spelling() << "' of the incoming token kind '" << kind << "'"; - Token token = buffer_.AddToken({.kind = kind, - .token_line = current_line(), - .column = ComputeColumn(position)}); + TokenIndex token = buffer_.AddToken({.kind = kind, + .token_line = current_line(), + .column = ComputeColumn(position)}); ++position; return token; } auto Lexer::LexOpeningSymbolToken(llvm::StringRef source_text, TokenKind kind, ssize_t& position) -> LexResult { - Token token = LexOneCharSymbolToken(source_text, kind, position); + TokenIndex token = LexOneCharSymbolToken(source_text, kind, position); open_groups_.push_back(token); return token; } @@ -1004,10 +1004,10 @@ auto Lexer::LexClosingSymbolToken(llvm::StringRef source_text, TokenKind kind, CARBON_DIAGNOSTIC(UnmatchedClosing, Error, "Closing symbol without a corresponding opening symbol."); emitter_.Emit(source_text.begin() + position, UnmatchedClosing); - Token token = buffer_.AddToken({.kind = TokenKind::Error, - .token_line = current_line(), - .column = ComputeColumn(position), - .error_length = 1}); + TokenIndex token = buffer_.AddToken({.kind = TokenKind::Error, + .token_line = current_line(), + .column = ComputeColumn(position), + .error_length = 1}); ++position; return token; }; @@ -1017,7 +1017,7 @@ auto Lexer::LexClosingSymbolToken(llvm::StringRef source_text, TokenKind kind, return unmatched_error(); } - Token opening_token = open_groups_.back(); + TokenIndex opening_token = open_groups_.back(); // Close any invalid open groups first. if (LLVM_UNLIKELY(buffer_.GetTokenInfo(opening_token).kind != kind.opening_symbol())) { @@ -1033,7 +1033,7 @@ auto Lexer::LexClosingSymbolToken(llvm::StringRef source_text, TokenKind kind, open_groups_.pop_back(); // Now that the groups are all matched up, lex the actual token. - Token token = LexOneCharSymbolToken(source_text, kind, position); + TokenIndex token = LexOneCharSymbolToken(source_text, kind, position); // Note that it is important to get fresh token infos here as lexing the // open token would invalidate any pointers. @@ -1059,9 +1059,9 @@ auto Lexer::LexSymbolToken(llvm::StringRef source_text, ssize_t& position) return LexError(source_text, position); } - Token token = buffer_.AddToken({.kind = kind, - .token_line = current_line(), - .column = ComputeColumn(position)}); + TokenIndex token = buffer_.AddToken({.kind = kind, + .token_line = current_line(), + .column = ComputeColumn(position)}); position += kind.fixed_spelling().size(); return token; } @@ -1119,7 +1119,7 @@ auto Lexer::CloseInvalidOpenGroups(TokenKind kind, ssize_t position) -> void { int column = ComputeColumn(position); do { - Token opening_token = open_groups_.back(); + TokenIndex opening_token = open_groups_.back(); TokenKind opening_kind = buffer_.GetTokenInfo(opening_token).kind; if (kind == opening_kind.closing_symbol()) { return; @@ -1133,11 +1133,11 @@ auto Lexer::CloseInvalidOpenGroups(TokenKind kind, ssize_t position) -> void { CARBON_CHECK(!buffer_.tokens().empty()) << "Must have a prior opening token!"; - Token prev_token = buffer_.tokens().end()[-1]; + TokenIndex prev_token = buffer_.tokens().end()[-1]; // TODO: do a smarter backwards scan for where to put the closing // token. - Token closing_token = buffer_.AddToken( + TokenIndex closing_token = buffer_.AddToken( {.kind = opening_kind.closing_symbol(), .has_trailing_space = buffer_.HasTrailingWhitespace(prev_token), .is_recovery = true, diff --git a/toolchain/lex/token_kind.def b/toolchain/lex/token_kind.def index e6c575c2b669..e4135c76c803 100644 --- a/toolchain/lex/token_kind.def +++ b/toolchain/lex/token_kind.def @@ -21,7 +21,7 @@ // - CARBON_KEYWORD_TOKEN(Name, Spelling) // Defines a keyword which has the provided spelling, such as `if`. // Spellings must be unique. -// - CARBON_TOKEN_WITH_VIRTUAL_NODE(Token) +// - CARBON_TOKEN_WITH_VIRTUAL_NODE(TokenIndex) // Wrapped around one of the abokve _TOKEN macros, indicates that this // token has one additional virtual node in the parse tree. // diff --git a/toolchain/lex/tokenized_buffer.cpp b/toolchain/lex/tokenized_buffer.cpp index d4af7a8acb0f..2e7f12815e37 100644 --- a/toolchain/lex/tokenized_buffer.cpp +++ b/toolchain/lex/tokenized_buffer.cpp @@ -19,23 +19,23 @@ namespace Carbon::Lex { -auto TokenizedBuffer::GetKind(Token token) const -> TokenKind { +auto TokenizedBuffer::GetKind(TokenIndex token) const -> TokenKind { return GetTokenInfo(token).kind; } -auto TokenizedBuffer::GetLine(Token token) const -> Line { +auto TokenizedBuffer::GetLine(TokenIndex token) const -> LineIndex { return GetTokenInfo(token).token_line; } -auto TokenizedBuffer::GetLineNumber(Token token) const -> int { +auto TokenizedBuffer::GetLineNumber(TokenIndex token) const -> int { return GetLineNumber(GetLine(token)); } -auto TokenizedBuffer::GetColumnNumber(Token token) const -> int { +auto TokenizedBuffer::GetColumnNumber(TokenIndex token) const -> int { return GetTokenInfo(token).column + 1; } -auto TokenizedBuffer::GetTokenText(Token token) const -> llvm::StringRef { +auto TokenizedBuffer::GetTokenText(TokenIndex token) const -> llvm::StringRef { const auto& token_info = GetTokenInfo(token); llvm::StringRef fixed_spelling = token_info.kind.fixed_spelling(); if (!fixed_spelling.empty()) { @@ -90,82 +90,83 @@ auto TokenizedBuffer::GetTokenText(Token token) const -> llvm::StringRef { return value_stores_->identifiers().Get(token_info.ident_id); } -auto TokenizedBuffer::GetIdentifier(Token token) const -> IdentifierId { +auto TokenizedBuffer::GetIdentifier(TokenIndex token) const -> IdentifierId { const auto& token_info = GetTokenInfo(token); CARBON_CHECK(token_info.kind == TokenKind::Identifier) << token_info.kind; return token_info.ident_id; } -auto TokenizedBuffer::GetIntegerLiteral(Token token) const -> IntegerId { +auto TokenizedBuffer::GetIntegerLiteral(TokenIndex token) const -> IntegerId { const auto& token_info = GetTokenInfo(token); CARBON_CHECK(token_info.kind == TokenKind::IntegerLiteral) << token_info.kind; return token_info.integer_id; } -auto TokenizedBuffer::GetRealLiteral(Token token) const -> RealId { +auto TokenizedBuffer::GetRealLiteral(TokenIndex token) const -> RealId { const auto& token_info = GetTokenInfo(token); CARBON_CHECK(token_info.kind == TokenKind::RealLiteral) << token_info.kind; return token_info.real_id; } -auto TokenizedBuffer::GetStringLiteral(Token token) const -> StringLiteralId { +auto TokenizedBuffer::GetStringLiteral(TokenIndex token) const + -> StringLiteralId { const auto& token_info = GetTokenInfo(token); CARBON_CHECK(token_info.kind == TokenKind::StringLiteral) << token_info.kind; return token_info.string_literal_id; } -auto TokenizedBuffer::GetTypeLiteralSize(Token token) const +auto TokenizedBuffer::GetTypeLiteralSize(TokenIndex token) const -> const llvm::APInt& { const auto& token_info = GetTokenInfo(token); CARBON_CHECK(token_info.kind.is_sized_type_literal()) << token_info.kind; return value_stores_->integers().Get(token_info.integer_id); } -auto TokenizedBuffer::GetMatchedClosingToken(Token opening_token) const - -> Token { +auto TokenizedBuffer::GetMatchedClosingToken(TokenIndex opening_token) const + -> TokenIndex { const auto& opening_token_info = GetTokenInfo(opening_token); CARBON_CHECK(opening_token_info.kind.is_opening_symbol()) << opening_token_info.kind; return opening_token_info.closing_token; } -auto TokenizedBuffer::GetMatchedOpeningToken(Token closing_token) const - -> Token { +auto TokenizedBuffer::GetMatchedOpeningToken(TokenIndex closing_token) const + -> TokenIndex { const auto& closing_token_info = GetTokenInfo(closing_token); CARBON_CHECK(closing_token_info.kind.is_closing_symbol()) << closing_token_info.kind; return closing_token_info.opening_token; } -auto TokenizedBuffer::HasLeadingWhitespace(Token token) const -> bool { +auto TokenizedBuffer::HasLeadingWhitespace(TokenIndex token) const -> bool { auto it = TokenIterator(token); return it == tokens().begin() || GetTokenInfo(*(it - 1)).has_trailing_space; } -auto TokenizedBuffer::HasTrailingWhitespace(Token token) const -> bool { +auto TokenizedBuffer::HasTrailingWhitespace(TokenIndex token) const -> bool { return GetTokenInfo(token).has_trailing_space; } -auto TokenizedBuffer::IsRecoveryToken(Token token) const -> bool { +auto TokenizedBuffer::IsRecoveryToken(TokenIndex token) const -> bool { return GetTokenInfo(token).is_recovery; } -auto TokenizedBuffer::GetLineNumber(Line line) const -> int { +auto TokenizedBuffer::GetLineNumber(LineIndex line) const -> int { return line.index + 1; } -auto TokenizedBuffer::GetNextLine(Line line) const -> Line { - Line next(line.index + 1); +auto TokenizedBuffer::GetNextLine(LineIndex line) const -> LineIndex { + LineIndex next(line.index + 1); CARBON_DCHECK(static_cast(next.index) < line_infos_.size()); return next; } -auto TokenizedBuffer::GetPrevLine(Line line) const -> Line { +auto TokenizedBuffer::GetPrevLine(LineIndex line) const -> LineIndex { CARBON_CHECK(line.index > 0); - return Line(line.index - 1); + return LineIndex(line.index - 1); } -auto TokenizedBuffer::GetIndentColumnNumber(Line line) const -> int { +auto TokenizedBuffer::GetIndentColumnNumber(LineIndex line) const -> int { return GetLineInfo(line).indent + 1; } @@ -191,7 +192,8 @@ static auto ComputeDecimalPrintedWidth(int number) -> int { return static_cast(std::log10(number)) + 1; } -auto TokenizedBuffer::GetTokenPrintWidths(Token token) const -> PrintWidths { +auto TokenizedBuffer::GetTokenPrintWidths(TokenIndex token) const + -> PrintWidths { PrintWidths widths = {}; widths.index = ComputeDecimalPrintedWidth(token_infos_.size()); widths.kind = GetKind(token).name().size(); @@ -212,11 +214,11 @@ auto TokenizedBuffer::Print(llvm::raw_ostream& output_stream) const -> void { PrintWidths widths = {}; widths.index = ComputeDecimalPrintedWidth((token_infos_.size())); - for (Token token : tokens()) { + for (TokenIndex token : tokens()) { widths.Widen(GetTokenPrintWidths(token)); } - for (Token token : tokens()) { + for (TokenIndex token : tokens()) { PrintToken(output_stream, token, widths); output_stream << "\n"; } @@ -224,12 +226,13 @@ auto TokenizedBuffer::Print(llvm::raw_ostream& output_stream) const -> void { } auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream, - Token token) const -> void { + TokenIndex token) const -> void { PrintToken(output_stream, token, {}); } -auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream, Token token, - PrintWidths widths) const -> void { +auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream, + TokenIndex token, PrintWidths widths) const + -> void { widths.Widen(GetTokenPrintWidths(token)); int token_index = token.index; const auto& token_info = GetTokenInfo(token); @@ -292,31 +295,31 @@ auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream, Token token, output_stream << " },"; } -auto TokenizedBuffer::GetLineInfo(Line line) -> LineInfo& { +auto TokenizedBuffer::GetLineInfo(LineIndex line) -> LineInfo& { return line_infos_[line.index]; } -auto TokenizedBuffer::GetLineInfo(Line line) const -> const LineInfo& { +auto TokenizedBuffer::GetLineInfo(LineIndex line) const -> const LineInfo& { return line_infos_[line.index]; } -auto TokenizedBuffer::AddLine(LineInfo info) -> Line { +auto TokenizedBuffer::AddLine(LineInfo info) -> LineIndex { line_infos_.push_back(info); - return Line(static_cast(line_infos_.size()) - 1); + return LineIndex(static_cast(line_infos_.size()) - 1); } -auto TokenizedBuffer::GetTokenInfo(Token token) -> TokenInfo& { +auto TokenizedBuffer::GetTokenInfo(TokenIndex token) -> TokenInfo& { return token_infos_[token.index]; } -auto TokenizedBuffer::GetTokenInfo(Token token) const -> const TokenInfo& { +auto TokenizedBuffer::GetTokenInfo(TokenIndex token) const -> const TokenInfo& { return token_infos_[token.index]; } -auto TokenizedBuffer::AddToken(TokenInfo info) -> Token { +auto TokenizedBuffer::AddToken(TokenInfo info) -> TokenIndex { token_infos_.push_back(info); expected_parse_tree_size_ += info.kind.expected_parse_tree_size(); - return Token(static_cast(token_infos_.size()) - 1); + return TokenIndex(static_cast(token_infos_.size()) - 1); } auto TokenIterator::Print(llvm::raw_ostream& output) const -> void { @@ -368,7 +371,8 @@ auto TokenizedBuffer::SourceBufferLocationTranslator::GetLocation( .column_number = column_number + 1}; } -auto TokenLocationTranslator::GetLocation(Token token) -> DiagnosticLocation { +auto TokenLocationTranslator::GetLocation(TokenIndex token) + -> DiagnosticLocation { // Map the token location into a position within the source buffer. const auto& token_info = buffer_->GetTokenInfo(token); const auto& line_info = buffer_->GetLineInfo(token_info.token_line); diff --git a/toolchain/lex/tokenized_buffer.h b/toolchain/lex/tokenized_buffer.h index 019488d90cec..3babb9dba8e1 100644 --- a/toolchain/lex/tokenized_buffer.h +++ b/toolchain/lex/tokenized_buffer.h @@ -28,54 +28,55 @@ class TokenizedBuffer; // A lightweight handle to a lexed token in a `TokenizedBuffer`. // -// `Token` objects are designed to be passed by value, not reference or +// `TokenIndex` objects are designed to be passed by value, not reference or // pointer. They are also designed to be small and efficient to store in data // structures. // -// `Token` objects from the same `TokenizedBuffer` can be compared with each -// other, both for being the same token within the buffer, and to establish +// `TokenIndex` objects from the same `TokenizedBuffer` can be compared with +// each other, both for being the same token within the buffer, and to establish // relative position within the token stream that has been lexed out of the -// buffer. `Token` objects from different `TokenizedBuffer`s cannot be +// buffer. `TokenIndex` objects from different `TokenizedBuffer`s cannot be // meaningfully compared. // -// All other APIs to query a `Token` are on the `TokenizedBuffer`. -struct Token : public ComparableIndexBase { - static const Token Invalid; +// All other APIs to query a `TokenIndex` are on the `TokenizedBuffer`. +struct TokenIndex : public ComparableIndexBase { + static const TokenIndex Invalid; // Comments aren't tokenized, so this is the first token after FileStart. - static const Token FirstNonCommentToken; + static const TokenIndex FirstNonCommentToken; using ComparableIndexBase::ComparableIndexBase; }; -constexpr Token Token::Invalid(Token::InvalidIndex); -constexpr Token Token::FirstNonCommentToken(1); +constexpr TokenIndex TokenIndex::Invalid(TokenIndex::InvalidIndex); +constexpr TokenIndex TokenIndex::FirstNonCommentToken(1); // A lightweight handle to a lexed line in a `TokenizedBuffer`. // -// `Line` objects are designed to be passed by value, not reference or +// `LineIndex` objects are designed to be passed by value, not reference or // pointer. They are also designed to be small and efficient to store in data // structures. // -// Each `Line` object refers to a specific line in the source code that was +// Each `LineIndex` object refers to a specific line in the source code that was // lexed. They can be compared directly to establish that they refer to the // same line or the relative position of different lines within the source. // -// All other APIs to query a `Line` are on the `TokenizedBuffer`. -struct Line : public ComparableIndexBase { - static const Line Invalid; +// All other APIs to query a `LineIndex` are on the `TokenizedBuffer`. +struct LineIndex : public ComparableIndexBase { + static const LineIndex Invalid; using ComparableIndexBase::ComparableIndexBase; }; -constexpr Line Line::Invalid(Line::InvalidIndex); +constexpr LineIndex LineIndex::Invalid(LineIndex::InvalidIndex); // Random-access iterator over tokens within the buffer. class TokenIterator - : public llvm::iterator_facade_base< - TokenIterator, std::random_access_iterator_tag, const Token, int>, + : public llvm::iterator_facade_base, public Printable { public: TokenIterator() = delete; - explicit TokenIterator(Token token) : token_(token) {} + explicit TokenIterator(TokenIndex token) : token_(token) {} auto operator==(const TokenIterator& rhs) const -> bool { return token_ == rhs.token_; @@ -84,7 +85,7 @@ class TokenIterator return token_ < rhs.token_; } - auto operator*() const -> const Token& { return token_; } + auto operator*() const -> const TokenIndex& { return token_; } using iterator_facade_base::operator-; auto operator-(const TokenIterator& rhs) const -> int { @@ -106,18 +107,19 @@ class TokenIterator private: friend class TokenizedBuffer; - Token token_; + TokenIndex token_; }; // A diagnostic location translator that maps token locations into source // buffer locations. -class TokenLocationTranslator : public DiagnosticLocationTranslator { +class TokenLocationTranslator + : public DiagnosticLocationTranslator { public: explicit TokenLocationTranslator(const TokenizedBuffer* buffer) : buffer_(buffer) {} // Map the given token into a diagnostic location. - auto GetLocation(Token token) -> DiagnosticLocation override; + auto GetLocation(TokenIndex token) -> DiagnosticLocation override; private: const TokenizedBuffer* buffer_; @@ -133,65 +135,65 @@ class TokenLocationTranslator : public DiagnosticLocationTranslator { // `HasError` returning true. class TokenizedBuffer : public Printable { public: - auto GetKind(Token token) const -> TokenKind; - auto GetLine(Token token) const -> Line; + auto GetKind(TokenIndex token) const -> TokenKind; + auto GetLine(TokenIndex token) const -> LineIndex; // Returns the 1-based line number. - auto GetLineNumber(Token token) const -> int; + auto GetLineNumber(TokenIndex token) const -> int; // Returns the 1-based column number. - auto GetColumnNumber(Token token) const -> int; + auto GetColumnNumber(TokenIndex token) const -> int; // Returns the source text lexed into this token. - auto GetTokenText(Token token) const -> llvm::StringRef; + auto GetTokenText(TokenIndex token) const -> llvm::StringRef; // Returns the identifier associated with this token. The token kind must be // an `Identifier`. - auto GetIdentifier(Token token) const -> IdentifierId; + auto GetIdentifier(TokenIndex token) const -> IdentifierId; // Returns the value of an `IntegerLiteral()` token. - auto GetIntegerLiteral(Token token) const -> IntegerId; + auto GetIntegerLiteral(TokenIndex token) const -> IntegerId; // Returns the value of an `RealLiteral()` token. - auto GetRealLiteral(Token token) const -> RealId; + auto GetRealLiteral(TokenIndex token) const -> RealId; // Returns the value of a `StringLiteral()` token. - auto GetStringLiteral(Token token) const -> StringLiteralId; + auto GetStringLiteral(TokenIndex token) const -> StringLiteralId; // Returns the size specified in a `*TypeLiteral()` token. - auto GetTypeLiteralSize(Token token) const -> const llvm::APInt&; + auto GetTypeLiteralSize(TokenIndex token) const -> const llvm::APInt&; // Returns the closing token matched with the given opening token. // // The given token must be an opening token kind. - auto GetMatchedClosingToken(Token opening_token) const -> Token; + auto GetMatchedClosingToken(TokenIndex opening_token) const -> TokenIndex; // Returns the opening token matched with the given closing token. // // The given token must be a closing token kind. - auto GetMatchedOpeningToken(Token closing_token) const -> Token; + auto GetMatchedOpeningToken(TokenIndex closing_token) const -> TokenIndex; // Returns whether the given token has leading whitespace. - auto HasLeadingWhitespace(Token token) const -> bool; + auto HasLeadingWhitespace(TokenIndex token) const -> bool; // Returns whether the given token has trailing whitespace. - auto HasTrailingWhitespace(Token token) const -> bool; + auto HasTrailingWhitespace(TokenIndex token) const -> bool; // Returns whether the token was created as part of an error recovery effort. // // For example, a closing paren inserted to match an unmatched paren. - auto IsRecoveryToken(Token token) const -> bool; + auto IsRecoveryToken(TokenIndex token) const -> bool; // Returns the 1-based line number. - auto GetLineNumber(Line line) const -> int; + auto GetLineNumber(LineIndex line) const -> int; // Returns the 1-based indentation column number. - auto GetIndentColumnNumber(Line line) const -> int; + auto GetIndentColumnNumber(LineIndex line) const -> int; // Returns the next line handle. - auto GetNextLine(Line line) const -> Line; + auto GetNextLine(LineIndex line) const -> LineIndex; // Returns the previous line handle. - auto GetPrevLine(Line line) const -> Line; + auto GetPrevLine(LineIndex line) const -> LineIndex; // Prints a description of the tokenized stream to the provided `raw_ostream`. // @@ -216,14 +218,15 @@ class TokenizedBuffer : public Printable { // Prints a description of a single token. See `Print` for details on the // format. - auto PrintToken(llvm::raw_ostream& output_stream, Token token) const -> void; + auto PrintToken(llvm::raw_ostream& output_stream, TokenIndex token) const + -> void; // Returns true if the buffer has errors that were detected at lexing time. auto has_errors() const -> bool { return has_errors_; } auto tokens() const -> llvm::iterator_range { - return llvm::make_range(TokenIterator(Token(0)), - TokenIterator(Token(token_infos_.size()))); + return llvm::make_range(TokenIterator(TokenIndex(0)), + TokenIterator(TokenIndex(token_infos_.size()))); } auto size() const -> int { return token_infos_.size(); } @@ -277,8 +280,8 @@ class TokenizedBuffer : public Printable { // Whether the token was injected artificially during error recovery. bool is_recovery = false; - // Line on which the Token starts. - Line token_line; + // LineIndex on which the TokenIndex starts. + LineIndex token_line; // Zero-based byte offset of the token within its line. int32_t column; @@ -286,15 +289,15 @@ class TokenizedBuffer : public Printable { // We may have up to 32 bits of payload, based on the kind of token. union { static_assert( - sizeof(Token) <= sizeof(int32_t), + sizeof(TokenIndex) <= sizeof(int32_t), "Unable to pack token and identifier index into the same space!"); IdentifierId ident_id = IdentifierId::Invalid; StringLiteralId string_literal_id; IntegerId integer_id; RealId real_id; - Token closing_token; - Token opening_token; + TokenIndex closing_token; + TokenIndex opening_token; int32_t error_length; }; }; @@ -330,14 +333,14 @@ class TokenizedBuffer : public Printable { SourceBuffer& source) : value_stores_(&value_stores), source_(&source) {} - auto GetLineInfo(Line line) -> LineInfo&; - auto GetLineInfo(Line line) const -> const LineInfo&; - auto AddLine(LineInfo info) -> Line; - auto GetTokenInfo(Token token) -> TokenInfo&; - auto GetTokenInfo(Token token) const -> const TokenInfo&; - auto AddToken(TokenInfo info) -> Token; - auto GetTokenPrintWidths(Token token) const -> PrintWidths; - auto PrintToken(llvm::raw_ostream& output_stream, Token token, + auto GetLineInfo(LineIndex line) -> LineInfo&; + auto GetLineInfo(LineIndex line) const -> const LineInfo&; + auto AddLine(LineInfo info) -> LineIndex; + auto GetTokenInfo(TokenIndex token) -> TokenInfo&; + auto GetTokenInfo(TokenIndex token) const -> const TokenInfo&; + auto AddToken(TokenInfo info) -> TokenIndex; + auto GetTokenPrintWidths(TokenIndex token) const -> PrintWidths; + auto PrintToken(llvm::raw_ostream& output_stream, TokenIndex token, PrintWidths widths) const -> void; // Used to allocate computed string literals. @@ -366,7 +369,7 @@ class TokenizedBuffer : public Printable { using LexerDiagnosticEmitter = DiagnosticEmitter; // A diagnostic emitter that uses tokens as its source of location information. -using TokenDiagnosticEmitter = DiagnosticEmitter; +using TokenDiagnosticEmitter = DiagnosticEmitter; } // namespace Carbon::Lex diff --git a/toolchain/lex/tokenized_buffer_fuzzer.cpp b/toolchain/lex/tokenized_buffer_fuzzer.cpp index dce6a6d4e755..07d7e3984f1e 100644 --- a/toolchain/lex/tokenized_buffer_fuzzer.cpp +++ b/toolchain/lex/tokenized_buffer_fuzzer.cpp @@ -44,7 +44,7 @@ extern "C" int LLVMFuzzerTestOneInput(const unsigned char* data, // // TODO: We should enhance this to do more sanity checks on the resulting // token stream. - for (Lex::Token token : buffer.tokens()) { + for (Lex::TokenIndex token : buffer.tokens()) { int line_number = buffer.GetLineNumber(token); CARBON_CHECK(line_number > 0) << "Invalid line number!"; CARBON_CHECK(line_number < INT_MAX) << "Invalid line number!"; diff --git a/toolchain/lex/tokenized_buffer_test.cpp b/toolchain/lex/tokenized_buffer_test.cpp index 229816c4788f..afe593918c06 100644 --- a/toolchain/lex/tokenized_buffer_test.cpp +++ b/toolchain/lex/tokenized_buffer_test.cpp @@ -588,7 +588,7 @@ TEST_F(LexerTest, Whitespace) { // EOF false}; int pos = 0; - for (Token token : buffer.tokens()) { + for (TokenIndex token : buffer.tokens()) { SCOPED_TRACE( llvm::formatv("Token #{0}: '{1}'", token, buffer.GetTokenText(token))); @@ -856,7 +856,7 @@ TEST_F(LexerTest, InvalidStringLiterals) { // We should have formed at least one error token. bool found_error = false; - for (Token token : buffer.tokens()) { + for (TokenIndex token : buffer.tokens()) { if (buffer.GetKind(token) == TokenKind::Error) { found_error = true; break; diff --git a/toolchain/parse/context.cpp b/toolchain/parse/context.cpp index bb3ac589030b..0e838b02840e 100644 --- a/toolchain/parse/context.cpp +++ b/toolchain/parse/context.cpp @@ -56,7 +56,7 @@ Context::Context(Tree& tree, Lex::TokenizedBuffer& tokens, << tokens_->GetKind(*end_); } -auto Context::AddLeafNode(NodeKind kind, Lex::Token token, bool has_error) +auto Context::AddLeafNode(NodeKind kind, Lex::TokenIndex token, bool has_error) -> void { CheckNodeMatchesLexerToken(kind, tokens_->GetKind(token), has_error); tree_->node_impls_.push_back( @@ -66,7 +66,7 @@ auto Context::AddLeafNode(NodeKind kind, Lex::Token token, bool has_error) } } -auto Context::AddNode(NodeKind kind, Lex::Token token, int subtree_start, +auto Context::AddNode(NodeKind kind, Lex::TokenIndex token, int subtree_start, bool has_error) -> void { CheckNodeMatchesLexerToken(kind, tokens_->GetKind(token), has_error); int subtree_size = tree_->size() - subtree_start + 1; @@ -77,9 +77,9 @@ auto Context::AddNode(NodeKind kind, Lex::Token token, int subtree_start, } } -auto Context::ConsumeAndAddOpenParen(Lex::Token default_token, +auto Context::ConsumeAndAddOpenParen(Lex::TokenIndex default_token, NodeKind start_kind) - -> std::optional { + -> std::optional { if (auto open_paren = ConsumeIf(Lex::TokenKind::OpenParen)) { AddLeafNode(start_kind, *open_paren, /*has_error=*/false); return open_paren; @@ -93,7 +93,7 @@ auto Context::ConsumeAndAddOpenParen(Lex::Token default_token, } } -auto Context::ConsumeAndAddCloseSymbol(Lex::Token expected_open, +auto Context::ConsumeAndAddCloseSymbol(Lex::TokenIndex expected_open, StateStackEntry state, NodeKind close_kind) -> void { Lex::TokenKind open_token_kind = tokens().GetKind(expected_open); @@ -126,13 +126,13 @@ auto Context::ConsumeAndAddLeafNodeIf(Lex::TokenKind token_kind, return true; } -auto Context::ConsumeChecked(Lex::TokenKind kind) -> Lex::Token { +auto Context::ConsumeChecked(Lex::TokenKind kind) -> Lex::TokenIndex { CARBON_CHECK(PositionIs(kind)) << "Required " << kind << ", found " << PositionKind(); return Consume(); } -auto Context::ConsumeIf(Lex::TokenKind kind) -> std::optional { +auto Context::ConsumeIf(Lex::TokenKind kind) -> std::optional { if (!PositionIs(kind)) { return std::nullopt; } @@ -150,10 +150,10 @@ auto Context::ConsumeIfPatternKeyword(Lex::TokenKind keyword_token, } auto Context::FindNextOf(std::initializer_list desired_kinds) - -> std::optional { + -> std::optional { auto new_position = position_; while (true) { - Lex::Token token = *new_position; + Lex::TokenIndex token = *new_position; Lex::TokenKind kind = tokens().GetKind(token); if (kind.IsOneOf(desired_kinds)) { return token; @@ -183,19 +183,19 @@ auto Context::SkipMatchingGroup() -> bool { return true; } -auto Context::SkipPastLikelyEnd(Lex::Token skip_root) - -> std::optional { +auto Context::SkipPastLikelyEnd(Lex::TokenIndex skip_root) + -> std::optional { if (position_ == end_) { return std::nullopt; } - Lex::Line root_line = tokens().GetLine(skip_root); + Lex::LineIndex root_line = tokens().GetLine(skip_root); int root_line_indent = tokens().GetIndentColumnNumber(root_line); // We will keep scanning through tokens on the same line as the root or // lines with greater indentation than root's line. - auto is_same_line_or_indent_greater_than_root = [&](Lex::Token t) { - Lex::Line l = tokens().GetLine(t); + auto is_same_line_or_indent_greater_than_root = [&](Lex::TokenIndex t) { + Lex::LineIndex l = tokens().GetLine(t); if (l == root_line) { return true; } @@ -229,7 +229,7 @@ auto Context::SkipPastLikelyEnd(Lex::Token skip_root) return std::nullopt; } -auto Context::SkipTo(Lex::Token t) -> void { +auto Context::SkipTo(Lex::TokenIndex t) -> void { CARBON_CHECK(t >= *position_) << "Tried to skip backwards from " << position_ << " to " << Lex::TokenIterator(t); position_ = Lex::TokenIterator(t); @@ -454,7 +454,7 @@ auto Context::PrintForStackDump(llvm::raw_ostream& output) const -> void { } auto Context::PrintTokenForStackDump(llvm::raw_ostream& output, - Lex::Token token) const -> void { + Lex::TokenIndex token) const -> void { output << " @ " << tokens_->GetLineNumber(tokens_->GetLine(token)) << ":" << tokens_->GetColumnNumber(token) << ": token " << token << " : " << tokens_->GetKind(token) << "\n"; diff --git a/toolchain/parse/context.h b/toolchain/parse/context.h index c6937be4511e..00c82766f92e 100644 --- a/toolchain/parse/context.h +++ b/toolchain/parse/context.h @@ -62,8 +62,8 @@ class Context { // Used to track state on state_stack_. struct StateStackEntry : public Printable { explicit StateStackEntry(State state, PrecedenceGroup ambient_precedence, - PrecedenceGroup lhs_precedence, Lex::Token token, - int32_t subtree_start) + PrecedenceGroup lhs_precedence, + Lex::TokenIndex token, int32_t subtree_start) : state(state), ambient_precedence(ambient_precedence), lhs_precedence(lhs_precedence), @@ -92,7 +92,7 @@ class Context { // A token providing context based on the subtree. This will typically be // the first token in the subtree, but may sometimes be a token within. It // will typically be used for the subtree's root node. - Lex::Token token; + Lex::TokenIndex token; // The offset within the Tree of the subtree start. int32_t subtree_start; }; @@ -114,15 +114,15 @@ class Context { llvm::raw_ostream* vlog_stream); // Adds a node to the parse tree that has no children (a leaf). - auto AddLeafNode(NodeKind kind, Lex::Token token, bool has_error = false) + auto AddLeafNode(NodeKind kind, Lex::TokenIndex token, bool has_error = false) -> void; // Adds a node to the parse tree that has children. - auto AddNode(NodeKind kind, Lex::Token token, int subtree_start, + auto AddNode(NodeKind kind, Lex::TokenIndex token, int subtree_start, bool has_error) -> void; // Returns the current position and moves past it. - auto Consume() -> Lex::Token { return *(position_++); } + auto Consume() -> Lex::TokenIndex { return *(position_++); } // Consumes the current token. Does not return it. auto ConsumeAndDiscard() -> void { ++position_; } @@ -130,16 +130,18 @@ class Context { // Parses an open paren token, possibly diagnosing if necessary. Creates a // leaf parse node of the specified start kind. The default_token is used when // there's no open paren. Returns the open paren token if it was found. - auto ConsumeAndAddOpenParen(Lex::Token default_token, NodeKind start_kind) - -> std::optional; + auto ConsumeAndAddOpenParen(Lex::TokenIndex default_token, + NodeKind start_kind) + -> std::optional; // Parses a closing symbol corresponding to the opening symbol // `expected_open`, possibly skipping forward and diagnosing if necessary. // Creates a parse node of the specified close kind. If `expected_open` is not // an opening symbol, the parse node will be associated with `state.token`, // no input will be consumed, and no diagnostic will be emitted. - auto ConsumeAndAddCloseSymbol(Lex::Token expected_open, StateStackEntry state, - NodeKind close_kind) -> void; + auto ConsumeAndAddCloseSymbol(Lex::TokenIndex expected_open, + StateStackEntry state, NodeKind close_kind) + -> void; // Composes `ConsumeIf` and `AddLeafNode`, returning false when ConsumeIf // fails. @@ -148,16 +150,16 @@ class Context { // Returns the current position and moves past it. Requires the token is the // expected kind. - auto ConsumeChecked(Lex::TokenKind kind) -> Lex::Token; + auto ConsumeChecked(Lex::TokenKind kind) -> Lex::TokenIndex; // If the current position's token matches this `Kind`, returns it and // advances to the next position. Otherwise returns an empty optional. - auto ConsumeIf(Lex::TokenKind kind) -> std::optional; + auto ConsumeIf(Lex::TokenKind kind) -> std::optional; // Find the next token of any of the given kinds at the current bracketing // level. auto FindNextOf(std::initializer_list desired_kinds) - -> std::optional; + -> std::optional; // If the token is an opening symbol for a matched group, skips to the matched // closing symbol and returns true. Otherwise, returns false. @@ -179,10 +181,11 @@ class Context { // declarations or statements across multiple lines should be indented. // // Returns a semicolon token if one is the likely end. - auto SkipPastLikelyEnd(Lex::Token skip_root) -> std::optional; + auto SkipPastLikelyEnd(Lex::TokenIndex skip_root) + -> std::optional; // Skip forward to the given token. Verifies that it is actually forward. - auto SkipTo(Lex::Token t) -> void; + auto SkipTo(Lex::TokenIndex t) -> void; // Returns true if the current token satisfies the lexical validity rules // for an infix operator. @@ -243,7 +246,7 @@ class Context { // Pushes a new state with a specific token for context. Used when forming a // new subtree with a token that isn't the start of the subtree. - auto PushState(State state, Lex::Token token) -> void { + auto PushState(State state, Lex::TokenIndex token) -> void { PushState(StateStackEntry(state, PrecedenceGroup::ForTopLevelExpr(), PrecedenceGroup::ForTopLevelExpr(), token, tree_->size())); @@ -337,18 +340,18 @@ class Context { auto set_packaging_state(PackagingState packaging_state) -> void { packaging_state_ = packaging_state; } - auto first_non_packaging_token() const -> Lex::Token { + auto first_non_packaging_token() const -> Lex::TokenIndex { return first_non_packaging_token_; } - auto set_first_non_packaging_token(Lex::Token token) -> void { + auto set_first_non_packaging_token(Lex::TokenIndex token) -> void { CARBON_CHECK(!first_non_packaging_token_.is_valid()); first_non_packaging_token_ = token; } private: // Prints a single token for a stack dump. Used by PrintForStackDump. - auto PrintTokenForStackDump(llvm::raw_ostream& output, Lex::Token token) const - -> void; + auto PrintTokenForStackDump(llvm::raw_ostream& output, + Lex::TokenIndex token) const -> void; Tree* tree_; Lex::TokenizedBuffer* tokens_; @@ -368,7 +371,7 @@ class Context { PackagingState packaging_state_ = PackagingState::FileStart; // The first non-packaging token, starting as invalid. Used for packaging // state warnings. - Lex::Token first_non_packaging_token_ = Lex::Token::Invalid; + Lex::TokenIndex first_non_packaging_token_ = Lex::TokenIndex::Invalid; }; // `clang-format` has a bug with spacing around `->` returns in macros. See diff --git a/toolchain/parse/handle_decl_name_and_params.cpp b/toolchain/parse/handle_decl_name_and_params.cpp index 75adae066cd2..55e14b762770 100644 --- a/toolchain/parse/handle_decl_name_and_params.cpp +++ b/toolchain/parse/handle_decl_name_and_params.cpp @@ -27,7 +27,7 @@ static auto HandleDeclNameAndParams(Context& context, State after_name) CARBON_DIAGNOSTIC(ExpectedDeclName, Error, "`{0}` introducer should be followed by a name.", Lex::TokenKind); - Lex::Token location = *context.position(); + Lex::TokenIndex location = *context.position(); if (context.tokens().GetKind(location) == Lex::TokenKind::FileEnd) { // The end of file is often an especially unhelpful location. If that's // the best we can do here, back up the location to the introducer itself. diff --git a/toolchain/parse/handle_import_and_package.cpp b/toolchain/parse/handle_import_and_package.cpp index dfc408581cbc..4dd97a9b9254 100644 --- a/toolchain/parse/handle_import_and_package.cpp +++ b/toolchain/parse/handle_import_and_package.cpp @@ -212,7 +212,7 @@ static auto HandlePackageAndLibraryDirectives(Context& context, auto intro_token = context.ConsumeChecked(intro_token_kind); context.AddLeafNode(intro, intro_token); - if (intro_token != Lex::Token::FirstNonCommentToken) { + if (intro_token != Lex::TokenIndex::FirstNonCommentToken) { CARBON_DIAGNOSTIC(PackageTooLate, Error, "The `{0}` directive must be the first non-comment line.", Lex::TokenKind); @@ -220,7 +220,7 @@ static auto HandlePackageAndLibraryDirectives(Context& context, "First non-comment line is here."); context.emitter() .Build(intro_token, PackageTooLate, intro_token_kind) - .Note(Lex::Token::FirstNonCommentToken, FirstNonCommentLine) + .Note(Lex::TokenIndex::FirstNonCommentToken, FirstNonCommentLine) .Emit(); on_parse_error(); return; diff --git a/toolchain/parse/handle_paren_condition.cpp b/toolchain/parse/handle_paren_condition.cpp index 139936e29e2e..de94cbf9cc37 100644 --- a/toolchain/parse/handle_paren_condition.cpp +++ b/toolchain/parse/handle_paren_condition.cpp @@ -11,7 +11,7 @@ static auto HandleParenCondition(Context& context, NodeKind start_kind, State finish_state) -> void { auto state = context.PopState(); - std::optional open_paren = + std::optional open_paren = context.ConsumeAndAddOpenParen(state.token, start_kind); if (open_paren) { state.token = *open_paren; diff --git a/toolchain/parse/handle_statement.cpp b/toolchain/parse/handle_statement.cpp index 20843e513b1d..cb4a44ec85fb 100644 --- a/toolchain/parse/handle_statement.cpp +++ b/toolchain/parse/handle_statement.cpp @@ -91,7 +91,7 @@ auto HandleStatementContinueFinish(Context& context) -> void { auto HandleStatementForHeader(Context& context) -> void { auto state = context.PopState(); - std::optional open_paren = + std::optional open_paren = context.ConsumeAndAddOpenParen(state.token, NodeKind::ForHeaderStart); if (open_paren) { state.token = *open_paren; diff --git a/toolchain/parse/handle_var.cpp b/toolchain/parse/handle_var.cpp index 05a4b519cf60..a06135e85812 100644 --- a/toolchain/parse/handle_var.cpp +++ b/toolchain/parse/handle_var.cpp @@ -8,7 +8,8 @@ namespace Carbon::Parse { // Handles VarAs(Decl|For). static auto HandleVar(Context& context, State finish_state, - Lex::Token returned_token = Lex::Token::Invalid) -> void { + Lex::TokenIndex returned_token = Lex::TokenIndex::Invalid) + -> void { auto state = context.PopState(); // The finished variable declaration will start at the `var` or `returned`. diff --git a/toolchain/parse/node_kind.def b/toolchain/parse/node_kind.def index 338d704a9e46..0e4d416b9eeb 100644 --- a/toolchain/parse/node_kind.def +++ b/toolchain/parse/node_kind.def @@ -17,7 +17,7 @@ // - CARBON_PARSE_NODE_KIND_CHILD_COUNT(Name, ChildCount, LexTokenKinds) // Defines a parse node with a set number of children, often 0. This count // must be correct even when the node contains errors. -// - CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(Name, Token) +// - CARBON_PARSE_NODE_KIND_TOKEN_LITERAL(Name, TokenIndex) // Defines a parse node that corresponds to a token that is a single-token // literal. The token is wrapped for LexTokenKinds. // diff --git a/toolchain/parse/tree.cpp b/toolchain/parse/tree.cpp index 645d4f23bf74..b8505c6aed89 100644 --- a/toolchain/parse/tree.cpp +++ b/toolchain/parse/tree.cpp @@ -93,7 +93,7 @@ auto Tree::node_kind(NodeId n) const -> NodeKind { return node_impls_[n.index].kind; } -auto Tree::node_token(NodeId n) const -> Lex::Token { +auto Tree::node_token(NodeId n) const -> Lex::TokenIndex { CARBON_CHECK(n.is_valid()); return node_impls_[n.index].token; } diff --git a/toolchain/parse/tree.h b/toolchain/parse/tree.h index de4844f8d745..74e3a94b9638 100644 --- a/toolchain/parse/tree.h +++ b/toolchain/parse/tree.h @@ -119,7 +119,7 @@ class Tree : public Printable { auto node_kind(NodeId n) const -> NodeKind; // Returns the token the given parse tree node models. - auto node_token(NodeId n) const -> Lex::Token; + auto node_token(NodeId n) const -> Lex::TokenIndex; auto node_subtree_size(NodeId n) const -> int32_t; @@ -185,7 +185,7 @@ class Tree : public Printable { // The in-memory representation of data used for a particular node in the // tree. struct NodeImpl { - explicit NodeImpl(NodeKind kind, bool has_error, Lex::Token token, + explicit NodeImpl(NodeKind kind, bool has_error, Lex::TokenIndex token, int subtree_size) : kind(kind), has_error(has_error), @@ -214,7 +214,7 @@ class Tree : public Printable { bool has_error = false; // The token root of this node. - Lex::Token token; + Lex::TokenIndex token; // The size of this node's subtree of the parse tree. This is the number of // nodes (and thus tokens) that are covered by this node (and its