diff --git a/toolchain/lex/tokenized_buffer.cpp b/toolchain/lex/tokenized_buffer.cpp index 9a1cf17aba87..d7dae43d8008 100644 --- a/toolchain/lex/tokenized_buffer.cpp +++ b/toolchain/lex/tokenized_buffer.cpp @@ -242,12 +242,13 @@ class [[clang::internal_linkage]] TokenizedBuffer::Lexer { bool formed_token_; }; - Lexer(TokenizedBuffer& buffer, DiagnosticConsumer& consumer) - : buffer_(&buffer), - translator_(&buffer), - emitter_(translator_, consumer), - token_translator_(&buffer), - token_emitter_(token_translator_, consumer) {} + Lexer(SourceBuffer& source, DiagnosticConsumer& consumer) + : buffer_(source), + consumer_(consumer), + translator_(&buffer_), + emitter_(translator_, consumer_), + token_translator_(&buffer_), + token_emitter_(token_translator_, consumer_) {} // Find all line endings and create the line data structures. Explicitly kept // out-of-line because this is a significant loop that is useful to have in @@ -270,149 +271,159 @@ class [[clang::internal_linkage]] TokenizedBuffer::Lexer { while (const char* nl = reinterpret_cast( memchr(&text[start], '\n', size - start))) { ssize_t nl_index = nl - text; - buffer_->AddLine(LineInfo(start, nl_index - start)); + buffer_.AddLine(LineInfo(start, nl_index - start)); start = nl_index + 1; } // The last line ends at the end of the file. - buffer_->AddLine(LineInfo(start, size - start)); + buffer_.AddLine(LineInfo(start, size - start)); // Now that all the infos are allocated, get a fresh pointer to the first // info for use while lexing. - current_line_ = Line(0); - current_line_info_ = &buffer_->GetLineInfo(current_line_); + line_index_ = 0; + } + + auto current_line() -> Line { return Line(line_index_); } + + auto current_line_info() -> LineInfo* { + return &buffer_.line_infos_[line_index_]; + } + + auto ComputeColumn(ssize_t position) -> int { + CARBON_DCHECK(position >= current_line_info()->start); + return position - current_line_info()->start; } // Perform the necessary bookkeeping to step past a newline at the current // line and column. - auto HandleNewline() -> void { - int next_start = current_line_info_->start + current_column_ + 1; - current_line_ = buffer_->GetNextLine(current_line_); - current_line_info_ = &buffer_->GetLineInfo(current_line_); - CARBON_DCHECK(next_start == current_line_info_->start); - current_column_ = 0; + auto HandleNewline(llvm::StringRef source_text, ssize_t& position) -> void { + CARBON_DCHECK(source_text[position] == '\n'); + CARBON_DCHECK(current_line_info()->start + current_line_info()->length == + position); + ++position; + ++line_index_; + CARBON_DCHECK(current_line_info()->start = position); set_indent_ = false; } auto NoteWhitespace() -> void { - buffer_->token_infos_.back().has_trailing_space = true; + buffer_.token_infos_.back().has_trailing_space = true; } - auto LexHorizontalWhitespace(llvm::StringRef& source_text) -> void { - CARBON_DCHECK(source_text.front() == ' ' || source_text.front() == '\t'); + auto LexHorizontalWhitespace(llvm::StringRef source_text, ssize_t& position) + -> void { + CARBON_DCHECK(source_text[position] == ' ' || + source_text[position] == '\t'); NoteWhitespace(); // Handle adjacent whitespace quickly. This comes up frequently for example // due to indentation. We don't expect *huge* runs, so just use a scalar // loop. While still scalar, this avoids repeated table dispatch and marking // whitespace. We use `ssize_t` in the loop for performance. - ssize_t ws_count = 1; - ssize_t size = source_text.size(); - while (ws_count < size && - (source_text[ws_count] == ' ' || source_text[ws_count] == '\t')) { - ++ws_count; + while (position < static_cast(source_text.size()) && + (source_text[position] == ' ' || source_text[position] == '\t')) { + ++position; } - current_column_ += ws_count; - source_text = source_text.drop_front(ws_count); } - auto LexVerticalWhitespace(llvm::StringRef& source_text) -> void { - CARBON_DCHECK(source_text.front() == '\n'); + auto LexVerticalWhitespace(llvm::StringRef source_text, ssize_t& position) + -> void { NoteWhitespace(); - source_text = source_text.drop_front(); - HandleNewline(); + HandleNewline(source_text, position); } - auto LexCommentOrSlash(llvm::StringRef& source_text) -> void { - CARBON_DCHECK(source_text.front() == '/'); + auto LexCommentOrSlash(llvm::StringRef source_text, ssize_t& position) + -> void { + CARBON_DCHECK(source_text[position] == '/'); // Both comments and slash symbols start with a `/`. We disambiguate with a // max-munch rule -- if the next character is another `/` then we lex it as // a comment start. If it isn't, then we lex as a slash. - if (source_text.size() > 1 && source_text[1] == '/') { - LexComment(source_text); + if (position + 1 < static_cast(source_text.size()) && + source_text[position + 1] == '/') { + LexComment(source_text, position); return; } // This code path should produce a token, make sure that happens. - LexResult result = LexSymbolToken(source_text); + LexResult result = LexSymbolToken(source_text, position); CARBON_CHECK(result) << "Failed to form a token!"; } - auto LexComment(llvm::StringRef& source_text) -> void { - CARBON_DCHECK(source_text.startswith("//")); + auto LexComment(llvm::StringRef source_text, ssize_t& position) -> void { + CARBON_DCHECK(source_text.substr(position).startswith("//")); // Any comment must be the only non-whitespace on the line. if (set_indent_) { CARBON_DIAGNOSTIC(TrailingComment, Error, "Trailing comments are not permitted."); - emitter_.Emit(source_text.begin(), TrailingComment); + emitter_.Emit(source_text.begin() + position, TrailingComment); } // The introducer '//' must be followed by whitespace or EOF. - if (source_text.size() > 2 && !IsSpace(source_text[2])) { + if (position + 2 < static_cast(source_text.size()) && + !IsSpace(source_text[position + 2])) { CARBON_DIAGNOSTIC(NoWhitespaceAfterCommentIntroducer, Error, "Whitespace is required after '//'."); - emitter_.Emit(source_text.begin() + 2, + emitter_.Emit(source_text.begin() + position + 2, NoWhitespaceAfterCommentIntroducer); } // Use the current line info to jump to the end of the line. - source_text = - source_text.drop_front(current_line_info_->length - current_column_); + position = current_line_info()->start + current_line_info()->length; // This may be the end of the file in which case we immediately return. - if (source_text.empty()) { + if (position == static_cast(source_text.size())) { // Finished lexing. return; } // Otherwise, lex the newline. - current_column_ = current_line_info_->length; - LexVerticalWhitespace(source_text); + LexVerticalWhitespace(source_text, position); } - auto LexNumericLiteral(llvm::StringRef& source_text) -> LexResult { - std::optional literal = NumericLiteral::Lex(source_text); + auto LexNumericLiteral(llvm::StringRef source_text, ssize_t& position) + -> LexResult { + std::optional literal = + NumericLiteral::Lex(source_text.substr(position)); if (!literal) { - return LexError(source_text); + return LexError(source_text, position); } - int int_column = current_column_; + int int_column = ComputeColumn(position); int token_size = literal->text().size(); - current_column_ += token_size; - source_text = source_text.drop_front(token_size); + position += token_size; if (!set_indent_) { - current_line_info_->indent = int_column; + current_line_info()->indent = int_column; set_indent_ = true; } return VariantMatch( literal->ComputeValue(emitter_), [&](NumericLiteral::IntegerValue&& value) { - auto token = buffer_->AddToken({.kind = TokenKind::IntegerLiteral, - .token_line = current_line_, - .column = int_column}); - buffer_->GetTokenInfo(token).literal_index = - buffer_->literal_int_storage_.size(); - buffer_->literal_int_storage_.push_back(std::move(value.value)); + auto token = buffer_.AddToken({.kind = TokenKind::IntegerLiteral, + .token_line = current_line(), + .column = int_column}); + buffer_.GetTokenInfo(token).literal_index = + buffer_.literal_int_storage_.size(); + buffer_.literal_int_storage_.push_back(std::move(value.value)); return token; }, [&](NumericLiteral::RealValue&& value) { - auto token = buffer_->AddToken({.kind = TokenKind::RealLiteral, - .token_line = current_line_, - .column = int_column}); - buffer_->GetTokenInfo(token).literal_index = - buffer_->literal_int_storage_.size(); - buffer_->literal_int_storage_.push_back(std::move(value.mantissa)); - buffer_->literal_int_storage_.push_back(std::move(value.exponent)); - CARBON_CHECK(buffer_->GetRealLiteral(token).is_decimal == + auto token = buffer_.AddToken({.kind = TokenKind::RealLiteral, + .token_line = current_line(), + .column = int_column}); + buffer_.GetTokenInfo(token).literal_index = + buffer_.literal_int_storage_.size(); + buffer_.literal_int_storage_.push_back(std::move(value.mantissa)); + buffer_.literal_int_storage_.push_back(std::move(value.exponent)); + CARBON_CHECK(buffer_.GetRealLiteral(token).is_decimal == (value.radix == NumericLiteral::Radix::Decimal)); return token; }, [&](NumericLiteral::UnrecoverableError) { - auto token = buffer_->AddToken({ + auto token = buffer_.AddToken({ .kind = TokenKind::Error, - .token_line = current_line_, + .token_line = current_line(), .column = int_column, .error_length = token_size, }); @@ -420,102 +431,106 @@ class [[clang::internal_linkage]] TokenizedBuffer::Lexer { }); } - auto LexStringLiteral(llvm::StringRef& source_text) -> LexResult { - std::optional literal = StringLiteral::Lex(source_text); + auto LexStringLiteral(llvm::StringRef source_text, ssize_t& position) + -> LexResult { + std::optional literal = + StringLiteral::Lex(source_text.substr(position)); if (!literal) { - return LexError(source_text); + return LexError(source_text, position); } - Line string_line = current_line_; - int string_column = current_column_; - int literal_size = literal->text().size(); - source_text = source_text.drop_front(literal_size); + Line string_line = current_line(); + int string_column = ComputeColumn(position); + ssize_t literal_size = literal->text().size(); + position += literal_size; if (!set_indent_) { - current_line_info_->indent = string_column; + current_line_info()->indent = string_column; set_indent_ = true; } // Update line and column information. - if (!literal->is_multi_line()) { - current_column_ += literal_size; - } else { - for (char c : literal->text()) { - if (c == '\n') { - HandleNewline(); - // The indentation of all lines in a multi-line string literal is - // that of the first line. - current_line_info_->indent = string_column; - set_indent_ = true; - } else { - ++current_column_; - } + if (literal->is_multi_line()) { + while (current_line_info()->start + current_line_info()->length < + position) { + ++line_index_; + current_line_info()->indent = string_column; } + // Note that we've updated the current line at this point, but + // `set_indent_` is already true from above. That remains correct as the + // last line of the multi-line literal *also* has its indent set. } if (literal->is_terminated()) { auto token = - buffer_->AddToken({.kind = TokenKind::StringLiteral, - .token_line = string_line, - .column = string_column, - .literal_index = static_cast( - buffer_->literal_string_storage_.size())}); - buffer_->literal_string_storage_.push_back( + buffer_.AddToken({.kind = TokenKind::StringLiteral, + .token_line = string_line, + .column = string_column, + .literal_index = static_cast( + buffer_.literal_string_storage_.size())}); + buffer_.literal_string_storage_.push_back( literal->ComputeValue(emitter_)); return token; } else { CARBON_DIAGNOSTIC(UnterminatedString, Error, "String is missing a terminator."); emitter_.Emit(literal->text().begin(), UnterminatedString); - return buffer_->AddToken({.kind = TokenKind::Error, - .token_line = string_line, - .column = string_column, - .error_length = literal_size}); + return buffer_.AddToken( + {.kind = TokenKind::Error, + .token_line = string_line, + .column = string_column, + .error_length = static_cast(literal_size)}); } } - auto LexOneCharSymbolToken(llvm::StringRef& source_text, TokenKind kind) - -> Token { + auto LexOneCharSymbolToken(llvm::StringRef source_text, TokenKind kind, + ssize_t& position) -> Token { // Verify in a debug build that the incoming token kind is correct. CARBON_DCHECK(kind != TokenKind::Error); CARBON_DCHECK(kind.fixed_spelling().size() == 1); - CARBON_DCHECK(source_text.front() == kind.fixed_spelling().front()) - << "Source text starts with '" << source_text.front() + CARBON_DCHECK(source_text[position] == kind.fixed_spelling().front()) + << "Source text starts with '" << source_text[position] << "' instead of the spelling '" << kind.fixed_spelling() << "' of the incoming token kind '" << kind << "'"; + int column = ComputeColumn(position); if (!set_indent_) { - current_line_info_->indent = current_column_; + current_line_info()->indent = column; set_indent_ = true; } - Token token = buffer_->AddToken( - {.kind = kind, .token_line = current_line_, .column = current_column_}); - ++current_column_; - source_text = source_text.drop_front(); + Token token = buffer_.AddToken( + {.kind = kind, .token_line = current_line(), .column = column}); + ++position; return token; } - auto LexOpeningSymbolToken(llvm::StringRef& source_text, TokenKind kind) - -> LexResult { - Token token = LexOneCharSymbolToken(source_text, kind); + auto LexOpeningSymbolToken(llvm::StringRef source_text, TokenKind kind, + ssize_t& position) -> LexResult { + Token token = LexOneCharSymbolToken(source_text, kind, position); open_groups_.push_back(token); return token; } - auto LexClosingSymbolToken(llvm::StringRef& source_text, TokenKind kind) - -> LexResult { + auto LexClosingSymbolToken(llvm::StringRef source_text, TokenKind kind, + ssize_t& position) -> LexResult { auto unmatched_error = [&] { + int column = ComputeColumn(position); + // We still need to set the indent here, and can't use the common code + // used by the non-error path. + if (!set_indent_) { + current_line_info()->indent = column; + set_indent_ = true; + } CARBON_DIAGNOSTIC( UnmatchedClosing, Error, "Closing symbol without a corresponding opening symbol."); - emitter_.Emit(source_text.begin(), UnmatchedClosing); - Token token = buffer_->AddToken({.kind = TokenKind::Error, - .token_line = current_line_, - .column = current_column_, - .error_length = 1}); - ++current_column_; - source_text = source_text.drop_front(); + emitter_.Emit(source_text.begin() + position, UnmatchedClosing); + Token token = buffer_.AddToken({.kind = TokenKind::Error, + .token_line = current_line(), + .column = column, + .error_length = 1}); + ++position; return token; }; @@ -526,34 +541,35 @@ class [[clang::internal_linkage]] TokenizedBuffer::Lexer { Token opening_token = open_groups_.back(); // Close any invalid open groups first. - if (LLVM_UNLIKELY(buffer_->GetTokenInfo(opening_token).kind != + if (LLVM_UNLIKELY(buffer_.GetTokenInfo(opening_token).kind != kind.opening_symbol())) { - CloseInvalidOpenGroups(kind); + CloseInvalidOpenGroups(kind, position); // This may exhaust the open groups so re-check and re-error if needed. if (open_groups_.empty()) { return unmatched_error(); } opening_token = open_groups_.back(); - CARBON_DCHECK(buffer_->GetTokenInfo(opening_token).kind == + CARBON_DCHECK(buffer_.GetTokenInfo(opening_token).kind == kind.opening_symbol()); } open_groups_.pop_back(); // Now that the groups are all matched up, lex the actual token. - Token token = LexOneCharSymbolToken(source_text, kind); + Token token = LexOneCharSymbolToken(source_text, kind, position); // Note that it is important to get fresh token infos here as lexing the // open token would invalidate any pointers. - buffer_->GetTokenInfo(opening_token).closing_token = token; - buffer_->GetTokenInfo(token).opening_token = opening_token; + buffer_.GetTokenInfo(opening_token).closing_token = token; + buffer_.GetTokenInfo(token).opening_token = opening_token; return token; } - auto LexSymbolToken(llvm::StringRef& source_text) -> LexResult { + auto LexSymbolToken(llvm::StringRef source_text, ssize_t& position) + -> LexResult { // One character symbols and grouping symbols are handled with dedicated // dispatch. We only lex the multi-character tokens here. - TokenKind kind = llvm::StringSwitch(source_text) + TokenKind kind = llvm::StringSwitch(source_text.substr(position)) #define CARBON_SYMBOL_TOKEN(Name, Spelling) \ .StartsWith(Spelling, TokenKind::Name) #define CARBON_ONE_CHAR_SYMBOL_TOKEN(TokenName, Spelling) @@ -562,18 +578,18 @@ class [[clang::internal_linkage]] TokenizedBuffer::Lexer { #include "toolchain/lex/token_kind.def" .Default(TokenKind::Error); if (kind == TokenKind::Error) { - return LexError(source_text); + return LexError(source_text, position); } + int column = ComputeColumn(position); if (!set_indent_) { - current_line_info_->indent = current_column_; + current_line_info()->indent = column; set_indent_ = true; } - Token token = buffer_->AddToken( - {.kind = kind, .token_line = current_line_, .column = current_column_}); - current_column_ += kind.fixed_spelling().size(); - source_text = source_text.drop_front(kind.fixed_spelling().size()); + Token token = buffer_.AddToken( + {.kind = kind, .token_line = current_line(), .column = column}); + position += kind.fixed_spelling().size(); return token; } @@ -607,9 +623,9 @@ class [[clang::internal_linkage]] TokenizedBuffer::Lexer { llvm::StringRef suffix = word.substr(1); if (!CanLexInteger(emitter_, suffix)) { - return buffer_->AddToken( + return buffer_.AddToken( {.kind = TokenKind::Error, - .token_line = current_line_, + .token_line = current_line(), .column = column, .error_length = static_cast(word.size())}); } @@ -618,23 +634,26 @@ class [[clang::internal_linkage]] TokenizedBuffer::Lexer { return LexResult::NoMatch(); } - auto token = buffer_->AddToken( - {.kind = *kind, .token_line = current_line_, .column = column}); - buffer_->GetTokenInfo(token).literal_index = - buffer_->literal_int_storage_.size(); - buffer_->literal_int_storage_.push_back(std::move(suffix_value)); + auto token = buffer_.AddToken( + {.kind = *kind, .token_line = current_line(), .column = column}); + buffer_.GetTokenInfo(token).literal_index = + buffer_.literal_int_storage_.size(); + buffer_.literal_int_storage_.push_back(std::move(suffix_value)); return token; } // Closes all open groups that cannot remain open across a closing symbol. // Users may pass `Error` to close all open groups. - [[gnu::noinline]] auto CloseInvalidOpenGroups(TokenKind kind) -> void { + [[gnu::noinline]] auto CloseInvalidOpenGroups(TokenKind kind, + ssize_t position) -> void { CARBON_CHECK(kind.is_closing_symbol() || kind == TokenKind::Error); CARBON_CHECK(!open_groups_.empty()); + int column = ComputeColumn(position); + do { Token opening_token = open_groups_.back(); - TokenKind opening_kind = buffer_->GetTokenInfo(opening_token).kind; + TokenKind opening_kind = buffer_.GetTokenInfo(opening_token).kind; if (kind == opening_kind.closing_symbol()) { return; } @@ -645,57 +664,58 @@ class [[clang::internal_linkage]] TokenizedBuffer::Lexer { "Closing symbol does not match most recent opening symbol."); token_emitter_.Emit(opening_token, MismatchedClosing); - CARBON_CHECK(!buffer_->tokens().empty()) + CARBON_CHECK(!buffer_.tokens().empty()) << "Must have a prior opening token!"; - Token prev_token = buffer_->tokens().end()[-1]; + Token prev_token = buffer_.tokens().end()[-1]; // TODO: do a smarter backwards scan for where to put the closing // token. - Token closing_token = buffer_->AddToken( + Token closing_token = buffer_.AddToken( {.kind = opening_kind.closing_symbol(), - .has_trailing_space = buffer_->HasTrailingWhitespace(prev_token), + .has_trailing_space = buffer_.HasTrailingWhitespace(prev_token), .is_recovery = true, - .token_line = current_line_, - .column = current_column_}); - TokenInfo& opening_token_info = buffer_->GetTokenInfo(opening_token); - TokenInfo& closing_token_info = buffer_->GetTokenInfo(closing_token); + .token_line = current_line(), + .column = column}); + TokenInfo& opening_token_info = buffer_.GetTokenInfo(opening_token); + TokenInfo& closing_token_info = buffer_.GetTokenInfo(closing_token); opening_token_info.closing_token = closing_token; closing_token_info.opening_token = opening_token; } while (!open_groups_.empty()); } auto GetOrCreateIdentifier(llvm::StringRef text) -> Identifier { - auto insert_result = buffer_->identifier_map_.insert( - {text, Identifier(buffer_->identifier_infos_.size())}); + auto insert_result = buffer_.identifier_map_.insert( + {text, Identifier(buffer_.identifier_infos_.size())}); if (insert_result.second) { - buffer_->identifier_infos_.push_back({text}); + buffer_.identifier_infos_.push_back({text}); } return insert_result.first->second; } - auto LexKeywordOrIdentifier(llvm::StringRef& source_text) -> LexResult { - if (static_cast(source_text.front()) > 0x7F) { + auto LexKeywordOrIdentifier(llvm::StringRef source_text, ssize_t& position) + -> LexResult { + if (static_cast(source_text[position]) > 0x7F) { // TODO: Need to add support for Unicode lexing. - return LexError(source_text); + return LexError(source_text, position); } - CARBON_CHECK(IsAlpha(source_text.front()) || source_text.front() == '_'); + CARBON_CHECK(IsAlpha(source_text[position]) || + source_text[position] == '_'); + int column = ComputeColumn(position); if (!set_indent_) { - current_line_info_->indent = current_column_; + current_line_info()->indent = column; set_indent_ = true; } // Take the valid characters off the front of the source buffer. - llvm::StringRef identifier_text = ScanForIdentifierPrefix(source_text); + llvm::StringRef identifier_text = + ScanForIdentifierPrefix(source_text.substr(position)); CARBON_CHECK(!identifier_text.empty()) << "Must have at least one character!"; - int identifier_column = current_column_; - current_column_ += identifier_text.size(); - source_text = source_text.drop_front(identifier_text.size()); + position += identifier_text.size(); // Check if the text is a type literal, and if so form such a literal. - if (LexResult result = - LexWordAsTypeLiteralToken(identifier_text, identifier_column)) { + if (LexResult result = LexWordAsTypeLiteralToken(identifier_text, column)) { return result; } @@ -705,82 +725,79 @@ class [[clang::internal_linkage]] TokenizedBuffer::Lexer { #include "toolchain/lex/token_kind.def" .Default(TokenKind::Error); if (kind != TokenKind::Error) { - return buffer_->AddToken({.kind = kind, - .token_line = current_line_, - .column = identifier_column}); + return buffer_.AddToken( + {.kind = kind, .token_line = current_line(), .column = column}); } // Otherwise we have a generic identifier. - return buffer_->AddToken({.kind = TokenKind::Identifier, - .token_line = current_line_, - .column = identifier_column, - .id = GetOrCreateIdentifier(identifier_text)}); + return buffer_.AddToken({.kind = TokenKind::Identifier, + .token_line = current_line(), + .column = column, + .id = GetOrCreateIdentifier(identifier_text)}); } - auto LexError(llvm::StringRef& source_text) -> LexResult { - llvm::StringRef error_text = source_text.take_while([](char c) { - if (IsAlnum(c)) { - return false; - } - switch (c) { - case '_': - case '\t': - case '\n': - return false; - default: - break; - } - return llvm::StringSwitch(llvm::StringRef(&c, 1)) + auto LexError(llvm::StringRef source_text, ssize_t& position) -> LexResult { + llvm::StringRef error_text = + source_text.substr(position).take_while([](char c) { + if (IsAlnum(c)) { + return false; + } + switch (c) { + case '_': + case '\t': + case '\n': + return false; + default: + break; + } + return llvm::StringSwitch(llvm::StringRef(&c, 1)) #define CARBON_SYMBOL_TOKEN(Name, Spelling) .StartsWith(Spelling, false) #include "toolchain/lex/token_kind.def" - .Default(true); - }); + .Default(true); + }); if (error_text.empty()) { // TODO: Reimplement this to use the lexer properly. In the meantime, // guarantee that we eat at least one byte. - error_text = source_text.take_front(1); + error_text = source_text.substr(position, 1); } - auto token = buffer_->AddToken( + auto token = buffer_.AddToken( {.kind = TokenKind::Error, - .token_line = current_line_, - .column = current_column_, + .token_line = current_line(), + .column = ComputeColumn(position), .error_length = static_cast(error_text.size())}); CARBON_DIAGNOSTIC(UnrecognizedCharacters, Error, "Encountered unrecognized characters while parsing."); emitter_.Emit(error_text.begin(), UnrecognizedCharacters); - current_column_ += error_text.size(); - source_text = source_text.drop_front(error_text.size()); + position += error_text.size(); return token; } - auto LexStartOfFile(llvm::StringRef& /*source_text*/) -> void { + auto LexStartOfFile(llvm::StringRef /*source_text*/) -> void { // Before lexing any source text, add the start-of-file token so that code // can assume a non-empty token buffer for the rest of lexing. Note that the // start-of-file always has trailing space because it *is* whitespace. - buffer_->AddToken({.kind = TokenKind::StartOfFile, - .has_trailing_space = true, - .token_line = current_line_, - .column = current_column_}); + buffer_.AddToken({.kind = TokenKind::StartOfFile, + .has_trailing_space = true, + .token_line = current_line(), + .column = 0}); } - auto LexEndOfFile(llvm::StringRef& source_text) -> void { - CARBON_DCHECK(source_text.empty()); - + auto LexEndOfFile(llvm::StringRef source_text, ssize_t position) -> void { + CARBON_CHECK(position == static_cast(source_text.size())); // Check if the last line is empty and not the first line (and only). If so, // re-pin the last line to be the prior one so that diagnostics and editors // can treat newlines as terminators even though we internally handle them // as separators in case of a missing newline on the last line. We do this // here instead of detecting this when we see the newline to avoid more // conditions along that fast path. - if (current_column_ == 0 && buffer_->GetLineNumber(current_line_) != 1) { - current_line_ = buffer_->GetPrevLine(current_line_); - current_line_info_ = &buffer_->GetLineInfo(current_line_); - current_column_ = current_line_info_->length; + if (position == current_line_info()->start && line_index_ != 0) { + --line_index_; + --position; } else { // Update the line length as this is also the end of a line. - current_line_info_->length = current_column_; + current_line_info()->length = ComputeColumn(position); } // The end-of-file token is always considered to be whitespace. @@ -789,12 +806,12 @@ class [[clang::internal_linkage]] TokenizedBuffer::Lexer { // Close any open groups. We do this after marking whitespace, it will // preserve that. if (!open_groups_.empty()) { - CloseInvalidOpenGroups(TokenKind::Error); + CloseInvalidOpenGroups(TokenKind::Error, position); } - buffer_->AddToken({.kind = TokenKind::EndOfFile, - .token_line = current_line_, - .column = current_column_}); + buffer_.AddToken({.kind = TokenKind::EndOfFile, + .token_line = current_line(), + .column = ComputeColumn(position)}); } // We use a collection of static member functions for table-based dispatch to @@ -805,32 +822,33 @@ class [[clang::internal_linkage]] TokenizedBuffer::Lexer { // build efficient dispatch tables out of them. All of them end by doing a // must-tail return call to this routine. It handles continuing the dispatch // chain. - static auto DispatchNext(Lexer& lexer, llvm::StringRef& source_text) -> void { + static auto DispatchNext(Lexer& lexer, llvm::StringRef source_text, + ssize_t position) -> void { + if (LLVM_LIKELY(position < static_cast(source_text.size()))) { + // The common case is to tail recurse based on the next character. Note + // that because this is a must-tail return, this cannot fail to tail-call + // and will not grow the stack. This is in essence a loop with dynamic + // tail dispatch to the next stage of the loop. + [[clang::musttail]] return DispatchTable[static_cast( + source_text[position])](lexer, source_text, position); + } + // When we finish the source text, stop recursing. We also hint this so that // the tail-dispatch is optimized as that's essentially the loop back-edge // and this is the loop exit. - if (LLVM_UNLIKELY(source_text.empty())) { - lexer.LexEndOfFile(source_text); - return; - } - - // The common case is to tail recurse based on the next character. Note that - // because this is a must-tail return, this cannot fail to tail-call and - // will not grow the stack. This is in essence a loop with dynamic tail - // dispatch to the next stage of the loop. - [[clang::musttail]] return DispatchTable[static_cast( - source_text.front())](lexer, source_text); + lexer.LexEndOfFile(source_text, position); } // Define a set of dispatch functions that simply forward to a method that // lexes a token. This includes validating that an actual token was produced, // and continuing the dispatch. -#define CARBON_DISPATCH_LEX_TOKEN(LexMethod) \ - static auto Dispatch##LexMethod(Lexer& lexer, llvm::StringRef& source_text) \ - ->void { \ - LexResult result = lexer.LexMethod(source_text); \ - CARBON_CHECK(result) << "Failed to form a token!"; \ - [[clang::musttail]] return DispatchNext(lexer, source_text); \ +#define CARBON_DISPATCH_LEX_TOKEN(LexMethod) \ + static auto Dispatch##LexMethod(Lexer& lexer, llvm::StringRef source_text, \ + ssize_t position) \ + ->void { \ + LexResult result = lexer.LexMethod(source_text, position); \ + CARBON_CHECK(result) << "Failed to form a token!"; \ + [[clang::musttail]] return DispatchNext(lexer, source_text, position); \ } CARBON_DISPATCH_LEX_TOKEN(LexError) CARBON_DISPATCH_LEX_TOKEN(LexSymbolToken) @@ -839,14 +857,14 @@ class [[clang::internal_linkage]] TokenizedBuffer::Lexer { CARBON_DISPATCH_LEX_TOKEN(LexStringLiteral) // A custom dispatch functions that pre-select the symbol token to lex. -#define CARBON_DISPATCH_LEX_SYMBOL_TOKEN(LexMethod) \ - static auto Dispatch##LexMethod##SymbolToken(Lexer& lexer, \ - llvm::StringRef& source_text) \ - ->void { \ - LexResult result = lexer.LexMethod##SymbolToken( \ - source_text, OneCharTokenKindTable[source_text.front()]); \ - CARBON_CHECK(result) << "Failed to form a token!"; \ - [[clang::musttail]] return DispatchNext(lexer, source_text); \ +#define CARBON_DISPATCH_LEX_SYMBOL_TOKEN(LexMethod) \ + static auto Dispatch##LexMethod##SymbolToken( \ + Lexer& lexer, llvm::StringRef source_text, ssize_t position) \ + ->void { \ + LexResult result = lexer.LexMethod##SymbolToken( \ + source_text, OneCharTokenKindTable[source_text[position]], position); \ + CARBON_CHECK(result) << "Failed to form a token!"; \ + [[clang::musttail]] return DispatchNext(lexer, source_text, position); \ } CARBON_DISPATCH_LEX_SYMBOL_TOKEN(LexOneChar) CARBON_DISPATCH_LEX_SYMBOL_TOKEN(LexOpening) @@ -854,11 +872,12 @@ class [[clang::internal_linkage]] TokenizedBuffer::Lexer { // Define a set of non-token dispatch functions that handle things like // whitespace and comments. -#define CARBON_DISPATCH_LEX_NON_TOKEN(LexMethod) \ - static auto Dispatch##LexMethod(Lexer& lexer, llvm::StringRef& source_text) \ - ->void { \ - lexer.LexMethod(source_text); \ - [[clang::musttail]] return DispatchNext(lexer, source_text); \ +#define CARBON_DISPATCH_LEX_NON_TOKEN(LexMethod) \ + static auto Dispatch##LexMethod(Lexer& lexer, llvm::StringRef source_text, \ + ssize_t position) \ + ->void { \ + lexer.LexMethod(source_text, position); \ + [[clang::musttail]] return DispatchNext(lexer, source_text, position); \ } CARBON_DISPATCH_LEX_NON_TOKEN(LexHorizontalWhitespace) CARBON_DISPATCH_LEX_NON_TOKEN(LexVerticalWhitespace) @@ -866,7 +885,9 @@ class [[clang::internal_linkage]] TokenizedBuffer::Lexer { // The main entry point for dispatching through the lexer's table. This method // should always fully consume the source text. - auto Dispatch(llvm::StringRef& source_text) -> void { + auto Lex() && -> TokenizedBuffer { + llvm::StringRef source_text = buffer_.source_->text(); + // First build up our line data structures. CreateLines(source_text); @@ -874,15 +895,18 @@ class [[clang::internal_linkage]] TokenizedBuffer::Lexer { // Manually enter the dispatch loop. This call will tail-recurse through the // dispatch table until everything from source_text is consumed. - DispatchNext(*this, source_text); + DispatchNext(*this, source_text, /*position=*/0); - CARBON_CHECK(source_text.empty()) - << "Finished lexer dispatch without consuming the entire source text!"; + if (consumer_.seen_error()) { + buffer_.has_errors_ = true; + } + + return std::move(buffer_); } private: - using DispatchFunctionT = auto(Lexer& lexer, llvm::StringRef& source_text) - -> void; + using DispatchFunctionT = auto(Lexer& lexer, llvm::StringRef source_text, + ssize_t position) -> void; using DispatchTableT = std::array; // Build a table of function pointers that we can use to dispatch to the @@ -987,21 +1011,21 @@ class [[clang::internal_linkage]] TokenizedBuffer::Lexer { static const std::array OneCharTokenKindTable; - TokenizedBuffer* buffer_; + TokenizedBuffer buffer_; + + ssize_t line_index_; + + bool set_indent_ = false; + + llvm::SmallVector open_groups_; + + ErrorTrackingDiagnosticConsumer consumer_; SourceBufferLocationTranslator translator_; LexerDiagnosticEmitter emitter_; TokenLocationTranslator token_translator_; TokenDiagnosticEmitter token_emitter_; - - Line current_line_ = Line::Invalid; - LineInfo* current_line_info_; - - int current_column_ = 0; - bool set_indent_ = false; - - llvm::SmallVector open_groups_; }; constexpr TokenizedBuffer::Lexer::DispatchTableT @@ -1022,18 +1046,8 @@ constexpr std::array auto TokenizedBuffer::Lex(SourceBuffer& source, DiagnosticConsumer& consumer) -> TokenizedBuffer { - TokenizedBuffer buffer(source); - ErrorTrackingDiagnosticConsumer error_tracking_consumer(consumer); - Lexer lexer(buffer, error_tracking_consumer); - - llvm::StringRef source_text = source.text(); - lexer.Dispatch(source_text); - - if (error_tracking_consumer.seen_error()) { - buffer.has_errors_ = true; - } - - return buffer; + Lexer lexer(source, consumer); + return std::move(lexer).Lex(); } auto TokenizedBuffer::GetKind(Token token) const -> TokenKind {