diff --git a/toolchain/lexer/tokenized_buffer.cpp b/toolchain/lexer/tokenized_buffer.cpp index cb543df0564e..668eb81f9465 100644 --- a/toolchain/lexer/tokenized_buffer.cpp +++ b/toolchain/lexer/tokenized_buffer.cpp @@ -86,32 +86,7 @@ auto VariantMatch(V&& v, Fs&&... fs) -> decltype(auto) { // the different lexing steps that may be used. It directly updates the provided // tokenized buffer with the lexed tokens. class TokenizedBuffer::Lexer { - TokenizedBuffer& buffer; - - SourceBufferLocationTranslator translator; - LexerDiagnosticEmitter emitter; - - TokenLocationTranslator token_translator; - TokenDiagnosticEmitter token_emitter; - - Line current_line; - LineInfo* current_line_info; - - int current_column = 0; - bool set_indent = false; - - llvm::SmallVector open_groups; - public: - Lexer(TokenizedBuffer& buffer, DiagnosticConsumer& consumer) - : buffer(buffer), - translator(buffer), - emitter(translator, consumer), - token_translator(buffer), - token_emitter(token_translator, consumer), - current_line(buffer.AddLine({0, 0, 0})), - current_line_info(&buffer.GetLineInfo(current_line)) {} - // Symbolic result of a lexing action. This indicates whether we successfully // lexed a token, or whether other lexing actions should be attempted. // @@ -119,9 +94,6 @@ class TokenizedBuffer::Lexer { // more self documenting, and by consuming the actual token constructively // when one is produced, it helps ensure the correct result is returned. class LexResult { - bool formed_token; - explicit LexResult(bool formed_token) : formed_token(formed_token) {} - public: // Consumes (and discard) a valid token to construct a result // indicating a token has been produced. Relies on implicit conversions. @@ -133,24 +105,38 @@ class TokenizedBuffer::Lexer { // Tests whether a token was produced by the lexing routine, and // the lexer can continue forming tokens. - explicit operator bool() const { return formed_token; } + explicit operator bool() const { return formed_token_; } + + private: + explicit LexResult(bool formed_token) : formed_token_(formed_token) {} + + bool formed_token_; }; + Lexer(TokenizedBuffer& buffer, DiagnosticConsumer& consumer) + : buffer_(buffer), + translator_(buffer), + emitter_(translator_, consumer), + token_translator_(buffer), + token_emitter_(token_translator_, consumer), + current_line_(buffer.AddLine({0, 0, 0})), + current_line_info_(&buffer.GetLineInfo(current_line_)) {} + // Perform the necessary bookkeeping to step past a newline at the current // line and column. auto HandleNewline() -> void { - current_line_info->length = current_column; + current_line_info_->length = current_column_; - current_line = - buffer.AddLine({current_line_info->start + current_column + 1, 0, 0}); - current_line_info = &buffer.GetLineInfo(current_line); - current_column = 0; - set_indent = false; + current_line_ = buffer_.AddLine( + {current_line_info_->start + current_column_ + 1, 0, 0}); + current_line_info_ = &buffer_.GetLineInfo(current_line_); + current_column_ = 0; + set_indent_ = false; } auto NoteWhitespace() -> void { - if (!buffer.token_infos_.empty()) { - buffer.token_infos_.back().has_trailing_space = true; + if (!buffer_.token_infos_.empty()) { + buffer_.token_infos_.back().has_trailing_space = true; } } @@ -162,16 +148,16 @@ class TokenizedBuffer::Lexer { // were whitespace. if (source_text.startswith("//")) { // Any comment must be the only non-whitespace on the line. - if (set_indent) { - emitter.EmitError(source_text.begin()); + if (set_indent_) { + emitter_.EmitError(source_text.begin()); } // The introducer '//' must be followed by whitespace or EOF. if (source_text.size() > 2 && !IsSpace(source_text[2])) { - emitter.EmitError( + emitter_.EmitError( source_text.begin() + 2); } while (!source_text.empty() && source_text.front() != '\n') { - ++current_column; + ++current_column_; source_text = source_text.drop_front(); } if (source_text.empty()) { @@ -194,7 +180,7 @@ class TokenizedBuffer::Lexer { // to avoid creating an empty line. source_text = source_text.drop_front(); if (source_text.empty()) { - current_line_info->length = current_column; + current_line_info_->length = current_column_; return false; } @@ -209,7 +195,7 @@ class TokenizedBuffer::Lexer { // whitespace as well as special handling to allow better tokenization // of operators. This is just a stub to check that our column // management works. - ++current_column; + ++current_column_; source_text = source_text.drop_front(); continue; } @@ -217,7 +203,7 @@ class TokenizedBuffer::Lexer { CHECK(source_text.empty()) << "Cannot reach here w/o finishing the text!"; // Update the line length as this is also the end of a line. - current_line_info->length = current_column; + current_line_info_->length = current_column_; return false; } @@ -228,43 +214,43 @@ class TokenizedBuffer::Lexer { return LexResult::NoMatch(); } - int int_column = current_column; + int int_column = current_column_; int token_size = literal->Text().size(); - current_column += token_size; + current_column_ += token_size; source_text = source_text.drop_front(token_size); - if (!set_indent) { - current_line_info->indent = int_column; - set_indent = true; + if (!set_indent_) { + current_line_info_->indent = int_column; + set_indent_ = true; } return VariantMatch( - literal->ComputeValue(emitter), + literal->ComputeValue(emitter_), [&](LexedNumericLiteral::IntegerValue&& value) { - auto token = buffer.AddToken({.kind = TokenKind::IntegerLiteral(), - .token_line = current_line, - .column = int_column}); - buffer.GetTokenInfo(token).literal_index = - buffer.literal_int_storage_.size(); - buffer.literal_int_storage_.push_back(std::move(value.value)); + auto token = buffer_.AddToken({.kind = TokenKind::IntegerLiteral(), + .token_line = current_line_, + .column = int_column}); + buffer_.GetTokenInfo(token).literal_index = + buffer_.literal_int_storage_.size(); + buffer_.literal_int_storage_.push_back(std::move(value.value)); return token; }, [&](LexedNumericLiteral::RealValue&& value) { - auto token = buffer.AddToken({.kind = TokenKind::RealLiteral(), - .token_line = current_line, - .column = int_column}); - buffer.GetTokenInfo(token).literal_index = - buffer.literal_int_storage_.size(); - buffer.literal_int_storage_.push_back(std::move(value.mantissa)); - buffer.literal_int_storage_.push_back(std::move(value.exponent)); - assert(buffer.GetRealLiteral(token).IsDecimal() == + auto token = buffer_.AddToken({.kind = TokenKind::RealLiteral(), + .token_line = current_line_, + .column = int_column}); + buffer_.GetTokenInfo(token).literal_index = + buffer_.literal_int_storage_.size(); + buffer_.literal_int_storage_.push_back(std::move(value.mantissa)); + buffer_.literal_int_storage_.push_back(std::move(value.exponent)); + assert(buffer_.GetRealLiteral(token).IsDecimal() == (value.radix == 10)); return token; }, [&](LexedNumericLiteral::UnrecoverableError) { - auto token = buffer.AddToken({ + auto token = buffer_.AddToken({ .kind = TokenKind::Error(), - .token_line = current_line, + .token_line = current_line_, .column = int_column, .error_length = token_size, }); @@ -279,39 +265,39 @@ class TokenizedBuffer::Lexer { return LexResult::NoMatch(); } - Line string_line = current_line; - int string_column = current_column; + Line string_line = current_line_; + int string_column = current_column_; int literal_size = literal->Text().size(); source_text = source_text.drop_front(literal_size); - if (!set_indent) { - current_line_info->indent = string_column; - set_indent = true; + if (!set_indent_) { + current_line_info_->indent = string_column; + set_indent_ = true; } // Update line and column information. if (!literal->IsMultiLine()) { - current_column += literal_size; + current_column_ += literal_size; } else { for (char c : literal->Text()) { if (c == '\n') { HandleNewline(); // The indentation of all lines in a multi-line string literal is // that of the first line. - current_line_info->indent = string_column; - set_indent = true; + current_line_info_->indent = string_column; + set_indent_ = true; } else { - ++current_column; + ++current_column_; } } } - auto token = buffer.AddToken({.kind = TokenKind::StringLiteral(), - .token_line = string_line, - .column = string_column}); - buffer.GetTokenInfo(token).literal_index = - buffer.literal_string_storage_.size(); - buffer.literal_string_storage_.push_back(literal->ComputeValue(emitter)); + auto token = buffer_.AddToken({.kind = TokenKind::StringLiteral(), + .token_line = string_line, + .column = string_column}); + buffer_.GetTokenInfo(token).literal_index = + buffer_.literal_string_storage_.size(); + buffer_.literal_string_storage_.push_back(literal->ComputeValue(emitter_)); return token; } @@ -325,22 +311,22 @@ class TokenizedBuffer::Lexer { return LexResult::NoMatch(); } - if (!set_indent) { - current_line_info->indent = current_column; - set_indent = true; + if (!set_indent_) { + current_line_info_->indent = current_column_; + set_indent_ = true; } CloseInvalidOpenGroups(kind); const char* location = source_text.begin(); - Token token = buffer.AddToken( - {.kind = kind, .token_line = current_line, .column = current_column}); - current_column += kind.GetFixedSpelling().size(); + Token token = buffer_.AddToken( + {.kind = kind, .token_line = current_line_, .column = current_column_}); + current_column_ += kind.GetFixedSpelling().size(); source_text = source_text.drop_front(kind.GetFixedSpelling().size()); // Opening symbols just need to be pushed onto our queue of opening groups. if (kind.IsOpeningSymbol()) { - open_groups.push_back(token); + open_groups_.push_back(token); return token; } @@ -349,22 +335,22 @@ class TokenizedBuffer::Lexer { return token; } - TokenInfo& closing_token_info = buffer.GetTokenInfo(token); + TokenInfo& closing_token_info = buffer_.GetTokenInfo(token); // Check that there is a matching opening symbol before we consume this as // a closing symbol. - if (open_groups.empty()) { + if (open_groups_.empty()) { closing_token_info.kind = TokenKind::Error(); closing_token_info.error_length = kind.GetFixedSpelling().size(); - emitter.EmitError(location); + emitter_.EmitError(location); // Note that this still returns true as we do consume a symbol. return token; } // Finally can handle a normal closing symbol. - Token opening_token = open_groups.pop_back_val(); - TokenInfo& opening_token_info = buffer.GetTokenInfo(opening_token); + Token opening_token = open_groups_.pop_back_val(); + TokenInfo& opening_token_info = buffer_.GetTokenInfo(opening_token); opening_token_info.closing_token = token; closing_token_info.opening_token = opening_token; return token; @@ -404,11 +390,11 @@ class TokenizedBuffer::Lexer { return LexResult::NoMatch(); } - auto token = buffer.AddToken( - {.kind = *kind, .token_line = current_line, .column = column}); - buffer.GetTokenInfo(token).literal_index = - buffer.literal_int_storage_.size(); - buffer.literal_int_storage_.push_back(std::move(suffix_value)); + auto token = buffer_.AddToken( + {.kind = *kind, .token_line = current_line_, .column = column}); + buffer_.GetTokenInfo(token).literal_index = + buffer_.literal_int_storage_.size(); + buffer_.literal_int_storage_.push_back(std::move(suffix_value)); return token; } @@ -419,39 +405,39 @@ class TokenizedBuffer::Lexer { return; } - while (!open_groups.empty()) { - Token opening_token = open_groups.back(); - TokenKind opening_kind = buffer.GetTokenInfo(opening_token).kind; + while (!open_groups_.empty()) { + Token opening_token = open_groups_.back(); + TokenKind opening_kind = buffer_.GetTokenInfo(opening_token).kind; if (kind == opening_kind.GetClosingSymbol()) { return; } - open_groups.pop_back(); - token_emitter.EmitError(opening_token); + open_groups_.pop_back(); + token_emitter_.EmitError(opening_token); - CHECK(!buffer.Tokens().empty()) << "Must have a prior opening token!"; - Token prev_token = buffer.Tokens().end()[-1]; + CHECK(!buffer_.Tokens().empty()) << "Must have a prior opening token!"; + Token prev_token = buffer_.Tokens().end()[-1]; // TODO: do a smarter backwards scan for where to put the closing // token. - Token closing_token = buffer.AddToken( + Token closing_token = buffer_.AddToken( {.kind = opening_kind.GetClosingSymbol(), - .has_trailing_space = buffer.HasTrailingWhitespace(prev_token), + .has_trailing_space = buffer_.HasTrailingWhitespace(prev_token), .is_recovery = true, - .token_line = current_line, - .column = current_column}); - TokenInfo& opening_token_info = buffer.GetTokenInfo(opening_token); - TokenInfo& closing_token_info = buffer.GetTokenInfo(closing_token); + .token_line = current_line_, + .column = current_column_}); + TokenInfo& opening_token_info = buffer_.GetTokenInfo(opening_token); + TokenInfo& closing_token_info = buffer_.GetTokenInfo(closing_token); opening_token_info.closing_token = closing_token; closing_token_info.opening_token = opening_token; } } auto GetOrCreateIdentifier(llvm::StringRef text) -> Identifier { - auto insert_result = buffer.identifier_map_.insert( - {text, Identifier(buffer.identifier_infos_.size())}); + auto insert_result = buffer_.identifier_map_.insert( + {text, Identifier(buffer_.identifier_infos_.size())}); if (insert_result.second) { - buffer.identifier_infos_.push_back({text}); + buffer_.identifier_infos_.push_back({text}); } return insert_result.first->second; } @@ -461,17 +447,17 @@ class TokenizedBuffer::Lexer { return LexResult::NoMatch(); } - if (!set_indent) { - current_line_info->indent = current_column; - set_indent = true; + if (!set_indent_) { + current_line_info_->indent = current_column_; + set_indent_ = true; } // Take the valid characters off the front of the source buffer. llvm::StringRef identifier_text = source_text.take_while([](char c) { return IsAlnum(c) || c == '_'; }); CHECK(!identifier_text.empty()) << "Must have at least one character!"; - int identifier_column = current_column; - current_column += identifier_text.size(); + int identifier_column = current_column_; + current_column_ += identifier_text.size(); source_text = source_text.drop_front(identifier_text.size()); // Check if the text is a type literal, and if so form such a literal. @@ -486,16 +472,16 @@ class TokenizedBuffer::Lexer { #include "toolchain/lexer/token_registry.def" .Default(TokenKind::Error()); if (kind != TokenKind::Error()) { - return buffer.AddToken({.kind = kind, - .token_line = current_line, - .column = identifier_column}); + return buffer_.AddToken({.kind = kind, + .token_line = current_line_, + .column = identifier_column}); } // Otherwise we have a generic identifier. - return buffer.AddToken({.kind = TokenKind::Identifier(), - .token_line = current_line, - .column = identifier_column, - .id = GetOrCreateIdentifier(identifier_text)}); + return buffer_.AddToken({.kind = TokenKind::Identifier(), + .token_line = current_line_, + .column = identifier_column, + .id = GetOrCreateIdentifier(identifier_text)}); } auto LexError(llvm::StringRef& source_text) -> LexResult { @@ -522,23 +508,40 @@ class TokenizedBuffer::Lexer { // Longer errors get to be two tokens. error_text = error_text.substr(0, std::numeric_limits::max()); - auto token = buffer.AddToken( + auto token = buffer_.AddToken( {.kind = TokenKind::Error(), - .token_line = current_line, - .column = current_column, + .token_line = current_line_, + .column = current_column_, .error_length = static_cast(error_text.size())}); - emitter.EmitError(error_text.begin()); + emitter_.EmitError(error_text.begin()); - current_column += error_text.size(); + current_column_ += error_text.size(); source_text = source_text.drop_front(error_text.size()); return token; } auto AddEndOfFileToken() -> void { - buffer.AddToken({.kind = TokenKind::EndOfFile(), - .token_line = current_line, - .column = current_column}); + buffer_.AddToken({.kind = TokenKind::EndOfFile(), + .token_line = current_line_, + .column = current_column_}); } + + private: + TokenizedBuffer& buffer_; + + SourceBufferLocationTranslator translator_; + LexerDiagnosticEmitter emitter_; + + TokenLocationTranslator token_translator_; + TokenDiagnosticEmitter token_emitter_; + + Line current_line_; + LineInfo* current_line_info_; + + int current_column_ = 0; + bool set_indent_ = false; + + llvm::SmallVector open_groups_; }; auto TokenizedBuffer::Lex(SourceBuffer& source, DiagnosticConsumer& consumer)