carbon-language/carbon-lang
1
0
Fork 0
mirror of https://github.com/carbon-language/carbon-lang.git synced 2026-10-04 22:02:52 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Rename Lex::{Token,Line} -> Lex::{Token,Line}Index. (#3433)

Browse Source 
As discussed [on
discord](https://discord.com/channels/655572317891461132/655578254970716160/1178878128714678282)
and today's toolchain discussion.
This commit is contained in:
Richard Smith
2023-11-29 20:33:58 +00:00
committed by GitHub
parent 332a368cee
commit eae630a3db
16 changed files with 182 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files
toolchain/lex/tokenized_buffer.cpp
+42 -38
View File
@@ -19,23 +19,23 @@
namespace Carbon::Lex {
auto TokenizedBuffer::GetKind(Token token) const -> TokenKind {
auto TokenizedBuffer::GetKind(TokenIndex token) const -> TokenKind {
return GetTokenInfo(token).kind;
}
auto TokenizedBuffer::GetLine(Token token) const -> Line {
auto TokenizedBuffer::GetLine(TokenIndex token) const -> LineIndex {
return GetTokenInfo(token).token_line;
}
auto TokenizedBuffer::GetLineNumber(Token token) const -> int {
auto TokenizedBuffer::GetLineNumber(TokenIndex token) const -> int {
return GetLineNumber(GetLine(token));
}
auto TokenizedBuffer::GetColumnNumber(Token token) const -> int {
auto TokenizedBuffer::GetColumnNumber(TokenIndex token) const -> int {
return GetTokenInfo(token).column + 1;
}
auto TokenizedBuffer::GetTokenText(Token token) const -> llvm::StringRef {
auto TokenizedBuffer::GetTokenText(TokenIndex token) const -> llvm::StringRef {
const auto& token_info = GetTokenInfo(token);
llvm::StringRef fixed_spelling = token_info.kind.fixed_spelling();
if (!fixed_spelling.empty()) {
@@ -90,82 +90,83 @@ auto TokenizedBuffer::GetTokenText(Token token) const -> llvm::StringRef {
return value_stores_->identifiers().Get(token_info.ident_id);
}
auto TokenizedBuffer::GetIdentifier(Token token) const -> IdentifierId {
auto TokenizedBuffer::GetIdentifier(TokenIndex token) const -> IdentifierId {
const auto& token_info = GetTokenInfo(token);
CARBON_CHECK(token_info.kind == TokenKind::Identifier) << token_info.kind;
return token_info.ident_id;
}
auto TokenizedBuffer::GetIntegerLiteral(Token token) const -> IntegerId {
auto TokenizedBuffer::GetIntegerLiteral(TokenIndex token) const -> IntegerId {
const auto& token_info = GetTokenInfo(token);
CARBON_CHECK(token_info.kind == TokenKind::IntegerLiteral) << token_info.kind;
return token_info.integer_id;
}
auto TokenizedBuffer::GetRealLiteral(Token token) const -> RealId {
auto TokenizedBuffer::GetRealLiteral(TokenIndex token) const -> RealId {
const auto& token_info = GetTokenInfo(token);
CARBON_CHECK(token_info.kind == TokenKind::RealLiteral) << token_info.kind;
return token_info.real_id;
}
auto TokenizedBuffer::GetStringLiteral(Token token) const -> StringLiteralId {
auto TokenizedBuffer::GetStringLiteral(TokenIndex token) const
-> StringLiteralId {
const auto& token_info = GetTokenInfo(token);
CARBON_CHECK(token_info.kind == TokenKind::StringLiteral) << token_info.kind;
return token_info.string_literal_id;
}
auto TokenizedBuffer::GetTypeLiteralSize(Token token) const
auto TokenizedBuffer::GetTypeLiteralSize(TokenIndex token) const
-> const llvm::APInt& {
const auto& token_info = GetTokenInfo(token);
CARBON_CHECK(token_info.kind.is_sized_type_literal()) << token_info.kind;
return value_stores_->integers().Get(token_info.integer_id);
}
auto TokenizedBuffer::GetMatchedClosingToken(Token opening_token) const
-> Token {
auto TokenizedBuffer::GetMatchedClosingToken(TokenIndex opening_token) const
-> TokenIndex {
const auto& opening_token_info = GetTokenInfo(opening_token);
CARBON_CHECK(opening_token_info.kind.is_opening_symbol())
<< opening_token_info.kind;
return opening_token_info.closing_token;
}
auto TokenizedBuffer::GetMatchedOpeningToken(Token closing_token) const
-> Token {
auto TokenizedBuffer::GetMatchedOpeningToken(TokenIndex closing_token) const
-> TokenIndex {
const auto& closing_token_info = GetTokenInfo(closing_token);
CARBON_CHECK(closing_token_info.kind.is_closing_symbol())
<< closing_token_info.kind;
return closing_token_info.opening_token;
}
auto TokenizedBuffer::HasLeadingWhitespace(Token token) const -> bool {
auto TokenizedBuffer::HasLeadingWhitespace(TokenIndex token) const -> bool {
auto it = TokenIterator(token);
return it == tokens().begin() || GetTokenInfo(*(it - 1)).has_trailing_space;
}
auto TokenizedBuffer::HasTrailingWhitespace(Token token) const -> bool {
auto TokenizedBuffer::HasTrailingWhitespace(TokenIndex token) const -> bool {
return GetTokenInfo(token).has_trailing_space;
}
auto TokenizedBuffer::IsRecoveryToken(Token token) const -> bool {
auto TokenizedBuffer::IsRecoveryToken(TokenIndex token) const -> bool {
return GetTokenInfo(token).is_recovery;
}
auto TokenizedBuffer::GetLineNumber(Line line) const -> int {
auto TokenizedBuffer::GetLineNumber(LineIndex line) const -> int {
return line.index + 1;
}
auto TokenizedBuffer::GetNextLine(Line line) const -> Line {
Line next(line.index + 1);
auto TokenizedBuffer::GetNextLine(LineIndex line) const -> LineIndex {
LineIndex next(line.index + 1);
CARBON_DCHECK(static_cast<size_t>(next.index) < line_infos_.size());
return next;
}
auto TokenizedBuffer::GetPrevLine(Line line) const -> Line {
auto TokenizedBuffer::GetPrevLine(LineIndex line) const -> LineIndex {
CARBON_CHECK(line.index > 0);
return Line(line.index - 1);
return LineIndex(line.index - 1);
}
auto TokenizedBuffer::GetIndentColumnNumber(Line line) const -> int {
auto TokenizedBuffer::GetIndentColumnNumber(LineIndex line) const -> int {
return GetLineInfo(line).indent + 1;
}
@@ -191,7 +192,8 @@ static auto ComputeDecimalPrintedWidth(int number) -> int {
return static_cast<int>(std::log10(number)) + 1;
}
auto TokenizedBuffer::GetTokenPrintWidths(Token token) const -> PrintWidths {
auto TokenizedBuffer::GetTokenPrintWidths(TokenIndex token) const
-> PrintWidths {
PrintWidths widths = {};
widths.index = ComputeDecimalPrintedWidth(token_infos_.size());
widths.kind = GetKind(token).name().size();
@@ -212,11 +214,11 @@ auto TokenizedBuffer::Print(llvm::raw_ostream& output_stream) const -> void {
PrintWidths widths = {};
widths.index = ComputeDecimalPrintedWidth((token_infos_.size()));
for (Token token : tokens()) {
for (TokenIndex token : tokens()) {
widths.Widen(GetTokenPrintWidths(token));
}
for (Token token : tokens()) {
for (TokenIndex token : tokens()) {
PrintToken(output_stream, token, widths);
output_stream << "\n";
}
@@ -224,12 +226,13 @@ auto TokenizedBuffer::Print(llvm::raw_ostream& output_stream) const -> void {
}
auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream,
Token token) const -> void {
TokenIndex token) const -> void {
PrintToken(output_stream, token, {});
}
auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream, Token token,
PrintWidths widths) const -> void {
auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream,
TokenIndex token, PrintWidths widths) const
-> void {
widths.Widen(GetTokenPrintWidths(token));
int token_index = token.index;
const auto& token_info = GetTokenInfo(token);
@@ -292,31 +295,31 @@ auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream, Token token,
output_stream << " },";
}
auto TokenizedBuffer::GetLineInfo(Line line) -> LineInfo& {
auto TokenizedBuffer::GetLineInfo(LineIndex line) -> LineInfo& {
return line_infos_[line.index];
}
auto TokenizedBuffer::GetLineInfo(Line line) const -> const LineInfo& {
auto TokenizedBuffer::GetLineInfo(LineIndex line) const -> const LineInfo& {
return line_infos_[line.index];
}
auto TokenizedBuffer::AddLine(LineInfo info) -> Line {
auto TokenizedBuffer::AddLine(LineInfo info) -> LineIndex {
line_infos_.push_back(info);
return Line(static_cast<int>(line_infos_.size()) - 1);
return LineIndex(static_cast<int>(line_infos_.size()) - 1);
}
auto TokenizedBuffer::GetTokenInfo(Token token) -> TokenInfo& {
auto TokenizedBuffer::GetTokenInfo(TokenIndex token) -> TokenInfo& {
return token_infos_[token.index];
}
auto TokenizedBuffer::GetTokenInfo(Token token) const -> const TokenInfo& {
auto TokenizedBuffer::GetTokenInfo(TokenIndex token) const -> const TokenInfo& {
return token_infos_[token.index];
}
auto TokenizedBuffer::AddToken(TokenInfo info) -> Token {
auto TokenizedBuffer::AddToken(TokenInfo info) -> TokenIndex {
token_infos_.push_back(info);
expected_parse_tree_size_ += info.kind.expected_parse_tree_size();
return Token(static_cast<int>(token_infos_.size()) - 1);
return TokenIndex(static_cast<int>(token_infos_.size()) - 1);
}
auto TokenIterator::Print(llvm::raw_ostream& output) const -> void {
@@ -368,7 +371,8 @@ auto TokenizedBuffer::SourceBufferLocationTranslator::GetLocation(
.column_number = column_number + 1};
}
auto TokenLocationTranslator::GetLocation(Token token) -> DiagnosticLocation {
auto TokenLocationTranslator::GetLocation(TokenIndex token)
-> DiagnosticLocation {
// Map the token location into a position within the source buffer.
const auto& token_info = buffer_->GetTokenInfo(token);
const auto& line_info = buffer_->GetLineInfo(token_info.token_line);