carbon-language/carbon-lang
1
0
Fork 0
mirror of https://github.com/carbon-language/carbon-lang.git synced 2026-10-04 17:51:05 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Rename Lex::{Token,Line} -> Lex::{Token,Line}Index. (#3433)

Browse Source 
As discussed [on
discord](https://discord.com/channels/655572317891461132/655578254970716160/1178878128714678282)
and today's toolchain discussion.
This commit is contained in:
Richard Smith
2023-11-29 20:33:58 +00:00
committed by GitHub
parent 332a368cee
commit eae630a3db
16 changed files with 182 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files
toolchain/parse/context.cpp
+16 -16
View File
@@ -56,7 +56,7 @@ Context::Context(Tree& tree, Lex::TokenizedBuffer& tokens,
<< tokens_->GetKind(*end_);
}
auto Context::AddLeafNode(NodeKind kind, Lex::Token token, bool has_error)
auto Context::AddLeafNode(NodeKind kind, Lex::TokenIndex token, bool has_error)
-> void {
CheckNodeMatchesLexerToken(kind, tokens_->GetKind(token), has_error);
tree_->node_impls_.push_back(
@@ -66,7 +66,7 @@ auto Context::AddLeafNode(NodeKind kind, Lex::Token token, bool has_error)
}
}
auto Context::AddNode(NodeKind kind, Lex::Token token, int subtree_start,
auto Context::AddNode(NodeKind kind, Lex::TokenIndex token, int subtree_start,
bool has_error) -> void {
CheckNodeMatchesLexerToken(kind, tokens_->GetKind(token), has_error);
int subtree_size = tree_->size() - subtree_start + 1;
@@ -77,9 +77,9 @@ auto Context::AddNode(NodeKind kind, Lex::Token token, int subtree_start,
}
}
auto Context::ConsumeAndAddOpenParen(Lex::Token default_token,
auto Context::ConsumeAndAddOpenParen(Lex::TokenIndex default_token,
NodeKind start_kind)
-> std::optional<Lex::Token> {
-> std::optional<Lex::TokenIndex> {
if (auto open_paren = ConsumeIf(Lex::TokenKind::OpenParen)) {
AddLeafNode(start_kind, *open_paren, /*has_error=*/false);
return open_paren;
@@ -93,7 +93,7 @@ auto Context::ConsumeAndAddOpenParen(Lex::Token default_token,
}
}
auto Context::ConsumeAndAddCloseSymbol(Lex::Token expected_open,
auto Context::ConsumeAndAddCloseSymbol(Lex::TokenIndex expected_open,
StateStackEntry state,
NodeKind close_kind) -> void {
Lex::TokenKind open_token_kind = tokens().GetKind(expected_open);
@@ -126,13 +126,13 @@ auto Context::ConsumeAndAddLeafNodeIf(Lex::TokenKind token_kind,
return true;
}
auto Context::ConsumeChecked(Lex::TokenKind kind) -> Lex::Token {
auto Context::ConsumeChecked(Lex::TokenKind kind) -> Lex::TokenIndex {
CARBON_CHECK(PositionIs(kind))
<< "Required " << kind << ", found " << PositionKind();
return Consume();
}
auto Context::ConsumeIf(Lex::TokenKind kind) -> std::optional<Lex::Token> {
auto Context::ConsumeIf(Lex::TokenKind kind) -> std::optional<Lex::TokenIndex> {
if (!PositionIs(kind)) {
return std::nullopt;
}
@@ -150,10 +150,10 @@ auto Context::ConsumeIfPatternKeyword(Lex::TokenKind keyword_token,
}
auto Context::FindNextOf(std::initializer_list<Lex::TokenKind> desired_kinds)
-> std::optional<Lex::Token> {
-> std::optional<Lex::TokenIndex> {
auto new_position = position_;
while (true) {
Lex::Token token = *new_position;
Lex::TokenIndex token = *new_position;
Lex::TokenKind kind = tokens().GetKind(token);
if (kind.IsOneOf(desired_kinds)) {
return token;
@@ -183,19 +183,19 @@ auto Context::SkipMatchingGroup() -> bool {
return true;
}
auto Context::SkipPastLikelyEnd(Lex::Token skip_root)
-> std::optional<Lex::Token> {
auto Context::SkipPastLikelyEnd(Lex::TokenIndex skip_root)
-> std::optional<Lex::TokenIndex> {
if (position_ == end_) {
return std::nullopt;
}
Lex::Line root_line = tokens().GetLine(skip_root);
Lex::LineIndex root_line = tokens().GetLine(skip_root);
int root_line_indent = tokens().GetIndentColumnNumber(root_line);
// We will keep scanning through tokens on the same line as the root or
// lines with greater indentation than root's line.
auto is_same_line_or_indent_greater_than_root = [&](Lex::Token t) {
Lex::Line l = tokens().GetLine(t);
auto is_same_line_or_indent_greater_than_root = [&](Lex::TokenIndex t) {
Lex::LineIndex l = tokens().GetLine(t);
if (l == root_line) {
return true;
}
@@ -229,7 +229,7 @@ auto Context::SkipPastLikelyEnd(Lex::Token skip_root)
return std::nullopt;
}
auto Context::SkipTo(Lex::Token t) -> void {
auto Context::SkipTo(Lex::TokenIndex t) -> void {
CARBON_CHECK(t >= *position_) << "Tried to skip backwards from " << position_
<< " to " << Lex::TokenIterator(t);
position_ = Lex::TokenIterator(t);
@@ -454,7 +454,7 @@ auto Context::PrintForStackDump(llvm::raw_ostream& output) const -> void {
}
auto Context::PrintTokenForStackDump(llvm::raw_ostream& output,
Lex::Token token) const -> void {
Lex::TokenIndex token) const -> void {
output << " @ " << tokens_->GetLineNumber(tokens_->GetLine(token)) << ":"
<< tokens_->GetColumnNumber(token) << ": token " << token << " : "
<< tokens_->GetKind(token) << "\n";