carbon-language/carbon-lang
1
0
Fork 0
mirror of https://github.com/carbon-language/carbon-lang.git synced 2026-10-05 22:02:55 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Fix GetTokenText for raw identifiers. (#7250)

Browse Source 
Include the `r#` in the spelling of the identifier.
This commit is contained in:
Richard Smith
2026-05-22 01:57:13 +00:00
committed by GitHub
parent 805b3eebce
commit 862b1c91f8
9 changed files with 143 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
toolchain/lex/tokenized_buffer.cpp
+28 -1
View File
@@ -107,7 +107,18 @@ auto TokenizedBuffer::GetTokenText(TokenIndex token) const -> llvm::StringRef {
CARBON_CHECK(token_info.kind() == TokenKind::Identifier, "{0}",
token_info.kind());
return value_stores_->identifiers().Get(token_info.ident_id());
// If this is a raw identifier, obtain its spelling from the source text.
auto ident = value_stores_->identifiers().Get(token_info.ident_id());
if (IsRawIdentifier(token)) {
llvm::StringRef raw_ident =
source_->text().substr(token_info.byte_offset(), ident.size() + 2);
CARBON_CHECK(raw_ident[0] == 'r' && raw_ident[1] == '#' &&
raw_ident.substr(2) == ident,
"`{0}` != `r#` + `{1}`", raw_ident, ident);
return raw_ident;
}
return ident;
}
auto TokenizedBuffer::GetIdentifier(TokenIndex token) const -> IdentifierId {
@@ -170,6 +181,22 @@ auto TokenizedBuffer::GetMatchedOpeningToken(TokenIndex closing_token) const
return closing_token_info.opening_token_index();
}
auto TokenizedBuffer::IsRawIdentifier(TokenIndex token) const -> bool {
const auto& token_info = token_infos_.Get(token);
if (token_info.kind() != TokenKind::Identifier) {
return false;
}
// Check if the spelling starts `r#`. A small nuance here: we also need to
// check the character after the `#` is an identifier start character, because
// `r#<non-identifier-character>` is lexed as an `r` token followed by a token
// starting with `#`. It suffices to check that character is the first
// character of the identifier.
auto token_text = source_->text().substr(token_info.byte_offset());
return token_text.starts_with("r#") &&
token_text[2] ==
value_stores_->identifiers().Get(token_info.ident_id()).front();
}
auto TokenizedBuffer::IsRecoveryToken(TokenIndex token) const -> bool {
if (recovery_tokens_.empty()) {
return false;