carbon-language/carbon-lang
1
0
Fork 0
mirror of https://github.com/carbon-language/carbon-lang.git synced 2026-10-06 07:24:42 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Change comments and lines to ValueStores (#5621)

Browse Source 
Moves LineInfo and CommentData out so that they can easily be set as
`ValueType` on the Index types. I've also been thinking about letting
`ValueStore` take `ValueType` as a parameter instead of requiring it to
be inferred this way, but for these it feels more consistent with the
rest of the toolchain to do it this way.

I'm not doing similar with `TokenInfo` just because the recovery token
splicing makes it more difficult to use `ValueStore`.
This commit is contained in:
Jon Ross-Perkins
2025-06-07 01:30:11 +00:00
committed by GitHub
parent 2e297b5258
commit 766ef7077d
3 changed files with 95 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files
toolchain/lex/tokenized_buffer.cpp
+23 -42
View File
@@ -33,7 +33,8 @@ auto TokenizedBuffer::GetLineNumber(TokenIndex token) const -> int {
auto TokenizedBuffer::GetColumnNumber(TokenIndex token) const -> int {
const auto& token_info = GetTokenInfo(token);
const auto& line_info = GetLineInfo(FindLineIndex(token_info.byte_offset()));
const auto& line_info =
line_infos_.Get(FindLineIndex(token_info.byte_offset()));
return token_info.byte_offset() - line_info.start + 1;
}
@@ -166,19 +167,8 @@ auto TokenizedBuffer::IsRecoveryToken(TokenIndex token) const -> bool {
return recovery_tokens_[token.index];
}
auto TokenizedBuffer::GetNextLine(LineIndex line) const -> LineIndex {
LineIndex next(line.index + 1);
CARBON_DCHECK(static_cast<size_t>(next.index) < line_infos_.size());
return next;
}
auto TokenizedBuffer::GetPrevLine(LineIndex line) const -> LineIndex {
CARBON_CHECK(line.index > 0);
return LineIndex(line.index - 1);
}
auto TokenizedBuffer::GetIndentColumnNumber(LineIndex line) const -> int {
return GetLineInfo(line).indent + 1;
return line_infos_.Get(line).indent + 1;
}
auto TokenizedBuffer::PrintWidths::Widen(const PrintWidths& widths) -> void {
@@ -325,9 +315,11 @@ auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream,
// This takes advantage of the lines being sorted by their starting byte offsets
// to do a binary search for the line that contains the provided offset.
auto TokenizedBuffer::FindLineIndex(int32_t byte_offset) const -> LineIndex {
CARBON_DCHECK(!line_infos_.empty());
const auto* line_it =
llvm::partition_point(line_infos_, [byte_offset](LineInfo line_info) {
CARBON_DCHECK(line_infos_.size() > 0);
auto line_range = line_infos_.values();
auto line_it =
llvm::partition_point(line_range, [byte_offset](LineInfo line_info) {
return line_info.start <= byte_offset;
});
--line_it;
@@ -335,49 +327,36 @@ auto TokenizedBuffer::FindLineIndex(int32_t byte_offset) const -> LineIndex {
// If this isn't the first line but it starts past the end of the source, then
// this is a synthetic line added for simplicity of lexing. Step back one
// further to find the last non-synthetic line.
if (line_it != line_infos_.begin() &&
if (line_it != line_range.begin() &&
line_it->start == static_cast<int32_t>(source_->text().size())) {
--line_it;
}
CARBON_DCHECK(line_it->start <= byte_offset);
return LineIndex(line_it - line_infos_.begin());
}
auto TokenizedBuffer::GetLineInfo(LineIndex line) -> LineInfo& {
return line_infos_[line.index];
}
auto TokenizedBuffer::GetLineInfo(LineIndex line) const -> const LineInfo& {
return line_infos_[line.index];
}
auto TokenizedBuffer::AddLine(LineInfo info) -> LineIndex {
line_infos_.push_back(info);
return LineIndex(static_cast<int>(line_infos_.size()) - 1);
return LineIndex(line_it - line_range.begin());
}
auto TokenizedBuffer::IsAfterComment(TokenIndex token,
CommentIndex comment_index) const -> bool {
const auto& comment_data = comments_[comment_index.index];
const auto& comment_data = comments_.Get(comment_index);
return GetTokenInfo(token).byte_offset() > comment_data.start;
}
auto TokenizedBuffer::GetCommentText(CommentIndex comment_index) const
-> llvm::StringRef {
const auto& comment_data = comments_[comment_index.index];
const auto& comment_data = comments_.Get(comment_index);
return source_->text().substr(comment_data.start, comment_data.length);
}
auto TokenizedBuffer::AddComment(int32_t indent, int32_t start, int32_t end)
-> void {
if (!comments_.empty()) {
auto& comment = comments_.back();
if (comments_.size() > 0) {
auto& comment = comments_.Get(CommentIndex(comments_.size() - 1));
if (comment.start + comment.length + indent == start) {
comment.length = end - comment.start;
return;
}
}
comments_.push_back({.start = start, .length = end - start});
comments_.Add({.start = start, .length = end - start});
}
auto TokenizedBuffer::CollectMemUsage(MemUsage& mem_usage,
@@ -394,23 +373,25 @@ auto TokenizedBuffer::SourcePointerToDiagnosticLoc(const char* loc) const
"location not within buffer");
int32_t offset = loc - source_->text().begin();
auto line_range = line_infos_.values();
// Find the first line starting after the given location.
const auto* next_line_it = llvm::partition_point(
line_infos_,
const auto next_line_it = llvm::partition_point(
line_range,
[offset](const LineInfo& line) { return line.start <= offset; });
// Step back one line to find the line containing the given position.
CARBON_CHECK(next_line_it != line_infos_.begin(),
CARBON_CHECK(next_line_it != line_range.begin(),
"location precedes the start of the first line");
const auto* line_it = std::prev(next_line_it);
int line_number = line_it - line_infos_.begin();
const auto line_it = std::prev(next_line_it);
int line_number = line_it - line_range.begin();
int column_number = offset - line_it->start;
// Grab the line from the buffer by slicing from this line to the next
// minus the newline. When on the last line, instead use the start to the end
// of the buffer.
llvm::StringRef text = source_->text();
llvm::StringRef line = next_line_it != line_infos_.end()
llvm::StringRef line = next_line_it != line_range.end()
? text.slice(line_it->start, next_line_it->start)
: text.substr(line_it->start);