carbon-language/carbon-lang
1
0
Fork 0
mirror of https://github.com/carbon-language/carbon-lang.git synced 2026-10-04 22:02:52 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Modify lex yaml output to elide FileStart/End in tests. (#4433)

Browse Source 
Trying to make split file tests of lex functionality shorter and easier
to read. numeric_literals.carbon in particular has an example of why I'm
interested in this (at the bottom). This also switches from `[]` list
format to `-` list format so that the trailing `]` is removed.

Trimming comments in tokenized_buffer.h because (1) it feels like it's
giving too much detail about what's printed, which has drifted slightly
and (2) it also feels like it's trying to justify YAML output, when
that's just what we're doing in general.

---------

Co-authored-by: Geoff Romer <gromer@google.com>
This commit is contained in:
Jon Ross-Perkins
2024-10-23 18:56:41 +00:00
committed by GitHub
co-authored by Geoff Romer
parent d1c6f0152e
commit 06f4eec91e
23 changed files with 233 additions and 311 deletions
Download Patch File Download Diff File Expand all files Collapse all files
toolchain/lex/tokenized_buffer.cpp
+11 -9
View File
@@ -215,13 +215,10 @@ auto TokenizedBuffer::GetTokenPrintWidths(TokenIndex token) const
return widths;
}
auto TokenizedBuffer::Print(llvm::raw_ostream& output_stream) const -> void {
if (tokens().begin() == tokens().end()) {
return;
}
auto TokenizedBuffer::Print(llvm::raw_ostream& output_stream,
bool omit_file_boundary_tokens) const -> void {
output_stream << "- filename: " << source_->filename() << "\n"
<< " tokens: [\n";
<< " tokens:\n";
PrintWidths widths = {};
widths.index = ComputeDecimalPrintedWidth((token_infos_.size()));
@@ -230,10 +227,15 @@ auto TokenizedBuffer::Print(llvm::raw_ostream& output_stream) const -> void {
}
for (TokenIndex token : tokens()) {
if (omit_file_boundary_tokens) {
auto kind = GetKind(token);
if (kind == TokenKind::FileStart || kind == TokenKind::FileEnd) {
continue;
}
}
PrintToken(output_stream, token, widths);
output_stream << "\n";
}
output_stream << " ]\n";
}
auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream,
@@ -254,7 +256,7 @@ auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream,
// justification manually in order to use the dynamically computed widths
// and get the quotes included.
output_stream << llvm::formatv(
" { index: {0}, kind: {1}, line: {2}, column: {3}, indent: {4}, "
" - { index: {0}, kind: {1}, line: {2}, column: {3}, indent: {4}, "
"spelling: '{5}'",
llvm::format_decimal(token_index, widths.index),
llvm::right_justify(
@@ -304,7 +306,7 @@ auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream,
output_stream << ", recovery: true";
}
output_stream << " },";
output_stream << " }";
}
// Find the line index corresponding to a specific byte offset within the source