carbon-language/carbon-lang
1
0
Fork 0
mirror of https://github.com/carbon-language/carbon-lang.git synced 2026-10-05 11:31:05 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Bracket the tokenized buffer output. (#2446)

Browse Source 
This makes TokenizedBuffer more consistent with ParseTree and SemanticsIR, which also wrap with [] to produce a sequence value.

It also makes it possible in driver.cpp to just prefix the line with a name, so it ends up with:

var_name: [
  (content)
]
Noticed this due to bracketing comments on #2443 and trying to think of better answers. With this, we can also say that the [] bracket a variable.
This commit is contained in:
Jon Ross-Perkins
2022-12-07 11:22:33 -08:00
committed by GitHub
parent 5f85822caa
commit 04f0288cd2
7 changed files with 118 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
toolchain/driver/driver.cpp
+3 -3
View File
@@ -171,7 +171,7 @@ auto Driver::RunDumpSubcommand(DiagnosticConsumer& consumer,
tokenized_source.Print(output_stream_);
return !tokenized_source.has_errors();
}
CARBON_VLOG() << tokenized_source;
CARBON_VLOG() << "tokenized_buffer: " << tokenized_source;
CARBON_VLOG() << "*** ParseTree::Parse ***\n";
auto parse_tree = ParseTree::Parse(tokenized_source, consumer, vlog_stream_);
@@ -181,7 +181,7 @@ auto Driver::RunDumpSubcommand(DiagnosticConsumer& consumer,
parse_tree.Print(output_stream_, parse_tree_preorder);
return !tokenized_source.has_errors() && !parse_tree.has_errors();
}
CARBON_VLOG() << parse_tree;
CARBON_VLOG() << "parse_tree: " << parse_tree;
const SemanticsIR builtin_ir = SemanticsIR::MakeBuiltinIR();
CARBON_VLOG() << "*** SemanticsIR::MakeFromParseTree ***\n";
@@ -194,7 +194,7 @@ auto Driver::RunDumpSubcommand(DiagnosticConsumer& consumer,
// TODO: Return false when SemanticsIR has errors (not supported right now).
return !tokenized_source.has_errors() && !parse_tree.has_errors();
}
CARBON_VLOG() << semantics_ir;
CARBON_VLOG() << "semantics_ir: " << semantics_ir;
llvm_unreachable("should handle all dump modes");
}
toolchain/driver/driver_test.cpp
+23 -23
View File
@@ -133,29 +133,29 @@ TEST(DriverTest, DumpTokens) {
auto tokenized_text = test_output_stream.TakeStr();
EXPECT_THAT(Yaml::Value::FromText(tokenized_text),
ElementsAre(Yaml::MappingValue{
{"token", Yaml::MappingValue{{"index", "0"},
{"kind", "Identifier"},
{"line", "1"},
{"column", "1"},
{"indent", "1"},
{"spelling", "Hello"},
{"identifier", "0"},
{"has_trailing_space", "true"}}},
{"token", Yaml::MappingValue{{"index", "1"},
{"kind", "Identifier"},
{"line", "1"},
{"column", "7"},
{"indent", "1"},
{"spelling", "World"},
{"identifier", "1"},
{"has_trailing_space", "true"}}},
{"token", Yaml::MappingValue{{"index", "2"},
{"kind", "EndOfFile"},
{"line", "1"},
{"column", "12"},
{"indent", "1"},
{"spelling", ""}}}}));
ElementsAre(Yaml::SequenceValue{
Yaml::MappingValue{{"index", "0"},
{"kind", "Identifier"},
{"line", "1"},
{"column", "1"},
{"indent", "1"},
{"spelling", "Hello"},
{"identifier", "0"},
{"has_trailing_space", "true"}},
Yaml::MappingValue{{"index", "1"},
{"kind", "Identifier"},
{"line", "1"},
{"column", "7"},
{"indent", "1"},
{"spelling", "World"},
{"identifier", "1"},
{"has_trailing_space", "true"}},
Yaml::MappingValue{{"index", "2"},
{"kind", "EndOfFile"},
{"line", "1"},
{"column", "12"},
{"indent", "1"},
{"spelling", ""}}}));
// Check that the subcommand dispatch works.
EXPECT_TRUE(driver.RunFullCommand({"dump", "tokens", test_file_path}));
toolchain/driver/testdata/errors_sorted_test.carbon
Vendored
+3 -1
View File
@@ -3,7 +3,9 @@
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// RUN: %{not} %{carbon-run-tokens}
// CHECK-COUNT-17:STDOUT: token: {{.*}}
// CHECK:STDOUT: [
// CHECK-COUNT-17:STDOUT: {{.*}}
// CHECK:STDOUT: ]
// CHECK:STDERR: {{.*}}/errors_sorted_test.carbon:[[@LINE+1]]:24: Closing symbol does not match most recent opening symbol.
fn run(String program) {
toolchain/driver/testdata/errors_streamed_test.carbon
Vendored
+3 -1
View File
@@ -4,7 +4,9 @@
//
// RUN: %{not} %{carbon} --print-errors=streamed dump tokens %s | \
// RUN: %{FileCheck-strict}
// CHECK-COUNT-17:STDOUT: token: {{.*}}
// CHECK:STDOUT: [
// CHECK-COUNT-17:STDOUT: {{.*}}
// CHECK:STDOUT: ]
fn run(String program) {
return True;
toolchain/lexer/testdata/carbon_test.carbon
Vendored
+14 -12
View File
@@ -4,19 +4,21 @@
//
// AUTOUPDATE
// RUN: %{carbon-run-tokens}
// CHECK:STDOUT: [
// CHECK:STDOUT: token: { index: 0, kind: 'Fn', line: {{ *}}[[@LINE+7]], column: 1, indent: 1, spelling: 'fn', has_trailing_space: true }
// CHECK:STDOUT: token: { index: 1, kind: 'Identifier', line: {{ *}}[[@LINE+6]], column: 4, indent: 1, spelling: 'run', identifier: 0 }
// CHECK:STDOUT: token: { index: 2, kind: 'OpenParen', line: {{ *}}[[@LINE+5]], column: 7, indent: 1, spelling: '(', closing_token: 5 }
// CHECK:STDOUT: token: { index: 3, kind: 'Identifier', line: {{ *}}[[@LINE+4]], column: 8, indent: 1, spelling: 'String', identifier: 1, has_trailing_space: true }
// CHECK:STDOUT: token: { index: 4, kind: 'Identifier', line: {{ *}}[[@LINE+3]], column: 15, indent: 1, spelling: 'program', identifier: 2 }
// CHECK:STDOUT: token: { index: 5, kind: 'CloseParen', line: {{ *}}[[@LINE+2]], column: 22, indent: 1, spelling: ')', opening_token: 2, has_trailing_space: true }
// CHECK:STDOUT: token: { index: 6, kind: 'OpenCurlyBrace', line: {{ *}}[[@LINE+1]], column: 24, indent: 1, spelling: '{', closing_token: 10, has_trailing_space: true }
// CHECK:STDOUT: { index: 0, kind: 'Fn', line: {{ *}}[[@LINE+7]], column: 1, indent: 1, spelling: 'fn', has_trailing_space: true },
// CHECK:STDOUT: { index: 1, kind: 'Identifier', line: {{ *}}[[@LINE+6]], column: 4, indent: 1, spelling: 'run', identifier: 0 },
// CHECK:STDOUT: { index: 2, kind: 'OpenParen', line: {{ *}}[[@LINE+5]], column: 7, indent: 1, spelling: '(', closing_token: 5 },
// CHECK:STDOUT: { index: 3, kind: 'Identifier', line: {{ *}}[[@LINE+4]], column: 8, indent: 1, spelling: 'String', identifier: 1, has_trailing_space: true },
// CHECK:STDOUT: { index: 4, kind: 'Identifier', line: {{ *}}[[@LINE+3]], column: 15, indent: 1, spelling: 'program', identifier: 2 },
// CHECK:STDOUT: { index: 5, kind: 'CloseParen', line: {{ *}}[[@LINE+2]], column: 22, indent: 1, spelling: ')', opening_token: 2, has_trailing_space: true },
// CHECK:STDOUT: { index: 6, kind: 'OpenCurlyBrace', line: {{ *}}[[@LINE+1]], column: 24, indent: 1, spelling: '{', closing_token: 10, has_trailing_space: true },
fn run(String program) {
// CHECK:STDOUT: token: { index: 7, kind: 'Return', line: {{ *}}[[@LINE+3]], column: 3, indent: 3, spelling: 'return', has_trailing_space: true }
// CHECK:STDOUT: token: { index: 8, kind: 'Identifier', line: {{ *}}[[@LINE+2]], column: 10, indent: 3, spelling: 'True', identifier: 3 }
// CHECK:STDOUT: token: { index: 9, kind: 'Semi', line: {{ *}}[[@LINE+1]], column: 14, indent: 3, spelling: ';', has_trailing_space: true }
// CHECK:STDOUT: { index: 7, kind: 'Return', line: {{ *}}[[@LINE+3]], column: 3, indent: 3, spelling: 'return', has_trailing_space: true },
// CHECK:STDOUT: { index: 8, kind: 'Identifier', line: {{ *}}[[@LINE+2]], column: 10, indent: 3, spelling: 'True', identifier: 3 },
// CHECK:STDOUT: { index: 9, kind: 'Semi', line: {{ *}}[[@LINE+1]], column: 14, indent: 3, spelling: ';', has_trailing_space: true },
return True;
// CHECK:STDOUT: token: { index: 10, kind: 'CloseCurlyBrace', line: {{ *}}[[@LINE+2]], column: 1, indent: 1, spelling: '}', opening_token: 6, has_trailing_space: true }
// CHECK:STDOUT: token: { index: 11, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: 2, indent: 1, spelling: '' }
// CHECK:STDOUT: { index: 10, kind: 'CloseCurlyBrace', line: {{ *}}[[@LINE+3]], column: 1, indent: 1, spelling: '}', opening_token: 6, has_trailing_space: true },
// CHECK:STDOUT: { index: 11, kind: 'EndOfFile', line: {{ *}}[[@LINE+2]], column: 2, indent: 1, spelling: '' },
// CHECK:STDOUT: ]
}
toolchain/lexer/tokenized_buffer.cpp
+4 -2
View File
@@ -788,10 +788,12 @@ auto TokenizedBuffer::Print(llvm::raw_ostream& output_stream) const -> void {
widths.Widen(GetTokenPrintWidths(token));
}
output_stream << "[\n";
for (Token token : tokens()) {
PrintToken(output_stream, token, widths);
output_stream << "\n";
}
output_stream << "]\n";
}
auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream,
@@ -810,7 +812,7 @@ auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream, Token token,
// justification manually in order to use the dynamically computed widths
// and get the quotes included.
output_stream << llvm::formatv(
"token: { index: {0}, kind: {1}, line: {2}, column: {3}, indent: {4}, "
"{ index: {0}, kind: {1}, line: {2}, column: {3}, indent: {4}, "
"spelling: '{5}'",
llvm::format_decimal(token_index, widths.index),
llvm::right_justify(
@@ -855,7 +857,7 @@ auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream, Token token,
output_stream << ", recovery: true";
}
output_stream << " }";
output_stream << " },";
}
auto TokenizedBuffer::GetLineInfo(Line line) -> LineInfo& {
toolchain/lexer/tokenized_buffer_test.cpp
+68 -79
View File
@@ -1032,22 +1032,6 @@ auto GetAndDropLine(llvm::StringRef& text) -> std::string {
return line.str();
}
TEST_F(LexerTest, PrintingBasic) {
auto buffer = Lex(";");
ASSERT_FALSE(buffer.has_errors());
std::string print_storage;
llvm::raw_string_ostream print_stream(print_storage);
buffer.Print(print_stream);
llvm::StringRef print = print_stream.str();
EXPECT_THAT(GetAndDropLine(print),
StrEq("token: { index: 0, kind: 'Semi', line: 1, column: 1, "
"indent: 1, spelling: ';', has_trailing_space: true }"));
EXPECT_THAT(GetAndDropLine(print),
StrEq("token: { index: 1, kind: 'EndOfFile', line: 1, column: 2, "
"indent: 1, spelling: '' }"));
EXPECT_TRUE(print.empty()) << print;
}
TEST_F(LexerTest, PrintingInteger) {
auto buffer = Lex("123");
ASSERT_FALSE(buffer.has_errors());
@@ -1055,11 +1039,13 @@ TEST_F(LexerTest, PrintingInteger) {
llvm::raw_string_ostream print_stream(print_storage);
buffer.Print(print_stream);
llvm::StringRef print = print_stream.str();
EXPECT_THAT(GetAndDropLine(print), StrEq("["));
EXPECT_THAT(GetAndDropLine(print),
StrEq("token: { index: 0, kind: 'IntegerLiteral', line: 1, "
StrEq("{ index: 0, kind: 'IntegerLiteral', line: 1, "
"column: 1, indent: 1, spelling: '123', value: `123`, "
"has_trailing_space: true }"));
"has_trailing_space: true },"));
EXPECT_THAT(GetAndDropLine(print), HasSubstr("'EndOfFile'"));
EXPECT_THAT(GetAndDropLine(print), StrEq("]"));
EXPECT_TRUE(print.empty()) << print;
}
@@ -1070,12 +1056,14 @@ TEST_F(LexerTest, PrintingReal) {
llvm::raw_string_ostream print_stream(print_storage);
buffer.Print(print_stream);
llvm::StringRef print = print_stream.str();
EXPECT_THAT(GetAndDropLine(print), StrEq("["));
EXPECT_THAT(
GetAndDropLine(print),
StrEq(
"token: { index: 0, kind: 'RealLiteral', line: 1, column: 1, indent: "
"1, spelling: '2.5', value: `25*10^-1`, has_trailing_space: true }"));
StrEq("{ index: 0, kind: 'RealLiteral', line: 1, column: 1, indent: "
"1, spelling: '2.5', value: `25*10^-1`, has_trailing_space: true "
"},"));
EXPECT_THAT(GetAndDropLine(print), HasSubstr("'EndOfFile'"));
EXPECT_THAT(GetAndDropLine(print), StrEq("]"));
EXPECT_TRUE(print.empty()) << print;
}
@@ -1087,25 +1075,27 @@ TEST_F(LexerTest, PrintingPadding) {
llvm::raw_string_ostream print_stream(print_storage);
buffer.Print(print_stream);
llvm::StringRef print = print_stream.str();
EXPECT_THAT(GetAndDropLine(print), StrEq("["));
EXPECT_THAT(GetAndDropLine(print),
StrEq("token: { index: 0, kind: 'OpenParen', line: 1, column: "
"1, indent: 1, spelling: '(', closing_token: 4 }"));
StrEq("{ index: 0, kind: 'OpenParen', line: 1, column: "
"1, indent: 1, spelling: '(', closing_token: 4 },"));
EXPECT_THAT(GetAndDropLine(print),
StrEq("token: { index: 1, kind: 'Semi', line: 1, column: "
"2, indent: 1, spelling: ';' }"));
StrEq("{ index: 1, kind: 'Semi', line: 1, column: "
"2, indent: 1, spelling: ';' },"));
EXPECT_THAT(GetAndDropLine(print),
StrEq("token: { index: 2, kind: 'Identifier', line: 1, column: "
"3, indent: 1, spelling: 'foo', identifier: 0 }"));
StrEq("{ index: 2, kind: 'Identifier', line: 1, column: "
"3, indent: 1, spelling: 'foo', identifier: 0 },"));
EXPECT_THAT(GetAndDropLine(print),
StrEq("token: { index: 3, kind: 'Semi', line: 1, column: "
"6, indent: 1, spelling: ';' }"));
StrEq("{ index: 3, kind: 'Semi', line: 1, column: "
"6, indent: 1, spelling: ';' },"));
EXPECT_THAT(GetAndDropLine(print),
StrEq("token: { index: 4, kind: 'CloseParen', line: 1, column: "
StrEq("{ index: 4, kind: 'CloseParen', line: 1, column: "
"7, indent: 1, spelling: ')', opening_token: 0, "
"has_trailing_space: true }"));
"has_trailing_space: true },"));
EXPECT_THAT(GetAndDropLine(print),
StrEq("token: { index: 5, kind: 'EndOfFile', line: 1, column: "
"8, indent: 1, spelling: '' }"));
StrEq("{ index: 5, kind: 'EndOfFile', line: 1, column: "
"8, indent: 1, spelling: '' },"));
EXPECT_THAT(GetAndDropLine(print), StrEq("]"));
EXPECT_TRUE(print.empty()) << print;
}
@@ -1117,22 +1107,20 @@ TEST_F(LexerTest, PrintingPaddingDigits) {
llvm::raw_string_ostream print_stream(print_storage);
buffer.Print(print_stream);
llvm::StringRef print = print_stream.str();
EXPECT_THAT(
GetAndDropLine(print),
StrEq("token: { index: 0, kind: 'Semi', line: 1, column: 1, "
"indent: 1, spelling: ';', has_trailing_space: true }"));
EXPECT_THAT(
GetAndDropLine(print),
StrEq("token: { index: 1, kind: 'Semi', line: 11, column: 9, "
"indent: 9, spelling: ';' }"));
EXPECT_THAT(
GetAndDropLine(print),
StrEq("token: { index: 2, kind: 'Semi', line: 11, column: 10, "
"indent: 9, spelling: ';', has_trailing_space: true }"));
EXPECT_THAT(
GetAndDropLine(print),
StrEq("token: { index: 3, kind: 'EndOfFile', line: 11, column: 11, "
"indent: 9, spelling: '' }"));
EXPECT_THAT(GetAndDropLine(print), StrEq("["));
EXPECT_THAT(GetAndDropLine(print),
StrEq("{ index: 0, kind: 'Semi', line: 1, column: 1, "
"indent: 1, spelling: ';', has_trailing_space: true },"));
EXPECT_THAT(GetAndDropLine(print),
StrEq("{ index: 1, kind: 'Semi', line: 11, column: 9, "
"indent: 9, spelling: ';' },"));
EXPECT_THAT(GetAndDropLine(print),
StrEq("{ index: 2, kind: 'Semi', line: 11, column: 10, "
"indent: 9, spelling: ';', has_trailing_space: true },"));
EXPECT_THAT(GetAndDropLine(print),
StrEq("{ index: 3, kind: 'EndOfFile', line: 11, column: 11, "
"indent: 9, spelling: '' },"));
EXPECT_THAT(GetAndDropLine(print), StrEq("]"));
EXPECT_TRUE(print.empty()) << print;
}
@@ -1146,34 +1134,34 @@ TEST_F(LexerTest, PrintingAsYaml) {
print_stream.flush();
EXPECT_THAT(Yaml::Value::FromText(print_output),
ElementsAre(Yaml::MappingValue{
{"token", Yaml::MappingValue{{"index", "0"},
{"kind", "Semi"},
{"line", "2"},
{"column", "2"},
{"indent", "2"},
{"spelling", ";"},
{"has_trailing_space", "true"}}},
{"token", Yaml::MappingValue{{"index", "1"},
{"kind", "Semi"},
{"line", "5"},
{"column", "1"},
{"indent", "1"},
{"spelling", ";"},
{"has_trailing_space", "true"}}},
{"token", Yaml::MappingValue{{"index", "2"},
{"kind", "Semi"},
{"line", "5"},
{"column", "3"},
{"indent", "1"},
{"spelling", ";"},
{"has_trailing_space", "true"}}},
{"token", Yaml::MappingValue{{"index", "3"},
{"kind", "EndOfFile"},
{"line", "15"},
{"column", "1"},
{"indent", "1"},
{"spelling", ""}}}}));
ElementsAre(Yaml::SequenceValue{
Yaml::MappingValue{{"index", "0"},
{"kind", "Semi"},
{"line", "2"},
{"column", "2"},
{"indent", "2"},
{"spelling", ";"},
{"has_trailing_space", "true"}},
Yaml::MappingValue{{"index", "1"},
{"kind", "Semi"},
{"line", "5"},
{"column", "1"},
{"indent", "1"},
{"spelling", ";"},
{"has_trailing_space", "true"}},
Yaml::MappingValue{{"index", "2"},
{"kind", "Semi"},
{"line", "5"},
{"column", "3"},
{"indent", "1"},
{"spelling", ";"},
{"has_trailing_space", "true"}},
Yaml::MappingValue{{"index", "3"},
{"kind", "EndOfFile"},
{"line", "15"},
{"column", "1"},
{"indent", "1"},
{"spelling", ""}}}));
}
TEST_F(LexerTest, PrintToken) {
@@ -1183,10 +1171,11 @@ TEST_F(LexerTest, PrintToken) {
llvm::raw_string_ostream print_stream(print_output);
buffer.Print(print_stream);
llvm::StringRef print = print_stream.str();
EXPECT_THAT(GetAndDropLine(print), StrEq("["));
EXPECT_THAT(GetAndDropLine(print),
StrEq("token: { index: 0, kind: 'IntegerLiteral', line: 1, "
StrEq("{ index: 0, kind: 'IntegerLiteral', line: 1, "
"column: 1, indent: 1, spelling: '0x9', value: `9`, "
"has_trailing_space: true }"));
"has_trailing_space: true },"));
}
} // namespace