From 04f0288cd2d49d85d833c7474f4d25db7db7fbd4 Mon Sep 17 00:00:00 2001 From: Jon Ross-Perkins Date: Wed, 7 Dec 2022 11:22:33 -0800 Subject: [PATCH] Bracket the tokenized buffer output. (#2446) This makes TokenizedBuffer more consistent with ParseTree and SemanticsIR, which also wrap with [] to produce a sequence value. It also makes it possible in driver.cpp to just prefix the line with a name, so it ends up with: var_name: [ (content) ] Noticed this due to bracketing comments on #2443 and trying to think of better answers. With this, we can also say that the [] bracket a variable. --- toolchain/driver/driver.cpp | 6 +- toolchain/driver/driver_test.cpp | 46 +++--- .../driver/testdata/errors_sorted_test.carbon | 4 +- .../testdata/errors_streamed_test.carbon | 4 +- toolchain/lexer/testdata/carbon_test.carbon | 26 ++-- toolchain/lexer/tokenized_buffer.cpp | 6 +- toolchain/lexer/tokenized_buffer_test.cpp | 147 ++++++++---------- 7 files changed, 118 insertions(+), 121 deletions(-) diff --git a/toolchain/driver/driver.cpp b/toolchain/driver/driver.cpp index 0db8a8090381..8636605b7d73 100644 --- a/toolchain/driver/driver.cpp +++ b/toolchain/driver/driver.cpp @@ -171,7 +171,7 @@ auto Driver::RunDumpSubcommand(DiagnosticConsumer& consumer, tokenized_source.Print(output_stream_); return !tokenized_source.has_errors(); } - CARBON_VLOG() << tokenized_source; + CARBON_VLOG() << "tokenized_buffer: " << tokenized_source; CARBON_VLOG() << "*** ParseTree::Parse ***\n"; auto parse_tree = ParseTree::Parse(tokenized_source, consumer, vlog_stream_); @@ -181,7 +181,7 @@ auto Driver::RunDumpSubcommand(DiagnosticConsumer& consumer, parse_tree.Print(output_stream_, parse_tree_preorder); return !tokenized_source.has_errors() && !parse_tree.has_errors(); } - CARBON_VLOG() << parse_tree; + CARBON_VLOG() << "parse_tree: " << parse_tree; const SemanticsIR builtin_ir = SemanticsIR::MakeBuiltinIR(); CARBON_VLOG() << "*** SemanticsIR::MakeFromParseTree ***\n"; @@ -194,7 +194,7 @@ auto Driver::RunDumpSubcommand(DiagnosticConsumer& consumer, // TODO: Return false when SemanticsIR has errors (not supported right now). return !tokenized_source.has_errors() && !parse_tree.has_errors(); } - CARBON_VLOG() << semantics_ir; + CARBON_VLOG() << "semantics_ir: " << semantics_ir; llvm_unreachable("should handle all dump modes"); } diff --git a/toolchain/driver/driver_test.cpp b/toolchain/driver/driver_test.cpp index 9a2ce136f57d..c67493f79d6c 100644 --- a/toolchain/driver/driver_test.cpp +++ b/toolchain/driver/driver_test.cpp @@ -133,29 +133,29 @@ TEST(DriverTest, DumpTokens) { auto tokenized_text = test_output_stream.TakeStr(); EXPECT_THAT(Yaml::Value::FromText(tokenized_text), - ElementsAre(Yaml::MappingValue{ - {"token", Yaml::MappingValue{{"index", "0"}, - {"kind", "Identifier"}, - {"line", "1"}, - {"column", "1"}, - {"indent", "1"}, - {"spelling", "Hello"}, - {"identifier", "0"}, - {"has_trailing_space", "true"}}}, - {"token", Yaml::MappingValue{{"index", "1"}, - {"kind", "Identifier"}, - {"line", "1"}, - {"column", "7"}, - {"indent", "1"}, - {"spelling", "World"}, - {"identifier", "1"}, - {"has_trailing_space", "true"}}}, - {"token", Yaml::MappingValue{{"index", "2"}, - {"kind", "EndOfFile"}, - {"line", "1"}, - {"column", "12"}, - {"indent", "1"}, - {"spelling", ""}}}})); + ElementsAre(Yaml::SequenceValue{ + Yaml::MappingValue{{"index", "0"}, + {"kind", "Identifier"}, + {"line", "1"}, + {"column", "1"}, + {"indent", "1"}, + {"spelling", "Hello"}, + {"identifier", "0"}, + {"has_trailing_space", "true"}}, + Yaml::MappingValue{{"index", "1"}, + {"kind", "Identifier"}, + {"line", "1"}, + {"column", "7"}, + {"indent", "1"}, + {"spelling", "World"}, + {"identifier", "1"}, + {"has_trailing_space", "true"}}, + Yaml::MappingValue{{"index", "2"}, + {"kind", "EndOfFile"}, + {"line", "1"}, + {"column", "12"}, + {"indent", "1"}, + {"spelling", ""}}})); // Check that the subcommand dispatch works. EXPECT_TRUE(driver.RunFullCommand({"dump", "tokens", test_file_path})); diff --git a/toolchain/driver/testdata/errors_sorted_test.carbon b/toolchain/driver/testdata/errors_sorted_test.carbon index de9ddf8ec1d6..1e1419f46e93 100644 --- a/toolchain/driver/testdata/errors_sorted_test.carbon +++ b/toolchain/driver/testdata/errors_sorted_test.carbon @@ -3,7 +3,9 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // // RUN: %{not} %{carbon-run-tokens} -// CHECK-COUNT-17:STDOUT: token: {{.*}} +// CHECK:STDOUT: [ +// CHECK-COUNT-17:STDOUT: {{.*}} +// CHECK:STDOUT: ] // CHECK:STDERR: {{.*}}/errors_sorted_test.carbon:[[@LINE+1]]:24: Closing symbol does not match most recent opening symbol. fn run(String program) { diff --git a/toolchain/driver/testdata/errors_streamed_test.carbon b/toolchain/driver/testdata/errors_streamed_test.carbon index 84d28551b696..504653be478f 100644 --- a/toolchain/driver/testdata/errors_streamed_test.carbon +++ b/toolchain/driver/testdata/errors_streamed_test.carbon @@ -4,7 +4,9 @@ // // RUN: %{not} %{carbon} --print-errors=streamed dump tokens %s | \ // RUN: %{FileCheck-strict} -// CHECK-COUNT-17:STDOUT: token: {{.*}} +// CHECK:STDOUT: [ +// CHECK-COUNT-17:STDOUT: {{.*}} +// CHECK:STDOUT: ] fn run(String program) { return True; diff --git a/toolchain/lexer/testdata/carbon_test.carbon b/toolchain/lexer/testdata/carbon_test.carbon index eeb4f10d0351..8ba0be71cdc2 100644 --- a/toolchain/lexer/testdata/carbon_test.carbon +++ b/toolchain/lexer/testdata/carbon_test.carbon @@ -4,19 +4,21 @@ // // AUTOUPDATE // RUN: %{carbon-run-tokens} +// CHECK:STDOUT: [ -// CHECK:STDOUT: token: { index: 0, kind: 'Fn', line: {{ *}}[[@LINE+7]], column: 1, indent: 1, spelling: 'fn', has_trailing_space: true } -// CHECK:STDOUT: token: { index: 1, kind: 'Identifier', line: {{ *}}[[@LINE+6]], column: 4, indent: 1, spelling: 'run', identifier: 0 } -// CHECK:STDOUT: token: { index: 2, kind: 'OpenParen', line: {{ *}}[[@LINE+5]], column: 7, indent: 1, spelling: '(', closing_token: 5 } -// CHECK:STDOUT: token: { index: 3, kind: 'Identifier', line: {{ *}}[[@LINE+4]], column: 8, indent: 1, spelling: 'String', identifier: 1, has_trailing_space: true } -// CHECK:STDOUT: token: { index: 4, kind: 'Identifier', line: {{ *}}[[@LINE+3]], column: 15, indent: 1, spelling: 'program', identifier: 2 } -// CHECK:STDOUT: token: { index: 5, kind: 'CloseParen', line: {{ *}}[[@LINE+2]], column: 22, indent: 1, spelling: ')', opening_token: 2, has_trailing_space: true } -// CHECK:STDOUT: token: { index: 6, kind: 'OpenCurlyBrace', line: {{ *}}[[@LINE+1]], column: 24, indent: 1, spelling: '{', closing_token: 10, has_trailing_space: true } +// CHECK:STDOUT: { index: 0, kind: 'Fn', line: {{ *}}[[@LINE+7]], column: 1, indent: 1, spelling: 'fn', has_trailing_space: true }, +// CHECK:STDOUT: { index: 1, kind: 'Identifier', line: {{ *}}[[@LINE+6]], column: 4, indent: 1, spelling: 'run', identifier: 0 }, +// CHECK:STDOUT: { index: 2, kind: 'OpenParen', line: {{ *}}[[@LINE+5]], column: 7, indent: 1, spelling: '(', closing_token: 5 }, +// CHECK:STDOUT: { index: 3, kind: 'Identifier', line: {{ *}}[[@LINE+4]], column: 8, indent: 1, spelling: 'String', identifier: 1, has_trailing_space: true }, +// CHECK:STDOUT: { index: 4, kind: 'Identifier', line: {{ *}}[[@LINE+3]], column: 15, indent: 1, spelling: 'program', identifier: 2 }, +// CHECK:STDOUT: { index: 5, kind: 'CloseParen', line: {{ *}}[[@LINE+2]], column: 22, indent: 1, spelling: ')', opening_token: 2, has_trailing_space: true }, +// CHECK:STDOUT: { index: 6, kind: 'OpenCurlyBrace', line: {{ *}}[[@LINE+1]], column: 24, indent: 1, spelling: '{', closing_token: 10, has_trailing_space: true }, fn run(String program) { - // CHECK:STDOUT: token: { index: 7, kind: 'Return', line: {{ *}}[[@LINE+3]], column: 3, indent: 3, spelling: 'return', has_trailing_space: true } - // CHECK:STDOUT: token: { index: 8, kind: 'Identifier', line: {{ *}}[[@LINE+2]], column: 10, indent: 3, spelling: 'True', identifier: 3 } - // CHECK:STDOUT: token: { index: 9, kind: 'Semi', line: {{ *}}[[@LINE+1]], column: 14, indent: 3, spelling: ';', has_trailing_space: true } + // CHECK:STDOUT: { index: 7, kind: 'Return', line: {{ *}}[[@LINE+3]], column: 3, indent: 3, spelling: 'return', has_trailing_space: true }, + // CHECK:STDOUT: { index: 8, kind: 'Identifier', line: {{ *}}[[@LINE+2]], column: 10, indent: 3, spelling: 'True', identifier: 3 }, + // CHECK:STDOUT: { index: 9, kind: 'Semi', line: {{ *}}[[@LINE+1]], column: 14, indent: 3, spelling: ';', has_trailing_space: true }, return True; -// CHECK:STDOUT: token: { index: 10, kind: 'CloseCurlyBrace', line: {{ *}}[[@LINE+2]], column: 1, indent: 1, spelling: '}', opening_token: 6, has_trailing_space: true } -// CHECK:STDOUT: token: { index: 11, kind: 'EndOfFile', line: {{ *}}[[@LINE+1]], column: 2, indent: 1, spelling: '' } +// CHECK:STDOUT: { index: 10, kind: 'CloseCurlyBrace', line: {{ *}}[[@LINE+3]], column: 1, indent: 1, spelling: '}', opening_token: 6, has_trailing_space: true }, +// CHECK:STDOUT: { index: 11, kind: 'EndOfFile', line: {{ *}}[[@LINE+2]], column: 2, indent: 1, spelling: '' }, +// CHECK:STDOUT: ] } diff --git a/toolchain/lexer/tokenized_buffer.cpp b/toolchain/lexer/tokenized_buffer.cpp index 087ff88d5e3f..176955ab80e1 100644 --- a/toolchain/lexer/tokenized_buffer.cpp +++ b/toolchain/lexer/tokenized_buffer.cpp @@ -788,10 +788,12 @@ auto TokenizedBuffer::Print(llvm::raw_ostream& output_stream) const -> void { widths.Widen(GetTokenPrintWidths(token)); } + output_stream << "[\n"; for (Token token : tokens()) { PrintToken(output_stream, token, widths); output_stream << "\n"; } + output_stream << "]\n"; } auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream, @@ -810,7 +812,7 @@ auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream, Token token, // justification manually in order to use the dynamically computed widths // and get the quotes included. output_stream << llvm::formatv( - "token: { index: {0}, kind: {1}, line: {2}, column: {3}, indent: {4}, " + "{ index: {0}, kind: {1}, line: {2}, column: {3}, indent: {4}, " "spelling: '{5}'", llvm::format_decimal(token_index, widths.index), llvm::right_justify( @@ -855,7 +857,7 @@ auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream, Token token, output_stream << ", recovery: true"; } - output_stream << " }"; + output_stream << " },"; } auto TokenizedBuffer::GetLineInfo(Line line) -> LineInfo& { diff --git a/toolchain/lexer/tokenized_buffer_test.cpp b/toolchain/lexer/tokenized_buffer_test.cpp index 2bceed8dc3c0..b7f0d34d14df 100644 --- a/toolchain/lexer/tokenized_buffer_test.cpp +++ b/toolchain/lexer/tokenized_buffer_test.cpp @@ -1032,22 +1032,6 @@ auto GetAndDropLine(llvm::StringRef& text) -> std::string { return line.str(); } -TEST_F(LexerTest, PrintingBasic) { - auto buffer = Lex(";"); - ASSERT_FALSE(buffer.has_errors()); - std::string print_storage; - llvm::raw_string_ostream print_stream(print_storage); - buffer.Print(print_stream); - llvm::StringRef print = print_stream.str(); - EXPECT_THAT(GetAndDropLine(print), - StrEq("token: { index: 0, kind: 'Semi', line: 1, column: 1, " - "indent: 1, spelling: ';', has_trailing_space: true }")); - EXPECT_THAT(GetAndDropLine(print), - StrEq("token: { index: 1, kind: 'EndOfFile', line: 1, column: 2, " - "indent: 1, spelling: '' }")); - EXPECT_TRUE(print.empty()) << print; -} - TEST_F(LexerTest, PrintingInteger) { auto buffer = Lex("123"); ASSERT_FALSE(buffer.has_errors()); @@ -1055,11 +1039,13 @@ TEST_F(LexerTest, PrintingInteger) { llvm::raw_string_ostream print_stream(print_storage); buffer.Print(print_stream); llvm::StringRef print = print_stream.str(); + EXPECT_THAT(GetAndDropLine(print), StrEq("[")); EXPECT_THAT(GetAndDropLine(print), - StrEq("token: { index: 0, kind: 'IntegerLiteral', line: 1, " + StrEq("{ index: 0, kind: 'IntegerLiteral', line: 1, " "column: 1, indent: 1, spelling: '123', value: `123`, " - "has_trailing_space: true }")); + "has_trailing_space: true },")); EXPECT_THAT(GetAndDropLine(print), HasSubstr("'EndOfFile'")); + EXPECT_THAT(GetAndDropLine(print), StrEq("]")); EXPECT_TRUE(print.empty()) << print; } @@ -1070,12 +1056,14 @@ TEST_F(LexerTest, PrintingReal) { llvm::raw_string_ostream print_stream(print_storage); buffer.Print(print_stream); llvm::StringRef print = print_stream.str(); + EXPECT_THAT(GetAndDropLine(print), StrEq("[")); EXPECT_THAT( GetAndDropLine(print), - StrEq( - "token: { index: 0, kind: 'RealLiteral', line: 1, column: 1, indent: " - "1, spelling: '2.5', value: `25*10^-1`, has_trailing_space: true }")); + StrEq("{ index: 0, kind: 'RealLiteral', line: 1, column: 1, indent: " + "1, spelling: '2.5', value: `25*10^-1`, has_trailing_space: true " + "},")); EXPECT_THAT(GetAndDropLine(print), HasSubstr("'EndOfFile'")); + EXPECT_THAT(GetAndDropLine(print), StrEq("]")); EXPECT_TRUE(print.empty()) << print; } @@ -1087,25 +1075,27 @@ TEST_F(LexerTest, PrintingPadding) { llvm::raw_string_ostream print_stream(print_storage); buffer.Print(print_stream); llvm::StringRef print = print_stream.str(); + EXPECT_THAT(GetAndDropLine(print), StrEq("[")); EXPECT_THAT(GetAndDropLine(print), - StrEq("token: { index: 0, kind: 'OpenParen', line: 1, column: " - "1, indent: 1, spelling: '(', closing_token: 4 }")); + StrEq("{ index: 0, kind: 'OpenParen', line: 1, column: " + "1, indent: 1, spelling: '(', closing_token: 4 },")); EXPECT_THAT(GetAndDropLine(print), - StrEq("token: { index: 1, kind: 'Semi', line: 1, column: " - "2, indent: 1, spelling: ';' }")); + StrEq("{ index: 1, kind: 'Semi', line: 1, column: " + "2, indent: 1, spelling: ';' },")); EXPECT_THAT(GetAndDropLine(print), - StrEq("token: { index: 2, kind: 'Identifier', line: 1, column: " - "3, indent: 1, spelling: 'foo', identifier: 0 }")); + StrEq("{ index: 2, kind: 'Identifier', line: 1, column: " + "3, indent: 1, spelling: 'foo', identifier: 0 },")); EXPECT_THAT(GetAndDropLine(print), - StrEq("token: { index: 3, kind: 'Semi', line: 1, column: " - "6, indent: 1, spelling: ';' }")); + StrEq("{ index: 3, kind: 'Semi', line: 1, column: " + "6, indent: 1, spelling: ';' },")); EXPECT_THAT(GetAndDropLine(print), - StrEq("token: { index: 4, kind: 'CloseParen', line: 1, column: " + StrEq("{ index: 4, kind: 'CloseParen', line: 1, column: " "7, indent: 1, spelling: ')', opening_token: 0, " - "has_trailing_space: true }")); + "has_trailing_space: true },")); EXPECT_THAT(GetAndDropLine(print), - StrEq("token: { index: 5, kind: 'EndOfFile', line: 1, column: " - "8, indent: 1, spelling: '' }")); + StrEq("{ index: 5, kind: 'EndOfFile', line: 1, column: " + "8, indent: 1, spelling: '' },")); + EXPECT_THAT(GetAndDropLine(print), StrEq("]")); EXPECT_TRUE(print.empty()) << print; } @@ -1117,22 +1107,20 @@ TEST_F(LexerTest, PrintingPaddingDigits) { llvm::raw_string_ostream print_stream(print_storage); buffer.Print(print_stream); llvm::StringRef print = print_stream.str(); - EXPECT_THAT( - GetAndDropLine(print), - StrEq("token: { index: 0, kind: 'Semi', line: 1, column: 1, " - "indent: 1, spelling: ';', has_trailing_space: true }")); - EXPECT_THAT( - GetAndDropLine(print), - StrEq("token: { index: 1, kind: 'Semi', line: 11, column: 9, " - "indent: 9, spelling: ';' }")); - EXPECT_THAT( - GetAndDropLine(print), - StrEq("token: { index: 2, kind: 'Semi', line: 11, column: 10, " - "indent: 9, spelling: ';', has_trailing_space: true }")); - EXPECT_THAT( - GetAndDropLine(print), - StrEq("token: { index: 3, kind: 'EndOfFile', line: 11, column: 11, " - "indent: 9, spelling: '' }")); + EXPECT_THAT(GetAndDropLine(print), StrEq("[")); + EXPECT_THAT(GetAndDropLine(print), + StrEq("{ index: 0, kind: 'Semi', line: 1, column: 1, " + "indent: 1, spelling: ';', has_trailing_space: true },")); + EXPECT_THAT(GetAndDropLine(print), + StrEq("{ index: 1, kind: 'Semi', line: 11, column: 9, " + "indent: 9, spelling: ';' },")); + EXPECT_THAT(GetAndDropLine(print), + StrEq("{ index: 2, kind: 'Semi', line: 11, column: 10, " + "indent: 9, spelling: ';', has_trailing_space: true },")); + EXPECT_THAT(GetAndDropLine(print), + StrEq("{ index: 3, kind: 'EndOfFile', line: 11, column: 11, " + "indent: 9, spelling: '' },")); + EXPECT_THAT(GetAndDropLine(print), StrEq("]")); EXPECT_TRUE(print.empty()) << print; } @@ -1146,34 +1134,34 @@ TEST_F(LexerTest, PrintingAsYaml) { print_stream.flush(); EXPECT_THAT(Yaml::Value::FromText(print_output), - ElementsAre(Yaml::MappingValue{ - {"token", Yaml::MappingValue{{"index", "0"}, - {"kind", "Semi"}, - {"line", "2"}, - {"column", "2"}, - {"indent", "2"}, - {"spelling", ";"}, - {"has_trailing_space", "true"}}}, - {"token", Yaml::MappingValue{{"index", "1"}, - {"kind", "Semi"}, - {"line", "5"}, - {"column", "1"}, - {"indent", "1"}, - {"spelling", ";"}, - {"has_trailing_space", "true"}}}, - {"token", Yaml::MappingValue{{"index", "2"}, - {"kind", "Semi"}, - {"line", "5"}, - {"column", "3"}, - {"indent", "1"}, - {"spelling", ";"}, - {"has_trailing_space", "true"}}}, - {"token", Yaml::MappingValue{{"index", "3"}, - {"kind", "EndOfFile"}, - {"line", "15"}, - {"column", "1"}, - {"indent", "1"}, - {"spelling", ""}}}})); + ElementsAre(Yaml::SequenceValue{ + Yaml::MappingValue{{"index", "0"}, + {"kind", "Semi"}, + {"line", "2"}, + {"column", "2"}, + {"indent", "2"}, + {"spelling", ";"}, + {"has_trailing_space", "true"}}, + Yaml::MappingValue{{"index", "1"}, + {"kind", "Semi"}, + {"line", "5"}, + {"column", "1"}, + {"indent", "1"}, + {"spelling", ";"}, + {"has_trailing_space", "true"}}, + Yaml::MappingValue{{"index", "2"}, + {"kind", "Semi"}, + {"line", "5"}, + {"column", "3"}, + {"indent", "1"}, + {"spelling", ";"}, + {"has_trailing_space", "true"}}, + Yaml::MappingValue{{"index", "3"}, + {"kind", "EndOfFile"}, + {"line", "15"}, + {"column", "1"}, + {"indent", "1"}, + {"spelling", ""}}})); } TEST_F(LexerTest, PrintToken) { @@ -1183,10 +1171,11 @@ TEST_F(LexerTest, PrintToken) { llvm::raw_string_ostream print_stream(print_output); buffer.Print(print_stream); llvm::StringRef print = print_stream.str(); + EXPECT_THAT(GetAndDropLine(print), StrEq("[")); EXPECT_THAT(GetAndDropLine(print), - StrEq("token: { index: 0, kind: 'IntegerLiteral', line: 1, " + StrEq("{ index: 0, kind: 'IntegerLiteral', line: 1, " "column: 1, indent: 1, spelling: '0x9', value: `9`, " - "has_trailing_space: true }")); + "has_trailing_space: true },")); } } // namespace