From f4f9b232913edda6caba7ece536ccf4ffceaf9bd Mon Sep 17 00:00:00 2001 From: Jon Meow <46229924+jonmeow@users.noreply.github.com> Date: Thu, 3 Mar 2022 09:57:41 -0800 Subject: [PATCH] Add int and real printing (#1116) --- toolchain/lexer/tokenized_buffer.cpp | 33 +++++++++----- toolchain/lexer/tokenized_buffer.h | 5 +++ toolchain/lexer/tokenized_buffer_test.cpp | 53 +++++++++++++++++++---- 3 files changed, 72 insertions(+), 19 deletions(-) diff --git a/toolchain/lexer/tokenized_buffer.cpp b/toolchain/lexer/tokenized_buffer.cpp index 197410394673..cddd07416c55 100644 --- a/toolchain/lexer/tokenized_buffer.cpp +++ b/toolchain/lexer/tokenized_buffer.cpp @@ -820,18 +820,29 @@ auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream, Token token, widths.indent), token_text); - if (token_info.kind == TokenKind::Identifier()) { - output_stream << ", identifier: " << GetIdentifier(token).index_; - } else if (token_info.kind.IsOpeningSymbol()) { - output_stream << ", closing_token: " - << GetMatchedClosingToken(token).index_; - } else if (token_info.kind.IsClosingSymbol()) { - output_stream << ", opening_token: " - << GetMatchedOpeningToken(token).index_; - } else if (token_info.kind == TokenKind::StringLiteral()) { - output_stream << ", value: `" << GetStringLiteral(token) << "`"; + switch (token_info.kind) { + case TokenKind::Identifier(): + output_stream << ", identifier: " << GetIdentifier(token).index_; + break; + case TokenKind::IntegerLiteral(): + output_stream << ", value: `" << GetIntegerLiteral(token) << "`"; + break; + case TokenKind::RealLiteral(): + output_stream << ", value: `" << GetRealLiteral(token) << "`"; + break; + case TokenKind::StringLiteral(): + output_stream << ", value: `" << GetStringLiteral(token) << "`"; + break; + default: + if (token_info.kind.IsOpeningSymbol()) { + output_stream << ", closing_token: " + << GetMatchedClosingToken(token).index_; + } else if (token_info.kind.IsClosingSymbol()) { + output_stream << ", opening_token: " + << GetMatchedOpeningToken(token).index_; + } + break; } - // TODO: Include value for numeric literals. if (token_info.has_trailing_space) { output_stream << ", has_trailing_space: true"; diff --git a/toolchain/lexer/tokenized_buffer.h b/toolchain/lexer/tokenized_buffer.h index b0d6638acb2c..7bb01b73f0b2 100644 --- a/toolchain/lexer/tokenized_buffer.h +++ b/toolchain/lexer/tokenized_buffer.h @@ -227,6 +227,11 @@ class TokenizedBuffer { // If true, the value is mantissa * 10^exponent. [[nodiscard]] auto IsDecimal() const -> bool { return is_decimal_; } + void Print(llvm::raw_ostream& output_stream) const { + output_stream << Mantissa() << "*" << (is_decimal_ ? "10" : "2") << "^" + << Exponent(); + } + private: friend class TokenizedBuffer; diff --git a/toolchain/lexer/tokenized_buffer_test.cpp b/toolchain/lexer/tokenized_buffer_test.cpp index 319edd4faf58..86b191451cc6 100644 --- a/toolchain/lexer/tokenized_buffer_test.cpp +++ b/toolchain/lexer/tokenized_buffer_test.cpp @@ -989,7 +989,7 @@ auto GetAndDropLine(llvm::StringRef& text) -> std::string { return line.str(); } -TEST_F(LexerTest, Printing) { +TEST_F(LexerTest, PrintingBasic) { auto buffer = Lex(";"); ASSERT_FALSE(buffer.HasErrors()); std::string print_storage; @@ -1003,13 +1003,47 @@ TEST_F(LexerTest, Printing) { StrEq("token: { index: 1, kind: 'EndOfFile', line: 1, column: 2, " "indent: 1, spelling: '' }")); EXPECT_TRUE(print.empty()) << print; +} - // Test kind padding. - buffer = Lex("(;foo;)"); +TEST_F(LexerTest, PrintingInteger) { + auto buffer = Lex("123"); ASSERT_FALSE(buffer.HasErrors()); - print_storage.clear(); + std::string print_storage; + llvm::raw_string_ostream print_stream(print_storage); buffer.Print(print_stream); - print = print_stream.str(); + llvm::StringRef print = print_stream.str(); + EXPECT_THAT(GetAndDropLine(print), + StrEq("token: { index: 0, kind: 'IntegerLiteral', line: 1, " + "column: 1, indent: 1, spelling: '123', value: `123`, " + "has_trailing_space: true }")); + EXPECT_THAT(GetAndDropLine(print), HasSubstr("'EndOfFile'")); + EXPECT_TRUE(print.empty()) << print; +} + +TEST_F(LexerTest, PrintingReal) { + auto buffer = Lex("2.5"); + ASSERT_FALSE(buffer.HasErrors()); + std::string print_storage; + llvm::raw_string_ostream print_stream(print_storage); + buffer.Print(print_stream); + llvm::StringRef print = print_stream.str(); + EXPECT_THAT( + GetAndDropLine(print), + StrEq( + "token: { index: 0, kind: 'RealLiteral', line: 1, column: 1, indent: " + "1, spelling: '2.5', value: `25*10^-1`, has_trailing_space: true }")); + EXPECT_THAT(GetAndDropLine(print), HasSubstr("'EndOfFile'")); + EXPECT_TRUE(print.empty()) << print; +} + +TEST_F(LexerTest, PrintingPadding) { + // Test kind padding. + auto buffer = Lex("(;foo;)"); + ASSERT_FALSE(buffer.HasErrors()); + std::string print_storage; + llvm::raw_string_ostream print_stream(print_storage); + buffer.Print(print_stream); + llvm::StringRef print = print_stream.str(); EXPECT_THAT(GetAndDropLine(print), StrEq("token: { index: 0, kind: 'OpenParen', line: 1, column: " "1, indent: 1, spelling: '(', closing_token: 4 }")); @@ -1030,13 +1064,16 @@ TEST_F(LexerTest, Printing) { StrEq("token: { index: 5, kind: 'EndOfFile', line: 1, column: " "8, indent: 1, spelling: '' }")); EXPECT_TRUE(print.empty()) << print; +} +TEST_F(LexerTest, PrintingPaddingDigits) { // Test digit padding with max values of 9, 10, and 11. - buffer = Lex(";\n\n\n\n\n\n\n\n\n\n ;;"); + auto buffer = Lex(";\n\n\n\n\n\n\n\n\n\n ;;"); ASSERT_FALSE(buffer.HasErrors()); - print_storage.clear(); + std::string print_storage; + llvm::raw_string_ostream print_stream(print_storage); buffer.Print(print_stream); - print = print_stream.str(); + llvm::StringRef print = print_stream.str(); EXPECT_THAT( GetAndDropLine(print), StrEq("token: { index: 0, kind: 'Semi', line: 1, column: 1, "