From 3d441691992e9afeeea29c534b944cea03aee14b Mon Sep 17 00:00:00 2001 From: Kareem Ergawy Date: Wed, 17 Aug 2022 18:29:07 +0200 Subject: [PATCH] Fix integer literal token printing. (#2050) Summary: An `llvm::APInt` is always treated as a signed value by `operator<<`; check [1]. This resulted in printing incorrect values for tokens that have their MSB set to 1. For example, a value 9 would be printed as -7 since its `APInt` object would be 4-bits wide. However, integer literals are always tokenized without the sign character so it is safe to treat the values as unsigned for printing pruposes. [1] https://llvm.org/doxygen/APInt_8h_source.html Co-authored-by: ergawy --- toolchain/lexer/tokenized_buffer.cpp | 4 +++- toolchain/lexer/tokenized_buffer_test.cpp | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/toolchain/lexer/tokenized_buffer.cpp b/toolchain/lexer/tokenized_buffer.cpp index d427b2e34cb5..8b082090269d 100644 --- a/toolchain/lexer/tokenized_buffer.cpp +++ b/toolchain/lexer/tokenized_buffer.cpp @@ -828,7 +828,9 @@ auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream, Token token, output_stream << ", identifier: " << GetIdentifier(token).index_; break; case TokenKind::IntegerLiteral(): - output_stream << ", value: `" << GetIntegerLiteral(token) << "`"; + output_stream << ", value: `"; + GetIntegerLiteral(token).print(output_stream, /*isSigned=*/false); + output_stream << "`"; break; case TokenKind::RealLiteral(): output_stream << ", value: `" << GetRealLiteral(token) << "`"; diff --git a/toolchain/lexer/tokenized_buffer_test.cpp b/toolchain/lexer/tokenized_buffer_test.cpp index d8131f29a9f9..26ea70afd883 100644 --- a/toolchain/lexer/tokenized_buffer_test.cpp +++ b/toolchain/lexer/tokenized_buffer_test.cpp @@ -1175,5 +1175,18 @@ TEST_F(LexerTest, PrintingAsYaml) { {"spelling", ""}}}})); } +TEST_F(LexerTest, PrintToken) { + auto buffer = Lex("0x9"); + ASSERT_FALSE(buffer.has_errors()); + std::string print_output; + llvm::raw_string_ostream print_stream(print_output); + buffer.Print(print_stream); + llvm::StringRef print = print_stream.str(); + EXPECT_THAT(GetAndDropLine(print), + StrEq("token: { index: 0, kind: 'IntegerLiteral', line: 1, " + "column: 1, indent: 1, spelling: '0x9', value: `9`, " + "has_trailing_space: true }")); +} + } // namespace } // namespace Carbon::Testing