diff --git a/toolchain/check/testdata/basics/type_literals.carbon b/toolchain/check/testdata/basics/type_literals.carbon index a3d103f873c1..54ae4cdc31b1 100644 --- a/toolchain/check/testdata/basics/type_literals.carbon +++ b/toolchain/check/testdata/basics/type_literals.carbon @@ -39,10 +39,18 @@ var test_i15: i15; // CHECK:STDERR: var test_i1000000000: i1000000000; // TODO: This diagnostic is not very good. -// CHECK:STDERR: fail_iN_bad_width.carbon:[[@LINE+4]]:33: error: integer literal with value 10000000000000000000 does not fit in i32 [IntLiteralTooLargeForI32] +// CHECK:STDERR: fail_iN_bad_width.carbon:[[@LINE+12]]:33: error: expected expression [ExpectedExpr] // CHECK:STDERR: var test_i10000000000000000000: i10000000000000000000; // CHECK:STDERR: ^~~~~~~~~~~~~~~~~~~~~ // CHECK:STDERR: +// CHECK:STDERR: fail_iN_bad_width.carbon:[[@LINE+8]]:33: error: semantics TODO: `HandleInvalidParse` [SemanticsTodo] +// CHECK:STDERR: var test_i10000000000000000000: i10000000000000000000; +// CHECK:STDERR: ^~~~~~~~~~~~~~~~~~~~~ +// CHECK:STDERR: +// CHECK:STDERR: fail_iN_bad_width.carbon:[[@LINE+4]]:34: error: found a type literal with a bit width using 20 digits, which is greater than the limit of 18 [TooManyTypeBitWidthDigits] +// CHECK:STDERR: var test_i10000000000000000000: i10000000000000000000; +// CHECK:STDERR: ^ +// CHECK:STDERR: var test_i10000000000000000000: i10000000000000000000; // --- uN.carbon @@ -76,10 +84,18 @@ var test_u15: u15; // CHECK:STDERR: var test_u1000000000: u1000000000; // TODO: This diagnostic is not very good. -// CHECK:STDERR: fail_uN_bad_width.carbon:[[@LINE+4]]:33: error: integer literal with value 10000000000000000000 does not fit in i32 [IntLiteralTooLargeForI32] +// CHECK:STDERR: fail_uN_bad_width.carbon:[[@LINE+12]]:33: error: expected expression [ExpectedExpr] // CHECK:STDERR: var test_u10000000000000000000: u10000000000000000000; // CHECK:STDERR: ^~~~~~~~~~~~~~~~~~~~~ // CHECK:STDERR: +// CHECK:STDERR: fail_uN_bad_width.carbon:[[@LINE+8]]:33: error: semantics TODO: `HandleInvalidParse` [SemanticsTodo] +// CHECK:STDERR: var test_u10000000000000000000: u10000000000000000000; +// CHECK:STDERR: ^~~~~~~~~~~~~~~~~~~~~ +// CHECK:STDERR: +// CHECK:STDERR: fail_uN_bad_width.carbon:[[@LINE+4]]:34: error: found a type literal with a bit width using 20 digits, which is greater than the limit of 18 [TooManyTypeBitWidthDigits] +// CHECK:STDERR: var test_u10000000000000000000: u10000000000000000000; +// CHECK:STDERR: ^ +// CHECK:STDERR: var test_u10000000000000000000: u10000000000000000000; // --- fail_fN_bad_width.carbon @@ -183,58 +199,7 @@ var test_f128: f128; // CHECK:STDOUT: %.6: i32 = int_literal 1000000000 [template] // CHECK:STDOUT: } // CHECK:STDOUT: -// CHECK:STDOUT: imports { -// CHECK:STDOUT: %Core: = namespace file.%Core.import, [template] { -// CHECK:STDOUT: .Int = %import_ref -// CHECK:STDOUT: import Core//prelude -// CHECK:STDOUT: import Core//prelude/operators -// CHECK:STDOUT: import Core//prelude/types -// CHECK:STDOUT: import Core//prelude/operators/arithmetic -// CHECK:STDOUT: import Core//prelude/operators/as -// CHECK:STDOUT: import Core//prelude/operators/bitwise -// CHECK:STDOUT: import Core//prelude/operators/comparison -// CHECK:STDOUT: import Core//prelude/types/bool -// CHECK:STDOUT: } -// CHECK:STDOUT: %import_ref: %Int.type = import_ref Core//prelude/types, inst+29, loaded [template = constants.%Int] -// CHECK:STDOUT: } -// CHECK:STDOUT: -// CHECK:STDOUT: file { -// CHECK:STDOUT: package: = namespace [template] { -// CHECK:STDOUT: .Core = imports.%Core -// CHECK:STDOUT: .test_i0 = %test_i0 -// CHECK:STDOUT: .test_i1 = %test_i1 -// CHECK:STDOUT: .test_i15 = %test_i15 -// CHECK:STDOUT: .test_i1000000000 = %test_i1000000000 -// CHECK:STDOUT: .test_i10000000000000000000 = %test_i10000000000000000000 -// CHECK:STDOUT: } -// CHECK:STDOUT: %Core.import = import Core -// CHECK:STDOUT: %i0.ref: = name_ref i0, [template = ] -// CHECK:STDOUT: %test_i0.var: ref = var test_i0 -// CHECK:STDOUT: %test_i0: ref = bind_name test_i0, %test_i0.var -// CHECK:STDOUT: %.loc12_14.1: i32 = int_literal 1 [template = constants.%.1] -// CHECK:STDOUT: %int.make_type_signed.loc12: init type = call constants.%Int(%.loc12_14.1) [template = constants.%.3] -// CHECK:STDOUT: %.loc12_14.2: type = value_of_initializer %int.make_type_signed.loc12 [template = constants.%.3] -// CHECK:STDOUT: %.loc12_14.3: type = converted %int.make_type_signed.loc12, %.loc12_14.2 [template = constants.%.3] -// CHECK:STDOUT: %test_i1.var: ref %.3 = var test_i1 -// CHECK:STDOUT: %test_i1: ref %.3 = bind_name test_i1, %test_i1.var -// CHECK:STDOUT: %.loc17_15.1: i32 = int_literal 15 [template = constants.%.4] -// CHECK:STDOUT: %int.make_type_signed.loc17: init type = call constants.%Int(%.loc17_15.1) [template = constants.%.5] -// CHECK:STDOUT: %.loc17_15.2: type = value_of_initializer %int.make_type_signed.loc17 [template = constants.%.5] -// CHECK:STDOUT: %.loc17_15.3: type = converted %int.make_type_signed.loc17, %.loc17_15.2 [template = constants.%.5] -// CHECK:STDOUT: %test_i15.var: ref %.5 = var test_i15 -// CHECK:STDOUT: %test_i15: ref %.5 = bind_name test_i15, %test_i15.var -// CHECK:STDOUT: %.loc22_23.1: i32 = int_literal 1000000000 [template = constants.%.6] -// CHECK:STDOUT: %int.make_type_signed.loc22: init type = call constants.%Int(%.loc22_23.1) [template = ] -// CHECK:STDOUT: %.loc22_23.2: type = value_of_initializer %int.make_type_signed.loc22 [template = ] -// CHECK:STDOUT: %.loc22_23.3: type = converted %int.make_type_signed.loc22, %.loc22_23.2 [template = ] -// CHECK:STDOUT: %test_i1000000000.var: ref = var test_i1000000000 -// CHECK:STDOUT: %test_i1000000000: ref = bind_name test_i1000000000, %test_i1000000000.var -// CHECK:STDOUT: %int.make_type_signed.loc28: init type = call constants.%Int() [template = ] -// CHECK:STDOUT: %.loc28_33.1: type = value_of_initializer %int.make_type_signed.loc28 [template = ] -// CHECK:STDOUT: %.loc28_33.2: type = converted %int.make_type_signed.loc28, %.loc28_33.1 [template = ] -// CHECK:STDOUT: %test_i10000000000000000000.var: ref = var test_i10000000000000000000 -// CHECK:STDOUT: %test_i10000000000000000000: ref = bind_name test_i10000000000000000000, %test_i10000000000000000000.var -// CHECK:STDOUT: } +// CHECK:STDOUT: file {} // CHECK:STDOUT: // CHECK:STDOUT: fn @Int(%size.param_patt: i32) -> type = "int.make_type_signed"; // CHECK:STDOUT: @@ -310,58 +275,7 @@ var test_f128: f128; // CHECK:STDOUT: %.6: i32 = int_literal 1000000000 [template] // CHECK:STDOUT: } // CHECK:STDOUT: -// CHECK:STDOUT: imports { -// CHECK:STDOUT: %Core: = namespace file.%Core.import, [template] { -// CHECK:STDOUT: .UInt = %import_ref -// CHECK:STDOUT: import Core//prelude -// CHECK:STDOUT: import Core//prelude/operators -// CHECK:STDOUT: import Core//prelude/types -// CHECK:STDOUT: import Core//prelude/operators/arithmetic -// CHECK:STDOUT: import Core//prelude/operators/as -// CHECK:STDOUT: import Core//prelude/operators/bitwise -// CHECK:STDOUT: import Core//prelude/operators/comparison -// CHECK:STDOUT: import Core//prelude/types/bool -// CHECK:STDOUT: } -// CHECK:STDOUT: %import_ref: %UInt.type = import_ref Core//prelude/types, inst+43, loaded [template = constants.%UInt] -// CHECK:STDOUT: } -// CHECK:STDOUT: -// CHECK:STDOUT: file { -// CHECK:STDOUT: package: = namespace [template] { -// CHECK:STDOUT: .Core = imports.%Core -// CHECK:STDOUT: .test_u0 = %test_u0 -// CHECK:STDOUT: .test_u1 = %test_u1 -// CHECK:STDOUT: .test_u15 = %test_u15 -// CHECK:STDOUT: .test_u1000000000 = %test_u1000000000 -// CHECK:STDOUT: .test_u10000000000000000000 = %test_u10000000000000000000 -// CHECK:STDOUT: } -// CHECK:STDOUT: %Core.import = import Core -// CHECK:STDOUT: %u0.ref: = name_ref u0, [template = ] -// CHECK:STDOUT: %test_u0.var: ref = var test_u0 -// CHECK:STDOUT: %test_u0: ref = bind_name test_u0, %test_u0.var -// CHECK:STDOUT: %.loc12_14.1: i32 = int_literal 1 [template = constants.%.1] -// CHECK:STDOUT: %int.make_type_unsigned.loc12: init type = call constants.%UInt(%.loc12_14.1) [template = constants.%.3] -// CHECK:STDOUT: %.loc12_14.2: type = value_of_initializer %int.make_type_unsigned.loc12 [template = constants.%.3] -// CHECK:STDOUT: %.loc12_14.3: type = converted %int.make_type_unsigned.loc12, %.loc12_14.2 [template = constants.%.3] -// CHECK:STDOUT: %test_u1.var: ref %.3 = var test_u1 -// CHECK:STDOUT: %test_u1: ref %.3 = bind_name test_u1, %test_u1.var -// CHECK:STDOUT: %.loc17_15.1: i32 = int_literal 15 [template = constants.%.4] -// CHECK:STDOUT: %int.make_type_unsigned.loc17: init type = call constants.%UInt(%.loc17_15.1) [template = constants.%.5] -// CHECK:STDOUT: %.loc17_15.2: type = value_of_initializer %int.make_type_unsigned.loc17 [template = constants.%.5] -// CHECK:STDOUT: %.loc17_15.3: type = converted %int.make_type_unsigned.loc17, %.loc17_15.2 [template = constants.%.5] -// CHECK:STDOUT: %test_u15.var: ref %.5 = var test_u15 -// CHECK:STDOUT: %test_u15: ref %.5 = bind_name test_u15, %test_u15.var -// CHECK:STDOUT: %.loc22_23.1: i32 = int_literal 1000000000 [template = constants.%.6] -// CHECK:STDOUT: %int.make_type_unsigned.loc22: init type = call constants.%UInt(%.loc22_23.1) [template = ] -// CHECK:STDOUT: %.loc22_23.2: type = value_of_initializer %int.make_type_unsigned.loc22 [template = ] -// CHECK:STDOUT: %.loc22_23.3: type = converted %int.make_type_unsigned.loc22, %.loc22_23.2 [template = ] -// CHECK:STDOUT: %test_u1000000000.var: ref = var test_u1000000000 -// CHECK:STDOUT: %test_u1000000000: ref = bind_name test_u1000000000, %test_u1000000000.var -// CHECK:STDOUT: %int.make_type_unsigned.loc28: init type = call constants.%UInt() [template = ] -// CHECK:STDOUT: %.loc28_33.1: type = value_of_initializer %int.make_type_unsigned.loc28 [template = ] -// CHECK:STDOUT: %.loc28_33.2: type = converted %int.make_type_unsigned.loc28, %.loc28_33.1 [template = ] -// CHECK:STDOUT: %test_u10000000000000000000.var: ref = var test_u10000000000000000000 -// CHECK:STDOUT: %test_u10000000000000000000: ref = bind_name test_u10000000000000000000, %test_u10000000000000000000.var -// CHECK:STDOUT: } +// CHECK:STDOUT: file {} // CHECK:STDOUT: // CHECK:STDOUT: fn @UInt(%size.param_patt: i32) -> type = "int.make_type_unsigned"; // CHECK:STDOUT: diff --git a/toolchain/diagnostics/diagnostic_kind.def b/toolchain/diagnostics/diagnostic_kind.def index 115399115dfa..d621d3040f1f 100644 --- a/toolchain/diagnostics/diagnostic_kind.def +++ b/toolchain/diagnostics/diagnostic_kind.def @@ -42,6 +42,7 @@ CARBON_DIAGNOSTIC_KIND(MultiLineStringWithDoubleQuotes) CARBON_DIAGNOSTIC_KIND(NoWhitespaceAfterCommentIntroducer) CARBON_DIAGNOSTIC_KIND(TooManyDigits) CARBON_DIAGNOSTIC_KIND(TooManyTokens) +CARBON_DIAGNOSTIC_KIND(TooManyTypeBitWidthDigits) CARBON_DIAGNOSTIC_KIND(TrailingComment) CARBON_DIAGNOSTIC_KIND(UnicodeEscapeMissingBracedDigits) CARBON_DIAGNOSTIC_KIND(UnicodeEscapeSurrogate) diff --git a/toolchain/lex/lex.cpp b/toolchain/lex/lex.cpp index db3074194464..728fa25bba27 100644 --- a/toolchain/lex/lex.cpp +++ b/toolchain/lex/lex.cpp @@ -1166,10 +1166,6 @@ auto Lexer::LexWordAsTypeLiteralToken(llvm::StringRef word, int32_t byte_offset) // Too short to form one of these tokens. return LexResult::NoMatch(); } - if (word[1] < '1' || word[1] > '9') { - // Doesn't start with a valid initial digit. - return LexResult::NoMatch(); - } TokenKind kind; switch (word.front()) { @@ -1186,17 +1182,49 @@ auto Lexer::LexWordAsTypeLiteralToken(llvm::StringRef word, int32_t byte_offset) return LexResult::NoMatch(); }; - llvm::StringRef suffix = word.substr(1); - if (!CanLexInt(emitter_, suffix)) { - return LexTokenWithPayload(TokenKind::Error, word.size(), byte_offset); - } - llvm::APInt suffix_value; - if (suffix.getAsInteger(10, suffix_value)) { + // No leading zeros allowed. + if ('1' > word[1] || word[1] > '9') { return LexResult::NoMatch(); } + llvm::StringRef suffix = word.substr(1); + + // Type bit-widths can't usefully be large integers so we restrict to small + // ones that are especially easy to parse into a normal integer variable by + // restricting the number of digits to round trip. + int64_t suffix_value; + constexpr ssize_t DigitLimit = + std::numeric_limits::digits10; + if (suffix.size() > DigitLimit) { + // See if this is not actually a type literal. + if (!llvm::all_of(suffix, IsDecimalDigit)) { + return LexResult::NoMatch(); + } + + // Otherwise, diagnose and produce an error token. + CARBON_DIAGNOSTIC(TooManyTypeBitWidthDigits, Error, + "found a type literal with a bit width using {0} digits, " + "which is greater than the limit of {1}", + size_t, size_t); + emitter_.Emit(word.begin() + 1, TooManyTypeBitWidthDigits, suffix.size(), + DigitLimit); + return LexTokenWithPayload(TokenKind::Error, word.size(), byte_offset); + } + + // It's tempting to do something more clever because we know the length ahead + // of time, but we expect these to be short (1-3 digits) and profiling doesn't + // show the loop as hot in the short cases. + suffix_value = suffix[0] - '0'; + for (char c : suffix.drop_front()) { + if (!IsDecimalDigit(c)) { + return LexResult::NoMatch(); + } + suffix_value = suffix_value * 10 + (c - '0'); + } + return LexTokenWithPayload( - kind, buffer_.value_stores_->ints().Add(std::move(suffix_value)).index, + kind, + buffer_.value_stores_->ints().Add(llvm::APInt(64, suffix_value)).index, byte_offset); } diff --git a/toolchain/lex/tokenized_buffer_test.cpp b/toolchain/lex/tokenized_buffer_test.cpp index b756447cb9db..f8d2b74ce0bb 100644 --- a/toolchain/lex/tokenized_buffer_test.cpp +++ b/toolchain/lex/tokenized_buffer_test.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -1023,27 +1024,54 @@ TEST_F(LexerTest, TypeLiterals) { } TEST_F(LexerTest, TypeLiteralTooManyDigits) { - std::string code = "i"; - constexpr int Count = 10000; - code.append(Count, '9'); - + // We increase the number of digits until the first one that is to large. Testing::MockDiagnosticConsumer consumer; - EXPECT_CALL(consumer, - HandleDiagnostic(IsSingleDiagnostic( - DiagnosticKind::TooManyDigits, DiagnosticLevel::Error, 1, 2, - HasSubstr(llvm::formatv(" {0} ", Count))))); + EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic( + DiagnosticKind::TooManyTypeBitWidthDigits, + DiagnosticLevel::Error, 1, 2, _))); + std::string code = "i"; + // A 128-bit APInt should be plenty large, but if needed in the future it can + // be widened without issue. + llvm::APInt bits = llvm::APInt::getZero(128); + for ([[maybe_unused]] int _ : llvm::seq(1, 30)) { + code.append("9"); + bits = bits * 10 + 9; + auto [buffer, value_stores] = + compile_helper_.GetTokenizedBufferWithSharedValueStore(code, &consumer); + if (buffer.has_errors()) { + ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef{ + {.kind = TokenKind::FileStart}, + {.kind = TokenKind::Error, .text = code}, + {.kind = TokenKind::FileEnd}, + })); + break; + } + ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef{ + {.kind = TokenKind::FileStart}, + {.kind = TokenKind::IntTypeLiteral, .text = code}, + {.kind = TokenKind::FileEnd}, + })); + auto token = buffer.tokens().begin()[1]; + EXPECT_TRUE(llvm::APInt::isSameValue( + value_stores.ints().Get(buffer.GetTypeLiteralSize(token)), bits)); + } + + // Make sure we can also gracefully reject very large number of digits without + // crashing or hanging, and show the correct number. + constexpr int Count = 10000; + EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic( + DiagnosticKind::TooManyTypeBitWidthDigits, + DiagnosticLevel::Error, 1, 2, + HasSubstr(llvm::formatv(" {0} ", Count))))); + code = "i"; + code.append(Count, '9'); auto& buffer = compile_helper_.GetTokenizedBuffer(code, &consumer); - EXPECT_TRUE(buffer.has_errors()); - ASSERT_THAT(buffer, - HasTokens(llvm::ArrayRef{ - {.kind = TokenKind::FileStart, .line = 1, .column = 1}, - {.kind = TokenKind::Error, - .line = 1, - .column = 1, - .indent_column = 1, - .text = code}, - {.kind = TokenKind::FileEnd, .line = 1, .column = Count + 2}, - })); + ASSERT_TRUE(buffer.has_errors()); + ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef{ + {.kind = TokenKind::FileStart}, + {.kind = TokenKind::Error, .text = code}, + {.kind = TokenKind::FileEnd}, + })); } TEST_F(LexerTest, DiagnosticTrailingComment) { diff --git a/toolchain/testing/compile_helper.cpp b/toolchain/testing/compile_helper.cpp index 2b79b946af21..80baa02829d4 100644 --- a/toolchain/testing/compile_helper.cpp +++ b/toolchain/testing/compile_helper.cpp @@ -17,9 +17,10 @@ auto CompileHelper::GetTokenizedBuffer(llvm::StringRef text, return token_storage_.front(); } -auto CompileHelper::GetTokenizedBufferWithSharedValueStore(llvm::StringRef text) +auto CompileHelper::GetTokenizedBufferWithSharedValueStore( + llvm::StringRef text, DiagnosticConsumer* consumer) -> std::pair { - auto& tokens = GetTokenizedBuffer(text); + auto& tokens = GetTokenizedBuffer(text, consumer); return {tokens, value_store_storage_.front()}; } diff --git a/toolchain/testing/compile_helper.h b/toolchain/testing/compile_helper.h index 428703252b76..3b6bc8bff4f3 100644 --- a/toolchain/testing/compile_helper.h +++ b/toolchain/testing/compile_helper.h @@ -25,7 +25,8 @@ class CompileHelper { -> Lex::TokenizedBuffer&; // Returns the result of lex along with shared values. - auto GetTokenizedBufferWithSharedValueStore(llvm::StringRef text) + auto GetTokenizedBufferWithSharedValueStore( + llvm::StringRef text, DiagnosticConsumer* consumer = nullptr) -> std::pair; // Returns the result of parse.