diff --git a/docs/mkdocs/docs/api/basic_json/number_float_t.md b/docs/mkdocs/docs/api/basic_json/number_float_t.md index 83c7011c5..f6508a6c3 100644 --- a/docs/mkdocs/docs/api/basic_json/number_float_t.md +++ b/docs/mkdocs/docs/api/basic_json/number_float_t.md @@ -55,6 +55,10 @@ This implementation does exactly follow this approach, as it uses double precisi smaller than `-1.79769313486232e+308` and values greater than `1.79769313486232e+308` will be stored as NaN internally and be serialized to `null`. +During deserialization (from JSON text or any of the binary formats), a finite number that does not fit into +`number_float_t` is rejected with [`out_of_range.406`](../../home/exceptions.md#jsonexceptionout_of_range406), for +example a double-precision number in a binary format when `number_float_t` is `#!cpp float`. + #### Storage Floating-point number values are stored directly inside a `basic_json` type. diff --git a/docs/mkdocs/docs/api/basic_json/number_integer_t.md b/docs/mkdocs/docs/api/basic_json/number_integer_t.md index 9a2ffab7f..00f530d57 100644 --- a/docs/mkdocs/docs/api/basic_json/number_integer_t.md +++ b/docs/mkdocs/docs/api/basic_json/number_integer_t.md @@ -47,8 +47,9 @@ With the default values for `NumberIntegerType` (`std::int64_t`), the default va When the default type is used, the maximal integer number that can be stored is `9223372036854775807` (INT64_MAX) and the minimal integer number that can be stored is `-9223372036854775808` (INT64_MIN). Integer numbers that are out of -range will yield over/underflow when used in a constructor. During deserialization, too large or small integer numbers -will automatically be stored as [`number_unsigned_t`](number_unsigned_t.md) or [`number_float_t`](number_float_t.md). +range will yield over/underflow when used in a constructor. During deserialization (from JSON text or any of the binary +formats), too large or small integer numbers will automatically be stored as [`number_unsigned_t`](number_unsigned_t.md) +or [`number_float_t`](number_float_t.md). [RFC 8259](https://tools.ietf.org/html/rfc8259) further states: > Note that when such software is used, numbers that are integers and are in the range $[-2^{53}+1, 2^{53}-1]$ are diff --git a/docs/mkdocs/docs/api/basic_json/number_unsigned_t.md b/docs/mkdocs/docs/api/basic_json/number_unsigned_t.md index 674f7711d..50ad2940b 100644 --- a/docs/mkdocs/docs/api/basic_json/number_unsigned_t.md +++ b/docs/mkdocs/docs/api/basic_json/number_unsigned_t.md @@ -48,8 +48,9 @@ With the default values for `NumberUnsignedType` (`std::uint64_t`), the default When the default type is used, the maximal integer number that can be stored is `18446744073709551615` (UINT64_MAX) and the minimal integer number that can be stored is `0`. Integer numbers that are out of range will yield over/underflow -when used in a constructor. During deserialization, too large or small integer numbers will automatically be stored -as [`number_integer_t`](number_integer_t.md) or [`number_float_t`](number_float_t.md). +when used in a constructor. During deserialization (from JSON text or any of the binary formats), too large or small +integer numbers will automatically be stored as [`number_integer_t`](number_integer_t.md) or +[`number_float_t`](number_float_t.md). [RFC 8259](https://tools.ietf.org/html/rfc8259) further states: > Note that when such software is used, numbers that are integers and are in the range $[-2^{53}+1, 2^{53}-1]$ are diff --git a/docs/mkdocs/docs/features/binary_formats/cbor.md b/docs/mkdocs/docs/features/binary_formats/cbor.md index 8e6acf0fb..15d480ee1 100644 --- a/docs/mkdocs/docs/features/binary_formats/cbor.md +++ b/docs/mkdocs/docs/features/binary_formats/cbor.md @@ -168,9 +168,9 @@ The library maps CBOR types to JSON value types as follows: !!! warning "Negative integer overflow" CBOR negative integers (major type 1) are decoded as `-1 - n`. If the encoded magnitude `n` is too large for the - result to fit into `number_integer_t` (`std::int64_t` by default), parsing fails with a - [`parse_error.112`](../../home/exceptions.md#jsonexceptionparse_error112) exception rather than overflowing - silently. + result to fit into `number_integer_t` (`std::int64_t` by default), the result is stored as `number_float_t`, like + a too small integer in JSON text. For example, `-18446744073709551616` (`0x3B` followed by eight `0xFF` bytes) is + stored as `-1.8446744073709552e+19`. !!! warning "Object keys" diff --git a/docs/mkdocs/docs/home/exceptions.md b/docs/mkdocs/docs/home/exceptions.md index bf18baab1..9c1c4ba89 100644 --- a/docs/mkdocs/docs/home/exceptions.md +++ b/docs/mkdocs/docs/home/exceptions.md @@ -331,9 +331,6 @@ An unexpected byte was read in a [binary format](../features/binary_formats/inde [json.exception.parse_error.112] parse error at byte 15: syntax error while parsing BSON binary: byte array length cannot be negative, is -1 ``` ``` - [json.exception.parse_error.112] parse error at byte 9: syntax error while parsing CBOR value: negative integer overflow - ``` - ``` [json.exception.parse_error.112] parse error at byte 5: syntax error while parsing BSON document: document size 6 does not match the number of bytes read (5) ``` @@ -847,13 +844,18 @@ The JSON Patch operations 'remove' and 'add' cannot be applied to the root eleme ### json.exception.out_of_range.406 -A parsed number could not be stored as without changing it to NaN or INF. +A parsed number could not be stored without changing it to NaN or INF. For the binary formats, this happens when a +finite floating-point number does not fit into [`number_float_t`](../api/basic_json/number_float_t.md), for example a +double-precision number when `number_float_t` is `#!cpp float`. -!!! failure "Example message" +!!! failure "Example messages" ``` number overflow parsing '10E1000' ``` + ``` + [json.exception.out_of_range.406] syntax error while parsing CBOR value: number overflow + ``` ### json.exception.out_of_range.407 diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index b0675c626..e3360aa97 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -559,7 +559,7 @@ class binary_reader case 0x01: // double { double number{}; - return get_number(input_format_t::bson, number) && sax->number_float(static_cast(number), ""); + return get_number(input_format_t::bson, number) && emit_float(input_format_t::bson, number); } case 0x02: // string @@ -600,19 +600,19 @@ class binary_reader case 0x10: // int32 { std::int32_t value{}; - return get_number(input_format_t::bson, value) && sax->number_integer(value); + return get_number(input_format_t::bson, value) && emit_signed(value); } case 0x12: // int64 { std::int64_t value{}; - return get_number(input_format_t::bson, value) && sax->number_integer(value); + return get_number(input_format_t::bson, value) && emit_signed(value); } case 0x11: // uint64 { std::uint64_t value{}; - return get_number(input_format_t::bson, value) && sax->number_unsigned(value); + return get_number(input_format_t::bson, value) && emit_unsigned(value); } default: // anything else is not supported (yet) @@ -638,14 +638,17 @@ class binary_reader { return false; } - const auto max_val = static_cast((std::numeric_limits::max)()); - if (number > max_val) + + // the value is -1 - number, which fits into number_integer_t + // whenever number does + if (JSON_HEDLEY_LIKELY(value_in_range_of(number))) { - return sax->parse_error(chars_read, get_token_string(), - parse_error::create(112, chars_read, - exception_message(input_format_t::cbor, "negative integer overflow", "value"), nullptr)); + return sax->number_integer(static_cast(-1) - static_cast(number)); } - return sax->number_integer(static_cast(-1) - static_cast(number)); + + // like the lexer does for JSON text, store a value too small for + // number_integer_t as number_float_t + return sax->number_float(static_cast(-1) - static_cast(number), ""); } /*! @@ -702,25 +705,25 @@ class binary_reader case 0x18: // Unsigned integer (one-byte uint8_t follows) { std::uint8_t number{}; - return get_number(input_format_t::cbor, number) && sax->number_unsigned(number); + return get_number(input_format_t::cbor, number) && emit_unsigned(number); } case 0x19: // Unsigned integer (two-byte uint16_t follows) { std::uint16_t number{}; - return get_number(input_format_t::cbor, number) && sax->number_unsigned(number); + return get_number(input_format_t::cbor, number) && emit_unsigned(number); } case 0x1A: // Unsigned integer (four-byte uint32_t follows) { std::uint32_t number{}; - return get_number(input_format_t::cbor, number) && sax->number_unsigned(number); + return get_number(input_format_t::cbor, number) && emit_unsigned(number); } case 0x1B: // Unsigned integer (eight-byte uint64_t follows) { std::uint64_t number{}; - return get_number(input_format_t::cbor, number) && sax->number_unsigned(number); + return get_number(input_format_t::cbor, number) && emit_unsigned(number); } // Negative integer -1-0x00..-1-0x17 (-1..-24) @@ -1165,13 +1168,13 @@ class binary_reader case 0xFA: // Single-Precision Float (four-byte IEEE 754) { float number{}; - return get_number(input_format_t::cbor, number) && sax->number_float(static_cast(number), ""); + return get_number(input_format_t::cbor, number) && emit_float(input_format_t::cbor, number); } case 0xFB: // Double-Precision Float (eight-byte IEEE 754) { double number{}; - return get_number(input_format_t::cbor, number) && sax->number_float(static_cast(number), ""); + return get_number(input_format_t::cbor, number) && emit_float(input_format_t::cbor, number); } default: // anything else (0xFF is handled inside the other types) @@ -1861,61 +1864,61 @@ class binary_reader case 0xCA: // float 32 { float number{}; - return get_number(input_format_t::msgpack, number) && sax->number_float(static_cast(number), ""); + return get_number(input_format_t::msgpack, number) && emit_float(input_format_t::msgpack, number); } case 0xCB: // float 64 { double number{}; - return get_number(input_format_t::msgpack, number) && sax->number_float(static_cast(number), ""); + return get_number(input_format_t::msgpack, number) && emit_float(input_format_t::msgpack, number); } case 0xCC: // uint 8 { std::uint8_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number); + return get_number(input_format_t::msgpack, number) && emit_unsigned(number); } case 0xCD: // uint 16 { std::uint16_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number); + return get_number(input_format_t::msgpack, number) && emit_unsigned(number); } case 0xCE: // uint 32 { std::uint32_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number); + return get_number(input_format_t::msgpack, number) && emit_unsigned(number); } case 0xCF: // uint 64 { std::uint64_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number); + return get_number(input_format_t::msgpack, number) && emit_unsigned(number); } case 0xD0: // int 8 { std::int8_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_integer(number); + return get_number(input_format_t::msgpack, number) && emit_signed(number); } case 0xD1: // int 16 { std::int16_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_integer(number); + return get_number(input_format_t::msgpack, number) && emit_signed(number); } case 0xD2: // int 32 { std::int32_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_integer(number); + return get_number(input_format_t::msgpack, number) && emit_signed(number); } case 0xD3: // int 64 { std::int64_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_integer(number); + return get_number(input_format_t::msgpack, number) && emit_signed(number); } case 0xDC: // array 16 @@ -2756,7 +2759,7 @@ class binary_reader { return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, exception_message(input_format, "excessive ndarray size caused overflow", "size"), nullptr)); } - if (JSON_HEDLEY_UNLIKELY(!sax->number_unsigned(static_cast(i)))) + if (JSON_HEDLEY_UNLIKELY(!emit_unsigned(i))) { return false; } @@ -2888,37 +2891,37 @@ class binary_reader break; } std::uint8_t number{}; - return get_number(input_format, number) && sax->number_unsigned(number); + return get_number(input_format, number) && emit_unsigned(number); } case 'U': { std::uint8_t number{}; - return get_number(input_format, number) && sax->number_unsigned(number); + return get_number(input_format, number) && emit_unsigned(number); } case 'i': { std::int8_t number{}; - return get_number(input_format, number) && sax->number_integer(number); + return get_number(input_format, number) && emit_signed(number); } case 'I': { std::int16_t number{}; - return get_number(input_format, number) && sax->number_integer(number); + return get_number(input_format, number) && emit_signed(number); } case 'l': { std::int32_t number{}; - return get_number(input_format, number) && sax->number_integer(number); + return get_number(input_format, number) && emit_signed(number); } case 'L': { std::int64_t number{}; - return get_number(input_format, number) && sax->number_integer(number); + return get_number(input_format, number) && emit_signed(number); } case 'u': @@ -2928,7 +2931,7 @@ class binary_reader break; } std::uint16_t number{}; - return get_number(input_format, number) && sax->number_unsigned(number); + return get_number(input_format, number) && emit_unsigned(number); } case 'm': @@ -2938,7 +2941,7 @@ class binary_reader break; } std::uint32_t number{}; - return get_number(input_format, number) && sax->number_unsigned(number); + return get_number(input_format, number) && emit_unsigned(number); } case 'M': @@ -2948,7 +2951,7 @@ class binary_reader break; } std::uint64_t number{}; - return get_number(input_format, number) && sax->number_unsigned(number); + return get_number(input_format, number) && emit_unsigned(number); } case 'h': @@ -3006,13 +3009,13 @@ class binary_reader case 'd': { float number{}; - return get_number(input_format, number) && sax->number_float(static_cast(number), ""); + return get_number(input_format, number) && emit_float(input_format, number); } case 'D': { double number{}; - return get_number(input_format, number) && sax->number_float(static_cast(number), ""); + return get_number(input_format, number) && emit_float(input_format, number); } case 'H': @@ -3479,13 +3482,13 @@ class binary_reader case 0x8E: // binary32 { float number{}; - return get_number(input_format_t::bon8, number) && sax->number_float(static_cast(number), ""); + return get_number(input_format_t::bon8, number) && emit_float(input_format_t::bon8, number); } case 0x8F: // binary64 { double number{}; - return get_number(input_format_t::bon8, number) && sax->number_float(static_cast(number), ""); + return get_number(input_format_t::bon8, number) && emit_float(input_format_t::bon8, number); } case 0xF8: @@ -3551,7 +3554,9 @@ class binary_reader @brief pass an integer to the SAX parser Non-negative integers are passed as unsigned, negative integers as signed - numbers, like the other binary formats do. + numbers, like the other binary formats do. A value that does not fit the + number type is passed as described for @ref emit_unsigned and + @ref emit_signed. @param[in] number the integer @return whether the SAX parser accepted the value @@ -3560,9 +3565,9 @@ class binary_reader { if (number >= 0) { - return sax->number_unsigned(static_cast(number)); + return emit_unsigned(static_cast(number)); } - return sax->number_integer(static_cast(number)); + return emit_signed(number); } /*! @@ -3619,8 +3624,7 @@ class binary_reader value = (value << 8) | static_cast(current); } - return negative ? sax->number_integer(static_cast(-(value + offset))) - : sax->number_unsigned(static_cast(value + offset)); + return emit_bon8_integer(negative ? -(value + offset) : value + offset); } /*! @@ -3917,6 +3921,79 @@ class binary_reader return true; } + /*! + @brief pass a signed integer read from the input to the SAX parser + + Like the lexer does for JSON text, a value that does not fit into + number_integer_t is passed as number_unsigned_t if it is non-negative and + fits there, and as number_float_t otherwise. With the default number + types, every integer the binary formats can encode fits, so this only + matters for narrower custom number types. + + @tparam NumberType a signed integer type + @param[in] number the integer + @return whether the SAX parser accepted the value + */ + template + bool emit_signed(const NumberType number) + { + if (JSON_HEDLEY_LIKELY(value_in_range_of(number))) + { + return sax->number_integer(static_cast(number)); + } + if (value_in_range_of(number)) + { + return sax->number_unsigned(static_cast(number)); + } + return sax->number_float(static_cast(number), ""); + } + + /*! + @brief pass an unsigned integer read from the input to the SAX parser + + Like the lexer does for JSON text, a value that does not fit into + number_unsigned_t is passed as number_float_t. + + @tparam NumberType an unsigned integer type + @param[in] number the integer + @return whether the SAX parser accepted the value + */ + template + bool emit_unsigned(const NumberType number) + { + if (JSON_HEDLEY_LIKELY(value_in_range_of(number))) + { + return sax->number_unsigned(static_cast(number)); + } + return sax->number_float(static_cast(number), ""); + } + + /*! + @brief pass a floating-point number read from the input to the SAX parser + + Like the lexer does for JSON text, a finite value that overflows + number_float_t is rejected instead of silently becoming infinity. Infinity + and NaN in the input are passed on unchanged. + + @tparam NumberType a floating-point type + @param[in] format the current format (for diagnostics) + @param[in] number the number + @return whether the SAX parser accepted the value + + @throw out_of_range.406 if a finite @a number overflows number_float_t + */ + template + bool emit_float(const input_format_t format, const NumberType number) + { + const auto result = static_cast(number); + if (JSON_HEDLEY_UNLIKELY(std::isfinite(number) && !std::isfinite(result))) + { + return sax->parse_error(chars_read, get_token_string(), + out_of_range::create(406, exception_message(format, "number overflow", "value"), nullptr)); + } + return sax->number_float(result, ""); + } + /*! @brief create a string by reading characters from the input diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 0e3cae486..152e6c511 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -13294,7 +13294,7 @@ class binary_reader case 0x01: // double { double number{}; - return get_number(input_format_t::bson, number) && sax->number_float(static_cast(number), ""); + return get_number(input_format_t::bson, number) && emit_float(input_format_t::bson, number); } case 0x02: // string @@ -13335,19 +13335,19 @@ class binary_reader case 0x10: // int32 { std::int32_t value{}; - return get_number(input_format_t::bson, value) && sax->number_integer(value); + return get_number(input_format_t::bson, value) && emit_signed(value); } case 0x12: // int64 { std::int64_t value{}; - return get_number(input_format_t::bson, value) && sax->number_integer(value); + return get_number(input_format_t::bson, value) && emit_signed(value); } case 0x11: // uint64 { std::uint64_t value{}; - return get_number(input_format_t::bson, value) && sax->number_unsigned(value); + return get_number(input_format_t::bson, value) && emit_unsigned(value); } default: // anything else is not supported (yet) @@ -13373,14 +13373,17 @@ class binary_reader { return false; } - const auto max_val = static_cast((std::numeric_limits::max)()); - if (number > max_val) + + // the value is -1 - number, which fits into number_integer_t + // whenever number does + if (JSON_HEDLEY_LIKELY(value_in_range_of(number))) { - return sax->parse_error(chars_read, get_token_string(), - parse_error::create(112, chars_read, - exception_message(input_format_t::cbor, "negative integer overflow", "value"), nullptr)); + return sax->number_integer(static_cast(-1) - static_cast(number)); } - return sax->number_integer(static_cast(-1) - static_cast(number)); + + // like the lexer does for JSON text, store a value too small for + // number_integer_t as number_float_t + return sax->number_float(static_cast(-1) - static_cast(number), ""); } /*! @@ -13437,25 +13440,25 @@ class binary_reader case 0x18: // Unsigned integer (one-byte uint8_t follows) { std::uint8_t number{}; - return get_number(input_format_t::cbor, number) && sax->number_unsigned(number); + return get_number(input_format_t::cbor, number) && emit_unsigned(number); } case 0x19: // Unsigned integer (two-byte uint16_t follows) { std::uint16_t number{}; - return get_number(input_format_t::cbor, number) && sax->number_unsigned(number); + return get_number(input_format_t::cbor, number) && emit_unsigned(number); } case 0x1A: // Unsigned integer (four-byte uint32_t follows) { std::uint32_t number{}; - return get_number(input_format_t::cbor, number) && sax->number_unsigned(number); + return get_number(input_format_t::cbor, number) && emit_unsigned(number); } case 0x1B: // Unsigned integer (eight-byte uint64_t follows) { std::uint64_t number{}; - return get_number(input_format_t::cbor, number) && sax->number_unsigned(number); + return get_number(input_format_t::cbor, number) && emit_unsigned(number); } // Negative integer -1-0x00..-1-0x17 (-1..-24) @@ -13900,13 +13903,13 @@ class binary_reader case 0xFA: // Single-Precision Float (four-byte IEEE 754) { float number{}; - return get_number(input_format_t::cbor, number) && sax->number_float(static_cast(number), ""); + return get_number(input_format_t::cbor, number) && emit_float(input_format_t::cbor, number); } case 0xFB: // Double-Precision Float (eight-byte IEEE 754) { double number{}; - return get_number(input_format_t::cbor, number) && sax->number_float(static_cast(number), ""); + return get_number(input_format_t::cbor, number) && emit_float(input_format_t::cbor, number); } default: // anything else (0xFF is handled inside the other types) @@ -14596,61 +14599,61 @@ class binary_reader case 0xCA: // float 32 { float number{}; - return get_number(input_format_t::msgpack, number) && sax->number_float(static_cast(number), ""); + return get_number(input_format_t::msgpack, number) && emit_float(input_format_t::msgpack, number); } case 0xCB: // float 64 { double number{}; - return get_number(input_format_t::msgpack, number) && sax->number_float(static_cast(number), ""); + return get_number(input_format_t::msgpack, number) && emit_float(input_format_t::msgpack, number); } case 0xCC: // uint 8 { std::uint8_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number); + return get_number(input_format_t::msgpack, number) && emit_unsigned(number); } case 0xCD: // uint 16 { std::uint16_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number); + return get_number(input_format_t::msgpack, number) && emit_unsigned(number); } case 0xCE: // uint 32 { std::uint32_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number); + return get_number(input_format_t::msgpack, number) && emit_unsigned(number); } case 0xCF: // uint 64 { std::uint64_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number); + return get_number(input_format_t::msgpack, number) && emit_unsigned(number); } case 0xD0: // int 8 { std::int8_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_integer(number); + return get_number(input_format_t::msgpack, number) && emit_signed(number); } case 0xD1: // int 16 { std::int16_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_integer(number); + return get_number(input_format_t::msgpack, number) && emit_signed(number); } case 0xD2: // int 32 { std::int32_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_integer(number); + return get_number(input_format_t::msgpack, number) && emit_signed(number); } case 0xD3: // int 64 { std::int64_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_integer(number); + return get_number(input_format_t::msgpack, number) && emit_signed(number); } case 0xDC: // array 16 @@ -15491,7 +15494,7 @@ class binary_reader { return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, exception_message(input_format, "excessive ndarray size caused overflow", "size"), nullptr)); } - if (JSON_HEDLEY_UNLIKELY(!sax->number_unsigned(static_cast(i)))) + if (JSON_HEDLEY_UNLIKELY(!emit_unsigned(i))) { return false; } @@ -15623,37 +15626,37 @@ class binary_reader break; } std::uint8_t number{}; - return get_number(input_format, number) && sax->number_unsigned(number); + return get_number(input_format, number) && emit_unsigned(number); } case 'U': { std::uint8_t number{}; - return get_number(input_format, number) && sax->number_unsigned(number); + return get_number(input_format, number) && emit_unsigned(number); } case 'i': { std::int8_t number{}; - return get_number(input_format, number) && sax->number_integer(number); + return get_number(input_format, number) && emit_signed(number); } case 'I': { std::int16_t number{}; - return get_number(input_format, number) && sax->number_integer(number); + return get_number(input_format, number) && emit_signed(number); } case 'l': { std::int32_t number{}; - return get_number(input_format, number) && sax->number_integer(number); + return get_number(input_format, number) && emit_signed(number); } case 'L': { std::int64_t number{}; - return get_number(input_format, number) && sax->number_integer(number); + return get_number(input_format, number) && emit_signed(number); } case 'u': @@ -15663,7 +15666,7 @@ class binary_reader break; } std::uint16_t number{}; - return get_number(input_format, number) && sax->number_unsigned(number); + return get_number(input_format, number) && emit_unsigned(number); } case 'm': @@ -15673,7 +15676,7 @@ class binary_reader break; } std::uint32_t number{}; - return get_number(input_format, number) && sax->number_unsigned(number); + return get_number(input_format, number) && emit_unsigned(number); } case 'M': @@ -15683,7 +15686,7 @@ class binary_reader break; } std::uint64_t number{}; - return get_number(input_format, number) && sax->number_unsigned(number); + return get_number(input_format, number) && emit_unsigned(number); } case 'h': @@ -15741,13 +15744,13 @@ class binary_reader case 'd': { float number{}; - return get_number(input_format, number) && sax->number_float(static_cast(number), ""); + return get_number(input_format, number) && emit_float(input_format, number); } case 'D': { double number{}; - return get_number(input_format, number) && sax->number_float(static_cast(number), ""); + return get_number(input_format, number) && emit_float(input_format, number); } case 'H': @@ -16214,13 +16217,13 @@ class binary_reader case 0x8E: // binary32 { float number{}; - return get_number(input_format_t::bon8, number) && sax->number_float(static_cast(number), ""); + return get_number(input_format_t::bon8, number) && emit_float(input_format_t::bon8, number); } case 0x8F: // binary64 { double number{}; - return get_number(input_format_t::bon8, number) && sax->number_float(static_cast(number), ""); + return get_number(input_format_t::bon8, number) && emit_float(input_format_t::bon8, number); } case 0xF8: @@ -16286,7 +16289,9 @@ class binary_reader @brief pass an integer to the SAX parser Non-negative integers are passed as unsigned, negative integers as signed - numbers, like the other binary formats do. + numbers, like the other binary formats do. A value that does not fit the + number type is passed as described for @ref emit_unsigned and + @ref emit_signed. @param[in] number the integer @return whether the SAX parser accepted the value @@ -16295,9 +16300,9 @@ class binary_reader { if (number >= 0) { - return sax->number_unsigned(static_cast(number)); + return emit_unsigned(static_cast(number)); } - return sax->number_integer(static_cast(number)); + return emit_signed(number); } /*! @@ -16354,8 +16359,7 @@ class binary_reader value = (value << 8) | static_cast(current); } - return negative ? sax->number_integer(static_cast(-(value + offset))) - : sax->number_unsigned(static_cast(value + offset)); + return emit_bon8_integer(negative ? -(value + offset) : value + offset); } /*! @@ -16652,6 +16656,79 @@ class binary_reader return true; } + /*! + @brief pass a signed integer read from the input to the SAX parser + + Like the lexer does for JSON text, a value that does not fit into + number_integer_t is passed as number_unsigned_t if it is non-negative and + fits there, and as number_float_t otherwise. With the default number + types, every integer the binary formats can encode fits, so this only + matters for narrower custom number types. + + @tparam NumberType a signed integer type + @param[in] number the integer + @return whether the SAX parser accepted the value + */ + template + bool emit_signed(const NumberType number) + { + if (JSON_HEDLEY_LIKELY(value_in_range_of(number))) + { + return sax->number_integer(static_cast(number)); + } + if (value_in_range_of(number)) + { + return sax->number_unsigned(static_cast(number)); + } + return sax->number_float(static_cast(number), ""); + } + + /*! + @brief pass an unsigned integer read from the input to the SAX parser + + Like the lexer does for JSON text, a value that does not fit into + number_unsigned_t is passed as number_float_t. + + @tparam NumberType an unsigned integer type + @param[in] number the integer + @return whether the SAX parser accepted the value + */ + template + bool emit_unsigned(const NumberType number) + { + if (JSON_HEDLEY_LIKELY(value_in_range_of(number))) + { + return sax->number_unsigned(static_cast(number)); + } + return sax->number_float(static_cast(number), ""); + } + + /*! + @brief pass a floating-point number read from the input to the SAX parser + + Like the lexer does for JSON text, a finite value that overflows + number_float_t is rejected instead of silently becoming infinity. Infinity + and NaN in the input are passed on unchanged. + + @tparam NumberType a floating-point type + @param[in] format the current format (for diagnostics) + @param[in] number the number + @return whether the SAX parser accepted the value + + @throw out_of_range.406 if a finite @a number overflows number_float_t + */ + template + bool emit_float(const input_format_t format, const NumberType number) + { + const auto result = static_cast(number); + if (JSON_HEDLEY_UNLIKELY(std::isfinite(number) && !std::isfinite(result))) + { + return sax->parse_error(chars_read, get_token_string(), + out_of_range::create(406, exception_message(format, "number overflow", "value"), nullptr)); + } + return sax->number_float(result, ""); + } + /*! @brief create a string by reading characters from the input diff --git a/tests/src/unit-binary_formats.cpp b/tests/src/unit-binary_formats.cpp index ed6d89911..f1fb06582 100644 --- a/tests/src/unit-binary_formats.cpp +++ b/tests/src/unit-binary_formats.cpp @@ -11,7 +11,12 @@ #include using nlohmann::json; +#include #include +#include +#include +#include +#include #include "make_test_data_available.hpp" TEST_CASE("Binary Formats" * doctest::skip()) @@ -224,3 +229,114 @@ TEST_CASE("Binary Formats" * doctest::skip()) CHECK((100.0 * double(ubjson_3_size) / double(json_size)) == Approx(89.450)); } } + +TEST_CASE("Binary formats with narrow number types") +{ + // Numbers that do not fit the number types are handled like the lexer + // handles them in JSON text: an integer that fits neither integer type is + // stored as a floating-point number, and a finite floating-point number + // that overflows number_float_t is rejected with out_of_range.406. + using narrow_json = nlohmann::basic_json; + using bytes = std::vector; + + struct binary_format + { + const char* name; + bytes (*encode)(const json&); + narrow_json (*decode)(const bytes&, bool); + }; + + const std::vector formats = + { + { + "CBOR", [](const json & j) { return json::to_cbor(j); }, + [](const bytes & v, bool allow_exceptions) + { + return narrow_json::from_cbor(v, true, allow_exceptions); + } + }, + { + "MessagePack", [](const json & j) { return json::to_msgpack(j); }, + [](const bytes & v, bool allow_exceptions) + { + return narrow_json::from_msgpack(v, true, allow_exceptions); + } + }, + { + "UBJSON", [](const json & j) { return json::to_ubjson(j); }, + [](const bytes & v, bool allow_exceptions) + { + return narrow_json::from_ubjson(v, true, allow_exceptions); + } + }, + { + "BJData", [](const json & j) { return json::to_bjdata(j); }, + [](const bytes & v, bool allow_exceptions) + { + return narrow_json::from_bjdata(v, true, allow_exceptions); + } + }, + { + // BSON can only store numbers as object members + "BSON", [](const json & j) { return json::to_bson(json{{"a", j}}); }, + [](const bytes & v, bool allow_exceptions) + { + const auto result = narrow_json::from_bson(v, true, allow_exceptions); + return result.is_discarded() ? result : result.at("a"); + } + }, + { + "BON8", [](const json & j) { return json::to_bon8(j); }, + [](const bytes & v, bool allow_exceptions) + { + return narrow_json::from_bon8(v, true, allow_exceptions); + } + }, + }; + + for (const auto& format : formats) + { + const std::string name = format.name; + INFO("format := ", name); + const auto roundtrip = [&format](const json & j) + { + return format.decode(format.encode(j), true); + }; + + // integers that fit keep their type + CHECK(roundtrip(json(-5)).is_number_integer()); + CHECK(roundtrip(json(-5)).get() == -5); + CHECK(roundtrip(json(3000000000u)).is_number_unsigned()); + CHECK(roundtrip(json(3000000000u)).get() == 3000000000u); + + // integers that fit neither integer type are stored as float + CHECK(roundtrip(json(5000000000u)).is_number_float()); + CHECK(roundtrip(json(5000000000u)).get() == 5000000000.0f); + if (name != "BON8") // BON8 cannot encode integers above INT64_MAX + { + CHECK(roundtrip(json(10000000000000000000u)).is_number_float()); + CHECK(roundtrip(json(10000000000000000000u)).get() == 10000000000000000000.0f); + } + CHECK(roundtrip(json(-3000000000)).is_number_float()); + CHECK(roundtrip(json(-3000000000)).get() == -3000000000.0f); + CHECK(roundtrip(json(-5000000000)).is_number_float()); + CHECK(roundtrip(json(-5000000000)).get() == -5000000000.0f); + + // floating-point numbers that fit + CHECK(roundtrip(json(1.5)).get() == 1.5f); + const auto just_above_max = std::nextafter(static_cast((std::numeric_limits::max)()), + std::numeric_limits::infinity()); + CHECK(roundtrip(json(just_above_max)).get() == (std::numeric_limits::max)()); + + // infinity and NaN are passed on + CHECK(std::isinf(roundtrip(json(std::numeric_limits::infinity())).get())); + CHECK(std::isnan(roundtrip(json(std::numeric_limits::quiet_NaN())).get())); + + // finite floating-point numbers that overflow number_float_t are rejected + const std::string message = "[json.exception.out_of_range.406] syntax error while parsing " + name + + " value: number overflow"; + CHECK_THROWS_WITH_AS(roundtrip(json(1e300)), message.c_str(), narrow_json::out_of_range&); + CHECK_THROWS_WITH_AS(roundtrip(json(-1e300)), message.c_str(), narrow_json::out_of_range&); + CHECK(format.decode(format.encode(json(1e300)), false).is_discarded()); + } +} diff --git a/tests/src/unit-cbor.cpp b/tests/src/unit-cbor.cpp index 6bd792f8a..fe81135d5 100644 --- a/tests/src/unit-cbor.cpp +++ b/tests/src/unit-cbor.cpp @@ -3146,7 +3146,8 @@ TEST_CASE("Tagged values") // CBOR encodes negative integers as: result = -1 - n // For type 0x3B, n is an 8-byte uint64_t. Valid range for n with // the default int64_t is [0, INT64_MAX], producing results in [INT64_MIN, -1]. - // When n > INT64_MAX, the result exceeds int64_t range and is rejected. + // When n > INT64_MAX, the result exceeds int64_t range and is stored + // as a floating-point number, as the lexer does for JSON text. SECTION("n = 0 is valid (result = -1)") { @@ -3167,33 +3168,34 @@ TEST_CASE("Tagged values") CHECK(result.get() == (std::numeric_limits::min)()); } - SECTION("n = INT64_MAX + 1 is rejected (overflow)") + SECTION("n = INT64_MAX + 1 is stored as float") { // n = INT64_MAX + 1 (0x8000000000000000) - // result = -1 - n = -9223372036854775809, which exceeds int64_t range + // result = -1 - n = -9223372036854775809, which exceeds int64_t range; + // the nearest double is -9223372036854775808.0 const std::vector input = {0x3B, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; - json _; - CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), - "[json.exception.parse_error.112] parse error at byte 9: syntax error while parsing CBOR value: negative integer overflow", - json::parse_error); + const auto result = json::from_cbor(input); + CHECK(result.is_number_float()); + CHECK(result.get() == -9223372036854775808.0); + CHECK(result == json::parse("-9223372036854775809")); } - SECTION("n = UINT64_MAX is rejected (overflow)") + SECTION("n = UINT64_MAX is stored as float") { // n = UINT64_MAX (0xFFFFFFFFFFFFFFFF) // result = -1 - n = -18446744073709551616, which exceeds int64_t range const std::vector input = {0x3B, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}; - json _; - CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), - "[json.exception.parse_error.112] parse error at byte 9: syntax error while parsing CBOR value: negative integer overflow", - json::parse_error); + const auto result = json::from_cbor(input); + CHECK(result.is_number_float()); + CHECK(result.get() == -18446744073709551616.0); + CHECK(result == json::parse("-18446744073709551616")); } - SECTION("overflow with allow_exceptions=false returns discarded") + SECTION("overflow with allow_exceptions=false is not an error") { const std::vector input = {0x3B, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; const auto result = json::from_cbor(input, true, false); - CHECK(result.is_discarded()); + CHECK(result.is_number_float()); } }