From 35173b9d2d82ab5b19ec7274a595a4e6aa7965e7 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 25 Sep 2026 22:14:47 +0200 Subject: [PATCH] Throw instead of writing MessagePack lengths beyond UINT32_MAX MessagePack stores the length of a string, binary value, array, or object in at most 32 bits. For a larger value, to_msgpack wrote no length at all, so the output could not be read back. It now throws out_of_range.412, which BSON already uses for its 32-bit length fields. The check lives in one function, so each length is written by an if/else chain that ends in a plain else, without a condition that can never be false. It is tested with string and binary types that report a size beyond UINT32_MAX without allocating it, like the BSON tests do. Signed-off-by: Niels Lohmann --- docs/mkdocs/docs/api/basic_json/to_msgpack.md | 10 +++ .../features/binary_formats/messagepack.md | 2 + docs/mkdocs/docs/home/exceptions.md | 14 +++-- .../nlohmann/detail/output/binary_writer.hpp | 32 +++++++--- single_include/nlohmann/json.hpp | 32 +++++++--- tests/src/unit-msgpack.cpp | 63 +++++++++++++++++++ 6 files changed, 133 insertions(+), 20 deletions(-) diff --git a/docs/mkdocs/docs/api/basic_json/to_msgpack.md b/docs/mkdocs/docs/api/basic_json/to_msgpack.md index 66b104f52..0330461ea 100644 --- a/docs/mkdocs/docs/api/basic_json/to_msgpack.md +++ b/docs/mkdocs/docs/api/basic_json/to_msgpack.md @@ -30,6 +30,15 @@ The exact mapping and its limitations are described on a [dedicated page](../../ 1. MessagePack serialization as a byte vector 2. (none) +## Exceptions + +- Throws [`out_of_range.412`](../../home/exceptions.md#jsonexceptionout_of_range412) if the length of a string, binary + value, array, or object exceeds 4294967295, the maximum MessagePack can store; example: + `"MessagePack length 4294967296 exceeds maximum of 4294967295"` +- Throws [`out_of_range.415`](../../home/exceptions.md#jsonexceptionout_of_range415) if the subtype of a binary value + exceeds 255, the maximum of the MessagePack ext type; example: + `"subtype 70000 is too large for the MessagePack ext type (max 255)"` + ## Exception safety Strong guarantee: if an exception is thrown, there are no changes in the JSON value. @@ -65,3 +74,4 @@ Linear in the size of the JSON value `j`. ## Version history - Added in version 2.0.9. +- Throws `out_of_range.412` and `out_of_range.415` since version 3.13.0. diff --git a/docs/mkdocs/docs/features/binary_formats/messagepack.md b/docs/mkdocs/docs/features/binary_formats/messagepack.md index a434909c4..0ca82c145 100644 --- a/docs/mkdocs/docs/features/binary_formats/messagepack.md +++ b/docs/mkdocs/docs/features/binary_formats/messagepack.md @@ -65,6 +65,8 @@ specification: - arrays with more than 4294967295 elements - objects with more than 4294967295 elements + Serializing such a value throws [`out_of_range.412`](../../home/exceptions.md#jsonexceptionout_of_range412). + !!! info "NaN/infinity handling" `NaN`, `Infinity`, and `-Infinity` are serialized as a MessagePack float 32 (type 0xCA, 5 bytes total), diff --git a/docs/mkdocs/docs/home/exceptions.md b/docs/mkdocs/docs/home/exceptions.md index ee76596f6..bf18baab1 100644 --- a/docs/mkdocs/docs/home/exceptions.md +++ b/docs/mkdocs/docs/home/exceptions.md @@ -932,19 +932,25 @@ A JSON Patch `add` operation cannot be applied because the target location's par ### json.exception.out_of_range.412 -BSON stores the length of documents, arrays, strings, and binary values in a signed 32-bit integer. This exception is thrown when a value is too large to be described by such a length field. +BSON stores the length of documents, arrays, strings, and binary values in a signed 32-bit integer, and MessagePack +stores the length of strings, binary values, arrays, and objects in at most an unsigned 32-bit integer. This exception +is thrown when a value is too large to be described by such a length field. -!!! failure "Example message" +!!! failure "Example messages" ``` BSON length 2147483661 exceeds maximum of 2147483647 ``` + ``` + MessagePack length 4294967296 exceeds maximum of 4294967295 + ``` !!! note - This exception was added in version 3.13.0. Before that, the length was silently truncated, and + This exception was added in version 3.13.0. Before that, the BSON length was silently truncated, and [`to_bson`](../api/basic_json/to_bson.md) produced documents with negative length prefixes that - [`from_bson`](../api/basic_json/from_bson.md) rejected. + [`from_bson`](../api/basic_json/from_bson.md) rejected; [`to_msgpack`](../api/basic_json/to_msgpack.md) wrote such + a value without any length, producing output that could not be read back. ### json.exception.out_of_range.413 diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 5a0722a18..9d2ed55d7 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -466,6 +466,22 @@ class binary_writer } } + /*! + @brief check that @a length fits into the 32 bits that MessagePack stores + the length of a string, binary value, array, or object in + @return the length as an unsigned 32-bit integer + @throw out_of_range.412 if @a length exceeds the range of std::uint32_t + */ + static std::uint32_t to_msgpack_length(const std::size_t length, const BasicJsonType& j) + { + if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(length))) + { + JSON_THROW(out_of_range::create(412, concat("MessagePack length ", std::to_string(length), " exceeds maximum of ", std::to_string((std::numeric_limits::max)())), &j)); + } + + return static_cast(length); + } + /*! @param[in] j JSON value to serialize */ @@ -606,7 +622,7 @@ class binary_writer case value_t::string: { // step 1: write control byte and the string length - const auto N = j.m_data.m_value.string->size(); + const auto N = to_msgpack_length(j.m_data.m_value.string->size(), j); if (N <= 31) { // fixstr @@ -624,7 +640,7 @@ class binary_writer oa.write_character(to_char_type(0xDA)); write_number(static_cast(N)); } - else if (N <= (std::numeric_limits::max)()) + else { // str 32 oa.write_character(to_char_type(0xDB)); @@ -641,7 +657,7 @@ class binary_writer case value_t::array: { // step 1: write control byte and the array size - const auto N = j.m_data.m_value.array->size(); + const auto N = to_msgpack_length(j.m_data.m_value.array->size(), j); if (N <= 15) { // fixarray @@ -653,7 +669,7 @@ class binary_writer oa.write_character(to_char_type(0xDC)); write_number(static_cast(N)); } - else if (N <= (std::numeric_limits::max)()) + else { // array 32 oa.write_character(to_char_type(0xDD)); @@ -675,7 +691,7 @@ class binary_writer const bool use_ext = j.m_data.m_value.binary->has_subtype(); // step 1: write control byte and the byte string length - const auto N = j.m_data.m_value.binary->size(); + const auto N = to_msgpack_length(j.m_data.m_value.binary->size(), j); if (N <= (std::numeric_limits::max)()) { std::uint8_t output_type{}; @@ -727,7 +743,7 @@ class binary_writer oa.write_character(to_char_type(output_type)); write_number(static_cast(N)); } - else if (N <= (std::numeric_limits::max)()) + else { const std::uint8_t output_type = use_ext ? 0xC9 // ext 32 @@ -759,7 +775,7 @@ class binary_writer case value_t::object: { // step 1: write control byte and the object size - const auto N = j.m_data.m_value.object->size(); + const auto N = to_msgpack_length(j.m_data.m_value.object->size(), j); if (N <= 15) { // fixmap @@ -771,7 +787,7 @@ class binary_writer oa.write_character(to_char_type(0xDE)); write_number(static_cast(N)); } - else if (N <= (std::numeric_limits::max)()) + else { // map 32 oa.write_character(to_char_type(0xDF)); diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 4981c3b02..cd523127e 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -19931,6 +19931,22 @@ class binary_writer } } + /*! + @brief check that @a length fits into the 32 bits that MessagePack stores + the length of a string, binary value, array, or object in + @return the length as an unsigned 32-bit integer + @throw out_of_range.412 if @a length exceeds the range of std::uint32_t + */ + static std::uint32_t to_msgpack_length(const std::size_t length, const BasicJsonType& j) + { + if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(length))) + { + JSON_THROW(out_of_range::create(412, concat("MessagePack length ", std::to_string(length), " exceeds maximum of ", std::to_string((std::numeric_limits::max)())), &j)); + } + + return static_cast(length); + } + /*! @param[in] j JSON value to serialize */ @@ -20071,7 +20087,7 @@ class binary_writer case value_t::string: { // step 1: write control byte and the string length - const auto N = j.m_data.m_value.string->size(); + const auto N = to_msgpack_length(j.m_data.m_value.string->size(), j); if (N <= 31) { // fixstr @@ -20089,7 +20105,7 @@ class binary_writer oa.write_character(to_char_type(0xDA)); write_number(static_cast(N)); } - else if (N <= (std::numeric_limits::max)()) + else { // str 32 oa.write_character(to_char_type(0xDB)); @@ -20106,7 +20122,7 @@ class binary_writer case value_t::array: { // step 1: write control byte and the array size - const auto N = j.m_data.m_value.array->size(); + const auto N = to_msgpack_length(j.m_data.m_value.array->size(), j); if (N <= 15) { // fixarray @@ -20118,7 +20134,7 @@ class binary_writer oa.write_character(to_char_type(0xDC)); write_number(static_cast(N)); } - else if (N <= (std::numeric_limits::max)()) + else { // array 32 oa.write_character(to_char_type(0xDD)); @@ -20140,7 +20156,7 @@ class binary_writer const bool use_ext = j.m_data.m_value.binary->has_subtype(); // step 1: write control byte and the byte string length - const auto N = j.m_data.m_value.binary->size(); + const auto N = to_msgpack_length(j.m_data.m_value.binary->size(), j); if (N <= (std::numeric_limits::max)()) { std::uint8_t output_type{}; @@ -20192,7 +20208,7 @@ class binary_writer oa.write_character(to_char_type(output_type)); write_number(static_cast(N)); } - else if (N <= (std::numeric_limits::max)()) + else { const std::uint8_t output_type = use_ext ? 0xC9 // ext 32 @@ -20224,7 +20240,7 @@ class binary_writer case value_t::object: { // step 1: write control byte and the object size - const auto N = j.m_data.m_value.object->size(); + const auto N = to_msgpack_length(j.m_data.m_value.object->size(), j); if (N <= 15) { // fixmap @@ -20236,7 +20252,7 @@ class binary_writer oa.write_character(to_char_type(0xDE)); write_number(static_cast(N)); } - else if (N <= (std::numeric_limits::max)()) + else { // map 32 oa.write_character(to_char_type(0xDF)); diff --git a/tests/src/unit-msgpack.cpp b/tests/src/unit-msgpack.cpp index 805b03a13..c1693065b 100644 --- a/tests/src/unit-msgpack.cpp +++ b/tests/src/unit-msgpack.cpp @@ -14,6 +14,7 @@ using nlohmann::json; using namespace nlohmann::literals; // NOLINT(google-build-using-namespace) #endif +#include // SIZE_MAX, UINT32_MAX #include #include #include @@ -2150,3 +2151,65 @@ TEST_CASE("MessagePack with std::byte") } } #endif + +namespace +{ +// types that report a size beyond UINT32_MAX without allocating that much +// memory, so the MessagePack length limit can be tested cheaply; see the +// similar types in unit-bson.cpp +std::size_t beyond_uint32_size() +{ + return static_cast((std::numeric_limits::max)()) + 1; +} + +class beyond_uint32_binary_t : public std::vector +{ + public: + using std::vector::vector; + + size_type size() const noexcept // NOLINT(readability-convert-member-functions-to-static) + { + return beyond_uint32_size(); + } +}; + +class beyond_uint32_string_t : public std::string +{ + public: + using std::string::string; + + size_type size() const noexcept // NOLINT(readability-convert-member-functions-to-static) + { + return beyond_uint32_size(); + } +}; + +using beyond_uint32_binary_json = nlohmann::basic_json < + std::map, std::vector, std::string, bool, std::int64_t, std::uint64_t, + double, std::allocator, nlohmann::adl_serializer, beyond_uint32_binary_t, void >; + +using beyond_uint32_string_json = nlohmann::basic_json < + std::map, std::vector, beyond_uint32_string_t, bool, std::int64_t, std::uint64_t, + double, std::allocator, nlohmann::adl_serializer, std::vector, void >; +} // namespace + +TEST_CASE("MessagePack lengths beyond UINT32_MAX cannot be serialized") +{ + // MessagePack stores the length of a string, binary value, array, or + // object in at most 32 bits; a larger one used to be written without any + // length at all +#if SIZE_MAX > UINT32_MAX + { + const char* const expected = "[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295"; + + const beyond_uint32_binary_json binary = beyond_uint32_binary_json::binary(beyond_uint32_binary_t{}); + CHECK_THROWS_WITH_AS(beyond_uint32_binary_json::to_msgpack(binary), expected, beyond_uint32_binary_json::out_of_range&); + + const beyond_uint32_binary_json ext = beyond_uint32_binary_json::binary(beyond_uint32_binary_t{}, 42); + CHECK_THROWS_WITH_AS(beyond_uint32_binary_json::to_msgpack(ext), expected, beyond_uint32_binary_json::out_of_range&); + + const beyond_uint32_string_json string = beyond_uint32_string_t("value"); + CHECK_THROWS_WITH_AS(beyond_uint32_string_json::to_msgpack(string), expected, beyond_uint32_string_json::out_of_range&); + } +#endif +}