From 5e5a087cb371f404321d86f30c7a56bd5a005ca4 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 16 Sep 2026 05:19:49 +0000 Subject: [PATCH] Validate UTF-8 in CBOR/MessagePack/BSON/UBJSON/BJData text strings on develop PR #5531 fixed the UTF-8-validation gap described in #5529, but it was merged onto the still-unmerged bson-sizes branch rather than develop, so develop was left with the original bug for all affected formats. A follow-up comment on #5529 reproduced this on develop and additionally found that UBJSON (and, by the same code path, BJData) has the identical gap, undocumented. Port the same fix directly onto develop: extract the UTF-8 DFA decoder out of serializer<>::decode() into a shared detail::decode()/is_valid_utf8() in string_utils.hpp, and call it from binary_reader::get_string() - the single choke point shared by all five binary readers - so malformed text strings are rejected at decode time (parse_error.113) instead of only failing later on dump() (type_error.316). Byte/binary payloads are unaffected. Add matching decode-time tests for CBOR, MessagePack, BSON, UBJSON, and BJData, and document the new behavior on all five binary format pages (the two UBJSON/BJData pages didn't get this note in #5531). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017sdieJCn6BHxzRMaXP49sP --- .../docs/features/binary_formats/bjdata.md | 8 + .../docs/features/binary_formats/bson.md | 9 + .../docs/features/binary_formats/cbor.md | 10 + .../features/binary_formats/messagepack.md | 8 + .../docs/features/binary_formats/ubjson.md | 8 + docs/mkdocs/docs/home/exceptions.md | 6 +- .../nlohmann/detail/input/binary_reader.hpp | 20 +- include/nlohmann/detail/output/serializer.hpp | 59 +--- include/nlohmann/detail/string_utils.hpp | 97 ++++++ single_include/nlohmann/json.hpp | 301 +++++++++++------- tests/src/unit-bjdata.cpp | 17 + tests/src/unit-bson.cpp | 26 ++ tests/src/unit-cbor.cpp | 21 ++ tests/src/unit-msgpack.cpp | 21 ++ tests/src/unit-ubjson.cpp | 17 + 15 files changed, 448 insertions(+), 180 deletions(-) diff --git a/docs/mkdocs/docs/features/binary_formats/bjdata.md b/docs/mkdocs/docs/features/binary_formats/bjdata.md index d3f63a9b8..9d1fb886e 100644 --- a/docs/mkdocs/docs/features/binary_formats/bjdata.md +++ b/docs/mkdocs/docs/features/binary_formats/bjdata.md @@ -204,6 +204,14 @@ The library maps BJData types to JSON value types as follows: The mapping is **complete** in the sense that any BJData value can be converted to a JSON value. +!!! warning "UTF-8 validation of string values" + + BJData does not specify an encoding for its `string`/`char` types, but this library requires them to be valid + UTF-8, consistent with the rest of the library. The bytes of every such string (object keys included) are + validated at decode time, and ill-formed UTF-8 is rejected with a + [`parse_error.113`](../../home/exceptions.md#jsonexceptionparse_error113) exception (or, with `allow_exceptions` + set to `false`, a discarded value), rather than only failing later when the resulting value is dumped. + ??? example ```cpp diff --git a/docs/mkdocs/docs/features/binary_formats/bson.md b/docs/mkdocs/docs/features/binary_formats/bson.md index 95c82e873..6f5603c8c 100644 --- a/docs/mkdocs/docs/features/binary_formats/bson.md +++ b/docs/mkdocs/docs/features/binary_formats/bson.md @@ -109,6 +109,15 @@ The library maps BSON record types to JSON value types as follows: If BSON input must be validated for strict specification compliance, validate it separately before passing it to `from_bson()`. +!!! warning "UTF-8 validation of string values" + + The BSON specification requires `string` values (type `0x02`) to be valid UTF-8. This library validates the + bytes of every such string at decode time and rejects ill-formed UTF-8 with a + [`parse_error.113`](../../home/exceptions.md#jsonexceptionparse_error113) exception (or, with `allow_exceptions` + set to `false`, a discarded value), rather than only failing later when the resulting value is dumped. Element + (key) names and `binary` values (type `0x05`) are unaffected and are never validated, since they are read + byte-by-byte as a C string, or are not required to hold text, respectively. + ??? example ```cpp diff --git a/docs/mkdocs/docs/features/binary_formats/cbor.md b/docs/mkdocs/docs/features/binary_formats/cbor.md index 670a23455..e4c257e27 100644 --- a/docs/mkdocs/docs/features/binary_formats/cbor.md +++ b/docs/mkdocs/docs/features/binary_formats/cbor.md @@ -176,6 +176,16 @@ The library maps CBOR types to JSON value types as follows: CBOR allows map keys of any type, whereas JSON only allows strings as keys in object values. Therefore, CBOR maps with keys other than UTF-8 strings are rejected. +!!! warning "UTF-8 validation of text strings" + + [RFC 8949, Section 3.1](https://www.rfc-editor.org/rfc/rfc8949.html#section-3.1) requires CBOR text strings + (major type 3) to be valid UTF-8. This library validates the bytes of every text string (object keys included) at + decode time and rejects ill-formed UTF-8 with a + [`parse_error.113`](../../home/exceptions.md#jsonexceptionparse_error113) exception (or, with + `allow_exceptions` set to `false`, a discarded value), rather than only failing later when the resulting value is + dumped. Byte strings (major type 2) are unaffected and are never validated, since they are not required to hold + text. + !!! warning "Tagged items" Tagged items (0xC0..0xDB) will throw a parse error by default. They can be ignored by passing `cbor_tag_handler_t::ignore` to function `from_cbor`, in which case the tag is skipped and the enclosed data item is parsed on its own. They can be stored by passing `cbor_tag_handler_t::store` to function `from_cbor`. Note that no tag is ever interpreted: for instance, a text string tagged with tag 0 (date/time) stays a string. diff --git a/docs/mkdocs/docs/features/binary_formats/messagepack.md b/docs/mkdocs/docs/features/binary_formats/messagepack.md index bd0c840f2..a434909c4 100644 --- a/docs/mkdocs/docs/features/binary_formats/messagepack.md +++ b/docs/mkdocs/docs/features/binary_formats/messagepack.md @@ -136,6 +136,14 @@ The library maps MessagePack types to JSON value types as follows: Any MessagePack output created by `to_msgpack` can be successfully parsed by `from_msgpack`. +!!! warning "UTF-8 validation of string values" + + The MessagePack specification requires `str` values (`fixstr`, `str 8`, `str 16`, `str 32`) to be valid UTF-8. + This library validates the bytes of every such string (object keys included) at decode time and rejects + ill-formed UTF-8 with a [`parse_error.113`](../../home/exceptions.md#jsonexceptionparse_error113) exception (or, + with `allow_exceptions` set to `false`, a discarded value), rather than only failing later when the resulting + value is dumped. `bin`/`ext`/`fixext` values are unaffected and are never validated, since they are not required + to hold text. ??? example diff --git a/docs/mkdocs/docs/features/binary_formats/ubjson.md b/docs/mkdocs/docs/features/binary_formats/ubjson.md index be545b9fe..589520ef4 100644 --- a/docs/mkdocs/docs/features/binary_formats/ubjson.md +++ b/docs/mkdocs/docs/features/binary_formats/ubjson.md @@ -120,6 +120,14 @@ The library maps UBJSON types to JSON value types as follows: The mapping is **complete** in the sense that any UBJSON value can be converted to a JSON value. +!!! warning "UTF-8 validation of string values" + + UBJSON does not specify an encoding for its `string`/`char` types, but this library requires them to be valid + UTF-8, consistent with the rest of the library. The bytes of every such string (object keys included) are + validated at decode time, and ill-formed UTF-8 is rejected with a + [`parse_error.113`](../../home/exceptions.md#jsonexceptionparse_error113) exception (or, with `allow_exceptions` + set to `false`, a discarded value), rather than only failing later when the resulting value is dumped. + ??? example ```cpp diff --git a/docs/mkdocs/docs/home/exceptions.md b/docs/mkdocs/docs/home/exceptions.md index 09cc8e178..fd3a732e2 100644 --- a/docs/mkdocs/docs/home/exceptions.md +++ b/docs/mkdocs/docs/home/exceptions.md @@ -340,7 +340,8 @@ An unexpected byte was read in a [binary format](../features/binary_formats/inde ### json.exception.parse_error.113 A string could not be read from a [binary format](../features/binary_formats/index.md): either a value that is not a -string was read where one was required (for instance as a map key), or the string's length specification is invalid. +string was read where one was required (for instance as a map key), the string's length specification is invalid, or +the string's bytes are not valid UTF-8. !!! failure "Example messages" @@ -356,6 +357,9 @@ string was read where one was required (for instance as a map key), or the strin ``` [json.exception.parse_error.113] parse error at byte 3: syntax error while parsing BJData string: string length must not be negative ``` + ``` + [json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR string: invalid string: ill-formed UTF-8 byte + ``` ### json.exception.parse_error.114 diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index df46eea58..bbd43e3fc 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -32,6 +32,7 @@ #include #include #include +#include #include NLOHMANN_JSON_NAMESPACE_BEGIN @@ -3304,7 +3305,24 @@ class binary_reader const NumberType len, string_t& result) { - return get_bytes(format, len, "string", result); + if (JSON_HEDLEY_UNLIKELY(!get_bytes(format, len, "string", result))) + { + return false; + } + + // RFC 8949 (CBOR) §3.1 and the MessagePack/BSON/UBJSON specifications + // all require text strings to be valid UTF-8; reject anything else + // right here so malformed input is caught at decode time instead of + // only surfacing later as a type_error.316 when the value is dumped + // (which would defeat allow_exceptions=false / strict discarding). + if (JSON_HEDLEY_UNLIKELY(!is_valid_utf8(result))) + { + return sax->parse_error(chars_read, get_token_string(), + parse_error::create(113, chars_read, + exception_message(format, "invalid string: ill-formed UTF-8 byte", "string"), nullptr)); + } + + return true; } /*! diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index f9e7f7840..131d3bfeb 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -31,6 +31,7 @@ #include #include #include +#include #include NLOHMANN_JSON_NAMESPACE_BEGIN @@ -57,8 +58,6 @@ class serializer using number_integer_t = typename BasicJsonType::number_integer_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t; using binary_char_t = typename BasicJsonType::binary_t::value_type; - static constexpr std::uint8_t UTF8_ACCEPT = 0; - static constexpr std::uint8_t UTF8_REJECT = 1; public: /*! @@ -1598,62 +1597,6 @@ class serializer } } - /*! - @brief check whether a string is UTF-8 encoded - - The function checks each byte of a string whether it is UTF-8 encoded. The - result of the check is stored in the @a state parameter. The function must - be called initially with state 0 (accept). State 1 means the string must - be rejected, because the current byte is not allowed. If the string is - completely processed, but the state is non-zero, the string ended - prematurely; that is, the last byte indicated more bytes should have - followed. - - @param[in,out] state the state of the decoding - @param[in,out] codep codepoint (valid only if resulting state is UTF8_ACCEPT) - @param[in] byte next byte to decode - @return new state - - @note The function has been edited: a std::array is used. - - @copyright Copyright (c) 2008-2009 Bjoern Hoehrmann - @sa http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ - */ - static std::uint8_t decode(std::uint8_t& state, std::uint32_t& codep, const std::uint8_t byte) noexcept - { - static const std::array utf8d = - { - { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 00..1F - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20..3F - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 40..5F - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 60..7F - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, // 80..9F - 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // A0..BF - 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C0..DF - 0xA, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x4, 0x3, 0x3, // E0..EF - 0xB, 0x6, 0x6, 0x6, 0x5, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, // F0..FF - 0x0, 0x1, 0x2, 0x3, 0x5, 0x8, 0x7, 0x1, 0x1, 0x1, 0x4, 0x6, 0x1, 0x1, 0x1, 0x1, // s0..s0 - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, // s1..s2 - 1, 2, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, // s3..s4 - 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, // s5..s6 - 1, 3, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // s7..s8 - } - }; - - JSON_ASSERT(static_cast(byte) < utf8d.size()); - const std::uint8_t type = utf8d[byte]; - - codep = (state != UTF8_ACCEPT) - ? (byte & 0x3fu) | (codep << 6u) - : (0xFFu >> type) & (byte); - - const std::size_t index = 256u + (static_cast(state) * 16u) + static_cast(type); - JSON_ASSERT(index < utf8d.size()); - state = utf8d[index]; - return state; - } - /* * Overload to make the compiler happy while it is instantiating * dump_integer for number_unsigned_t. diff --git a/include/nlohmann/detail/string_utils.hpp b/include/nlohmann/detail/string_utils.hpp index fe2f9109d..064d5ccf9 100644 --- a/include/nlohmann/detail/string_utils.hpp +++ b/include/nlohmann/detail/string_utils.hpp @@ -8,10 +8,13 @@ #pragma once +#include // array #include // size_t +#include // uint8_t, uint32_t #include // string, to_string #include +#include NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail @@ -33,5 +36,99 @@ StringType to_string(std::size_t value) return result; } +/////////////////// +// UTF-8 decoding // +/////////////////// + +// UTF-8 decoder states used by decode() below +static constexpr std::uint8_t UTF8_ACCEPT = 0; +static constexpr std::uint8_t UTF8_REJECT = 1; + +/*! +@brief process a byte of a UTF-8 sequence + +This is a single-byte step of a "shift-based" UTF-8 decoder originally +written by Björn Hoehrmann. See +http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details. + +This decoder is the single source of truth for UTF-8 validation in this +library: it is used both by the serializer (to escape and, in strict mode, +reject ill-formed UTF-8 when dumping a string) and by the binary readers +(to reject ill-formed UTF-8 in CBOR/MessagePack/BSON/UBJSON text strings at +decode time; see @ref is_valid_utf8 below). + +@param[in,out] state the current decoder state +@param[in,out] codep codepoint (valid only if resulting state is UTF8_ACCEPT) +@param[in] byte next byte to decode +@return new state + +@note Original source: http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ +@sa http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ +*/ +inline std::uint8_t decode(std::uint8_t& state, std::uint32_t& codep, const std::uint8_t byte) noexcept +{ + static const std::array utf8d = + { + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 00..1F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20..3F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 40..5F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 60..7F + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, // 80..9F + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // A0..BF + 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C0..DF + 0xA, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x4, 0x3, 0x3, // E0..EF + 0xB, 0x6, 0x6, 0x6, 0x5, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, // F0..FF + 0x0, 0x1, 0x2, 0x3, 0x5, 0x8, 0x7, 0x1, 0x1, 0x1, 0x4, 0x6, 0x1, 0x1, 0x1, 0x1, // s0..s0 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, // s1..s2 + 1, 2, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, // s3..s4 + 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, // s5..s6 + 1, 3, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // s7..s8 + } + }; + + JSON_ASSERT(static_cast(byte) < utf8d.size()); + const std::uint8_t type = utf8d[byte]; + + codep = (state != UTF8_ACCEPT) + ? (byte & 0x3fu) | (codep << 6u) + : (0xFFu >> type) & (byte); + + const std::size_t index = 256u + (static_cast(state) * 16u) + static_cast(type); + JSON_ASSERT(index < utf8d.size()); + state = utf8d[index]; + return state; +} + +/*! +@brief check whether a string consists solely of valid UTF-8 + +Used by the CBOR/MessagePack/BSON/UBJSON binary readers to reject text +strings that are not valid UTF-8 at decode time (RFC 8949 §3.1 and the +MessagePack/BSON specifications all require text strings to be UTF-8), so +that malformed input is caught immediately instead of only surfacing later +as a type_error.316 when the resulting value is dumped. + +@param[in] s the string to check +@return whether @a s is valid UTF-8 +*/ +template +inline bool is_valid_utf8(const StringType& s) noexcept +{ + std::uint8_t state = UTF8_ACCEPT; + std::uint32_t codepoint = 0; + + for (std::size_t i = 0; i < s.size(); ++i) + { + decode(state, codepoint, static_cast(s[i])); + if (state == UTF8_REJECT) + { + return false; + } + } + + return state == UTF8_ACCEPT; +} + } // namespace detail NLOHMANN_JSON_NAMESPACE_END diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 35443e141..ce44e7605 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -3824,71 +3824,71 @@ NLOHMANN_JSON_NAMESPACE_END // SPDX-License-Identifier: MIT #ifndef INCLUDE_NLOHMANN_JSON_FWD_HPP_ - #define INCLUDE_NLOHMANN_JSON_FWD_HPP_ +#define INCLUDE_NLOHMANN_JSON_FWD_HPP_ - #include // int64_t, uint64_t - #include // map - #include // allocator - #include // string - #include // vector +#include // int64_t, uint64_t +#include // map +#include // allocator +#include // string +#include // vector - // #include +// #include - /*! - @brief namespace for Niels Lohmann - @see https://github.com/nlohmann - @since version 1.0.0 - */ - NLOHMANN_JSON_NAMESPACE_BEGIN +/*! +@brief namespace for Niels Lohmann +@see https://github.com/nlohmann +@since version 1.0.0 +*/ +NLOHMANN_JSON_NAMESPACE_BEGIN - /*! - @brief default JSONSerializer template argument +/*! +@brief default JSONSerializer template argument - This serializer ignores the template arguments and uses ADL - ([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl)) - for serialization. - */ - template - struct adl_serializer; +This serializer ignores the template arguments and uses ADL +([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl)) +for serialization. +*/ +template +struct adl_serializer; - /// a class to store JSON values - /// @sa https://json.nlohmann.me/api/basic_json/ - template class ObjectType = - std::map, - template class ArrayType = std::vector, - class StringType = std::string, class BooleanType = bool, - class NumberIntegerType = std::int64_t, - class NumberUnsignedType = std::uint64_t, - class NumberFloatType = double, - template class AllocatorType = std::allocator, - template class JSONSerializer = - adl_serializer, - class BinaryType = std::vector, // cppcheck-suppress syntaxError - class CustomBaseClass = void> - class basic_json; +/// a class to store JSON values +/// @sa https://json.nlohmann.me/api/basic_json/ +template class ObjectType = + std::map, + template class ArrayType = std::vector, + class StringType = std::string, class BooleanType = bool, + class NumberIntegerType = std::int64_t, + class NumberUnsignedType = std::uint64_t, + class NumberFloatType = double, + template class AllocatorType = std::allocator, + template class JSONSerializer = + adl_serializer, + class BinaryType = std::vector, // cppcheck-suppress syntaxError + class CustomBaseClass = void> +class basic_json; - /// @brief JSON Pointer defines a string syntax for identifying a specific value within a JSON document - /// @sa https://json.nlohmann.me/api/json_pointer/ - template - class json_pointer; +/// @brief JSON Pointer defines a string syntax for identifying a specific value within a JSON document +/// @sa https://json.nlohmann.me/api/json_pointer/ +template +class json_pointer; - /*! - @brief default specialization - @sa https://json.nlohmann.me/api/json/ - */ - using json = basic_json<>; +/*! +@brief default specialization +@sa https://json.nlohmann.me/api/json/ +*/ +using json = basic_json<>; - /// @brief a minimal map-like container that preserves insertion order - /// @sa https://json.nlohmann.me/api/ordered_map/ - template - struct ordered_map; +/// @brief a minimal map-like container that preserves insertion order +/// @sa https://json.nlohmann.me/api/ordered_map/ +template +struct ordered_map; - /// @brief specialization that maintains the insertion order of object keys - /// @sa https://json.nlohmann.me/api/ordered_json/ - using ordered_json = basic_json; +/// @brief specialization that maintains the insertion order of object keys +/// @sa https://json.nlohmann.me/api/ordered_json/ +using ordered_json = basic_json; - NLOHMANN_JSON_NAMESPACE_END +NLOHMANN_JSON_NAMESPACE_END #endif // INCLUDE_NLOHMANN_JSON_FWD_HPP_ @@ -6007,7 +6007,7 @@ NLOHMANN_JSON_NAMESPACE_END // #include -// JSON_HAS_CPP_17 + // JSON_HAS_CPP_17 #ifdef JSON_HAS_CPP_17 #include // optional #endif @@ -6059,11 +6059,15 @@ NLOHMANN_JSON_NAMESPACE_END +#include // array #include // size_t +#include // uint8_t, uint32_t #include // string, to_string // #include +// #include + NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail @@ -6085,6 +6089,100 @@ StringType to_string(std::size_t value) return result; } +/////////////////// +// UTF-8 decoding // +/////////////////// + +// UTF-8 decoder states used by decode() below +static constexpr std::uint8_t UTF8_ACCEPT = 0; +static constexpr std::uint8_t UTF8_REJECT = 1; + +/*! +@brief process a byte of a UTF-8 sequence + +This is a single-byte step of a "shift-based" UTF-8 decoder originally +written by Björn Hoehrmann. See +http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details. + +This decoder is the single source of truth for UTF-8 validation in this +library: it is used both by the serializer (to escape and, in strict mode, +reject ill-formed UTF-8 when dumping a string) and by the binary readers +(to reject ill-formed UTF-8 in CBOR/MessagePack/BSON/UBJSON text strings at +decode time; see @ref is_valid_utf8 below). + +@param[in,out] state the current decoder state +@param[in,out] codep codepoint (valid only if resulting state is UTF8_ACCEPT) +@param[in] byte next byte to decode +@return new state + +@note Original source: http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ +@sa http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ +*/ +inline std::uint8_t decode(std::uint8_t& state, std::uint32_t& codep, const std::uint8_t byte) noexcept +{ + static const std::array utf8d = + { + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 00..1F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20..3F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 40..5F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 60..7F + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, // 80..9F + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // A0..BF + 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C0..DF + 0xA, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x4, 0x3, 0x3, // E0..EF + 0xB, 0x6, 0x6, 0x6, 0x5, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, // F0..FF + 0x0, 0x1, 0x2, 0x3, 0x5, 0x8, 0x7, 0x1, 0x1, 0x1, 0x4, 0x6, 0x1, 0x1, 0x1, 0x1, // s0..s0 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, // s1..s2 + 1, 2, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, // s3..s4 + 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, // s5..s6 + 1, 3, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // s7..s8 + } + }; + + JSON_ASSERT(static_cast(byte) < utf8d.size()); + const std::uint8_t type = utf8d[byte]; + + codep = (state != UTF8_ACCEPT) + ? (byte & 0x3fu) | (codep << 6u) + : (0xFFu >> type) & (byte); + + const std::size_t index = 256u + (static_cast(state) * 16u) + static_cast(type); + JSON_ASSERT(index < utf8d.size()); + state = utf8d[index]; + return state; +} + +/*! +@brief check whether a string consists solely of valid UTF-8 + +Used by the CBOR/MessagePack/BSON/UBJSON binary readers to reject text +strings that are not valid UTF-8 at decode time (RFC 8949 §3.1 and the +MessagePack/BSON specifications all require text strings to be UTF-8), so +that malformed input is caught immediately instead of only surfacing later +as a type_error.316 when the resulting value is dumped. + +@param[in] s the string to check +@return whether @a s is valid UTF-8 +*/ +template +inline bool is_valid_utf8(const StringType& s) noexcept +{ + std::uint8_t state = UTF8_ACCEPT; + std::uint32_t codepoint = 0; + + for (std::size_t i = 0; i < s.size(); ++i) + { + decode(state, codepoint, static_cast(s[i])); + if (state == UTF8_REJECT) + { + return false; + } + } + + return state == UTF8_ACCEPT; +} + } // namespace detail NLOHMANN_JSON_NAMESPACE_END @@ -12075,6 +12173,8 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include + // #include @@ -15348,7 +15448,24 @@ class binary_reader const NumberType len, string_t& result) { - return get_bytes(format, len, "string", result); + if (JSON_HEDLEY_UNLIKELY(!get_bytes(format, len, "string", result))) + { + return false; + } + + // RFC 8949 (CBOR) §3.1 and the MessagePack/BSON/UBJSON specifications + // all require text strings to be valid UTF-8; reject anything else + // right here so malformed input is caught at decode time instead of + // only surfacing later as a type_error.316 when the value is dumped + // (which would defeat allow_exceptions=false / strict discarding). + if (JSON_HEDLEY_UNLIKELY(!is_valid_utf8(result))) + { + return sax->parse_error(chars_read, get_token_string(), + parse_error::create(113, chars_read, + exception_message(format, "invalid string: ill-formed UTF-8 byte", "string"), nullptr)); + } + + return true; } /*! @@ -21873,6 +21990,8 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include + // #include @@ -21900,8 +22019,6 @@ class serializer using number_integer_t = typename BasicJsonType::number_integer_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t; using binary_char_t = typename BasicJsonType::binary_t::value_type; - static constexpr std::uint8_t UTF8_ACCEPT = 0; - static constexpr std::uint8_t UTF8_REJECT = 1; public: /*! @@ -23441,62 +23558,6 @@ class serializer } } - /*! - @brief check whether a string is UTF-8 encoded - - The function checks each byte of a string whether it is UTF-8 encoded. The - result of the check is stored in the @a state parameter. The function must - be called initially with state 0 (accept). State 1 means the string must - be rejected, because the current byte is not allowed. If the string is - completely processed, but the state is non-zero, the string ended - prematurely; that is, the last byte indicated more bytes should have - followed. - - @param[in,out] state the state of the decoding - @param[in,out] codep codepoint (valid only if resulting state is UTF8_ACCEPT) - @param[in] byte next byte to decode - @return new state - - @note The function has been edited: a std::array is used. - - @copyright Copyright (c) 2008-2009 Bjoern Hoehrmann - @sa http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ - */ - static std::uint8_t decode(std::uint8_t& state, std::uint32_t& codep, const std::uint8_t byte) noexcept - { - static const std::array utf8d = - { - { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 00..1F - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20..3F - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 40..5F - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 60..7F - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, // 80..9F - 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // A0..BF - 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C0..DF - 0xA, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x4, 0x3, 0x3, // E0..EF - 0xB, 0x6, 0x6, 0x6, 0x5, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, // F0..FF - 0x0, 0x1, 0x2, 0x3, 0x5, 0x8, 0x7, 0x1, 0x1, 0x1, 0x4, 0x6, 0x1, 0x1, 0x1, 0x1, // s0..s0 - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, // s1..s2 - 1, 2, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, // s3..s4 - 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, // s5..s6 - 1, 3, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // s7..s8 - } - }; - - JSON_ASSERT(static_cast(byte) < utf8d.size()); - const std::uint8_t type = utf8d[byte]; - - codep = (state != UTF8_ACCEPT) - ? (byte & 0x3fu) | (codep << 6u) - : (0xFFu >> type) & (byte); - - const std::size_t index = 256u + (static_cast(state) * 16u) + static_cast(type); - JSON_ASSERT(index < utf8d.size()); - state = utf8d[index]; - return state; - } - /* * Overload to make the compiler happy while it is instantiating * dump_integer for number_unsigned_t. @@ -24071,10 +24132,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool ignore_comments = false, const bool ignore_trailing_commas = false, const bool discard_number_values = false - ) + ) { return ::nlohmann::detail::parser(std::move(adapter), - std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_number_values); + std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_number_values); } private: @@ -24838,8 +24899,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::enable_if_t < !detail::is_basic_json::value && detail::is_compatible_type::value, int > = 0 > basic_json(CompatibleType && val) noexcept(noexcept( // NOLINT(bugprone-forwarding-reference-overload,bugprone-exception-escape) - JSONSerializer::to_json(std::declval(), - std::forward(val)))) + JSONSerializer::to_json(std::declval(), + std::forward(val)))) { JSONSerializer::to_json(*this, std::forward(val)); set_parents(); @@ -25661,7 +25722,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::has_from_json::value, int > = 0 > ValueType get_impl(detail::priority_tag<0> /*unused*/) const noexcept(noexcept( - JSONSerializer::from_json(std::declval(), std::declval()))) + JSONSerializer::from_json(std::declval(), std::declval()))) { auto ret = ValueType(); JSONSerializer::from_json(*this, ret); @@ -25703,7 +25764,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::has_non_default_from_json::value, int > = 0 > ValueType get_impl(detail::priority_tag<1> /*unused*/) const noexcept(noexcept( - JSONSerializer::from_json(std::declval()))) + JSONSerializer::from_json(std::declval()))) { return JSONSerializer::from_json(*this); } @@ -25853,7 +25914,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::has_from_json::value, int > = 0 > ValueType & get_to(ValueType& v) const noexcept(noexcept( - JSONSerializer::from_json(std::declval(), v))) + JSONSerializer::from_json(std::declval(), v))) { JSONSerializer::from_json(*this, v); return v; diff --git a/tests/src/unit-bjdata.cpp b/tests/src/unit-bjdata.cpp index 9338e663e..53f329411 100644 --- a/tests/src/unit-bjdata.cpp +++ b/tests/src/unit-bjdata.cpp @@ -2892,6 +2892,23 @@ TEST_CASE("BJData") CHECK(json::from_bjdata(vl, true, false).is_discarded()); } + SECTION("invalid UTF-8 in string (see #5529)") + { + // a BJData string of length 2 whose bytes are not valid + // UTF-8 (0xC0 0xAE is an overlong encoding of '.') must be + // rejected at decode time, matching every other kind of + // malformed binary input, rather than only failing later + // when the resulting value is dumped + std::vector const v = {'S', 'i', 0x02, 0xc0, 0xae}; + json _; + CHECK_THROWS_WITH_AS(_ = json::from_bjdata(v), "[json.exception.parse_error.113] parse error at byte 5: syntax error while parsing BJData string: invalid string: ill-formed UTF-8 byte", json::parse_error&); + CHECK(json::from_bjdata(v, true, false).is_discarded()); + + // valid UTF-8 must still round-trip + const json j = "h\xc3\xa9llo, w\xc3\xb6rld! \xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e"; // héllo, wörld! 日本語 + CHECK(json::from_bjdata(json::to_bjdata(j)) == j); + } + SECTION("parse bjdata markers in ubjson") { // create a single-character string for all number types diff --git a/tests/src/unit-bson.cpp b/tests/src/unit-bson.cpp index 153e12d30..cd961f15f 100644 --- a/tests/src/unit-bson.cpp +++ b/tests/src/unit-bson.cpp @@ -199,6 +199,32 @@ TEST_CASE("BSON") CHECK_THROWS_WITH_AS(_ = json::from_bson(v), "[json.exception.parse_error.112] parse error at byte 10: syntax error while parsing BSON string: string length must be at least 1, is -2147483648", json::parse_error&); } + SECTION("invalid UTF-8 in string (see #5529)") + { + // a BSON document with a string field "k" whose value bytes are not + // valid UTF-8 (0xC0 0xAE is an overlong encoding of '.') must be + // rejected at decode time, matching every other kind of malformed + // binary input, rather than only failing later when the resulting + // value is dumped + std::vector const v = + { + 0x0F, 0x00, 0x00, 0x00, // size (little endian) + 0x02, /// entry: string (UTF-8) + 'k', 0x00, // key "k" + 0x03, 0x00, 0x00, 0x00, // string length (including trailing zero byte) + 0xc0, 0xae, // ill-formed UTF-8 + 0x00, // string terminator + 0x00 // end marker + }; + json _; + CHECK_THROWS_WITH_AS(_ = json::from_bson(v), "[json.exception.parse_error.113] parse error at byte 13: syntax error while parsing BSON string: invalid string: ill-formed UTF-8 byte", json::parse_error&); + CHECK(json::from_bson(v, true, false).is_discarded()); + + // valid UTF-8 must still round-trip + const json j = {{"k", "h\xc3\xa9llo, w\xc3\xb6rld! \xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e"}}; // héllo, wörld! 日本語 + CHECK(json::from_bson(json::to_bson(j)) == j); + } + SECTION("objects") { SECTION("empty object") diff --git a/tests/src/unit-cbor.cpp b/tests/src/unit-cbor.cpp index 4c9107517..fc7fc614a 100644 --- a/tests/src/unit-cbor.cpp +++ b/tests/src/unit-cbor.cpp @@ -1833,6 +1833,27 @@ TEST_CASE("CBOR") CHECK(json::from_cbor(std::vector({0xa1, 0xff, 0x01}), true, false).is_discarded()); } + SECTION("invalid UTF-8 in string (see #5529)") + { + // a two-character text string (major type 3) whose bytes are not + // valid UTF-8 (0xC0 0xAE is an overlong encoding of '.') must be + // rejected at decode time, matching every other kind of + // malformed binary input, rather than only failing later when + // the resulting value is dumped + json _; + CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0x62, 0xc0, 0xae})), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR string: invalid string: ill-formed UTF-8 byte", json::parse_error&); + CHECK(json::from_cbor(std::vector({0x62, 0xc0, 0xae}), true, false).is_discarded()); + + // a CBOR byte string (major type 2) with the very same bytes is + // NOT text and must still be accepted as-is + CHECK_NOTHROW(_ = json::from_cbor(std::vector({0x42, 0xc0, 0xae}))); + CHECK(_ == json::binary(std::vector({0xc0, 0xae}))); + + // valid UTF-8 must still round-trip + const json j = "h\xc3\xa9llo, w\xc3\xb6rld! \xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e"; // héllo, wörld! 日本語 + CHECK(json::from_cbor(json::to_cbor(j)) == j); + } + SECTION("strict mode") { std::vector const vec = {0xf6, 0xf6}; diff --git a/tests/src/unit-msgpack.cpp b/tests/src/unit-msgpack.cpp index 74f7f4969..7dd3f3cf0 100644 --- a/tests/src/unit-msgpack.cpp +++ b/tests/src/unit-msgpack.cpp @@ -1554,6 +1554,27 @@ TEST_CASE("MessagePack") CHECK(json::from_msgpack(std::vector({0x81, 0xff, 0x01}), true, false).is_discarded()); } + SECTION("invalid UTF-8 in string (see #5529)") + { + // a fixstr of length 2 (0xA0 | 2) whose bytes are not valid UTF-8 + // (0xC0 0xAE is an overlong encoding of '.') must be rejected at + // decode time, matching every other kind of malformed binary + // input, rather than only failing later when the resulting + // value is dumped + json _; + CHECK_THROWS_WITH_AS(_ = json::from_msgpack(std::vector({0xa2, 0xc0, 0xae})), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing MessagePack string: invalid string: ill-formed UTF-8 byte", json::parse_error&); + CHECK(json::from_msgpack(std::vector({0xa2, 0xc0, 0xae}), true, false).is_discarded()); + + // a MessagePack bin8 blob with the very same bytes is NOT text + // and must still be accepted as-is + CHECK_NOTHROW(_ = json::from_msgpack(std::vector({0xc4, 0x02, 0xc0, 0xae}))); + CHECK(_ == json::binary(std::vector({0xc0, 0xae}))); + + // valid UTF-8 must still round-trip + const json j = "h\xc3\xa9llo, w\xc3\xb6rld! \xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e"; // héllo, wörld! 日本語 + CHECK(json::from_msgpack(json::to_msgpack(j)) == j); + } + SECTION("strict mode") { std::vector const vec = {0xc0, 0xc0}; diff --git a/tests/src/unit-ubjson.cpp b/tests/src/unit-ubjson.cpp index aafbbf5a4..a74734215 100644 --- a/tests/src/unit-ubjson.cpp +++ b/tests/src/unit-ubjson.cpp @@ -1927,6 +1927,23 @@ TEST_CASE("UBJSON") std::vector const v0 = {'S', 'i', 0}; CHECK(json::from_ubjson(v0) == json("")); } + + SECTION("invalid UTF-8 in string (see #5529)") + { + // a UBJSON string of length 2 whose bytes are not valid + // UTF-8 (0xC0 0xAE is an overlong encoding of '.') must be + // rejected at decode time, matching every other kind of + // malformed binary input, rather than only failing later + // when the resulting value is dumped + std::vector const v = {'S', 'i', 0x02, 0xc0, 0xae}; + json _; + CHECK_THROWS_WITH_AS(_ = json::from_ubjson(v), "[json.exception.parse_error.113] parse error at byte 5: syntax error while parsing UBJSON string: invalid string: ill-formed UTF-8 byte", json::parse_error&); + CHECK(json::from_ubjson(v, true, false).is_discarded()); + + // valid UTF-8 must still round-trip + const json j = "h\xc3\xa9llo, w\xc3\xb6rld! \xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e"; // héllo, wörld! 日本語 + CHECK(json::from_ubjson(json::to_ubjson(j)) == j); + } } SECTION("array")