From e564136c2299b3e47fd6154d8bf60692067c9d4c Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Tue, 22 Sep 2026 21:52:30 +0200 Subject: [PATCH 1/7] Make diff() account for member order in ordered_json objects (#5465) * Make diff() account for member order in ordered_json objects diff() compared source/target objects purely by key set, ignoring relative member order. For ordered_json (insertion-ordered, vector- backed object_t), two objects that differ only in member order are unequal via operator==, but diff() never emitted any patch operation to fix the order, so source.patch(diff(source, target)) == target could fail to hold. Fix by detecting when common keys appear in a different relative order in source vs. target (or when a new key would need to land somewhere other than the end), and in that case removing and re-adding the affected keys in target's order, which relies on patch()'s "add" op appending new keys at the end of an ordered_map. For plain json (std::map-backed, always key-sorted iteration) this is a no-op and the original minimal per-key diff path is unchanged. Signed-off-by: Niels Lohmann * Avoid redundant lookups in diff()'s object-order tracking The previous fix for ordered_json member order re-derived common-key order and suffix information with extra target.find()/source.find() calls layered on top of the pre-existing removed/added-key passes, instead of reusing those same passes. This roughly tripled the number of map lookups per diff() call for every object, including plain `json`, where the reordering path is never taken. Piggyback the order tracking (and the "add" op construction for new keys) onto the two passes the algorithm already needs to detect removed/added keys, and walk the fast path's recursion in lockstep with the precomputed common-key list instead of re-querying `target`. This restores diff() to its pre-existing lookup count; benchmarked at n=1000 keys, ordered_json::diff() was roughly 2x slower than baseline before this change and is back within noise of baseline after it. Signed-off-by: Niels Lohmann * Preserve diff()'s original op ordering and fix a slow-path deletion gap Splitting removed-key detection and common-key recursion into separate passes (for the earlier lookup-count fix) changed the emitted patch's op order: all "remove" ops now came before all recursive per-key diffs, instead of interleaved in source's iteration order as the original implementation did. This broke docs/mkdocs/docs/examples/diff.output's exact-match CI check (ci_test_examples) even though the patch was still semantically correct. Defer "remove" emission into the same walk that does the recursive diffs, so common keys and deleted keys are interleaved in source order again, matching historical output. While restructuring that walk, the reordering ("slow path") branch was only emitting "remove" for keys common to both objects, never for keys present in source but genuinely absent from target -- a key deleted alongside an actual reorder would silently survive the patch. Fixed by removing every source key in the slow path (both deleted and common keys need removing there; common keys are then re-added in target's order). Verified with a targeted reorder+deletion case and a fresh 20,000-case round-trip fuzz run (0 failures). Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- include/nlohmann/json.hpp | 133 +++++++++++++++++++++++++++---- single_include/nlohmann/json.hpp | 133 +++++++++++++++++++++++++++---- tests/src/unit-ordered_json.cpp | 81 +++++++++++++++++++ 3 files changed, 319 insertions(+), 28 deletions(-) diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 1aafbf78a..0294a268d 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -5332,34 +5332,139 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec case value_t::object: { - // first pass: traverse this object's elements + // first pass: record, for every source key, whether it is + // common to both objects (in source's iteration order) or + // was deleted (i.e., in source but not in target) -- this is + // a by-product of the target.find() call already needed to + // tell the two cases apart, so it adds no extra lookups. The + // "remove" ops themselves are emitted later, interleaved + // with the recursive per-key diffs in the fast path below, + // to match source's original iteration order (as the + // original, pre-reordering-aware implementation did) instead + // of grouping all removes before all recursive diffs. + std::vector common_keys_source_order; for (auto it = source.cbegin(); it != source.cend(); ++it) { - // escape the key name to be used in a JSON patch - const auto path_key = detail::concat(path, '/', detail::escape(it.key())); - if (target.find(it.key()) != target.end()) { - // recursive call to compare object values at key it - auto temp_diff = diff(it.value(), target[it.key()], path_key); - result.insert(result.end(), temp_diff.begin(), temp_diff.end()); + common_keys_source_order.push_back(it.key()); + } + } + + // second pass: find keys that were added (i.e., in target but + // not in source), and record the keys common to both, in + // target's iteration order -- again a by-product of the + // source.find() call already needed to detect added keys. At + // the same time, determine whether every added key comes + // after every common key in target's order (a precondition + // for the fast path below, which only ever appends new keys + // at the very end): for an object_t whose iteration order is + // a pure function of the key set (e.g. the default std::map, + // which always iterates in sorted key order), the order + // check further below is always true and this whole + // mechanism is effectively a no-op; it only matters for a + // reorderable object_t such as the one backing `ordered_json`. + // patch ops for keys that were added (i.e., in target but not + // in source); built here so the fast path below can reuse + // them without a second source.find() per target key. Only + // used by the fast path -- the slow (reordering) path + // rebuilds "add" ops for every key itself. + std::vector common_keys_target_order; + basic_json added_ops(value_t::array); + bool new_keys_form_suffix = true; + bool seen_new_key = false; + for (auto it = target.cbegin(); it != target.cend(); ++it) + { + if (source.find(it.key()) == source.end()) + { + seen_new_key = true; + const auto path_key = detail::concat(path, '/', detail::escape(it.key())); + added_ops.push_back( + { + {"op", "add"}, {"path", path_key}, + {"value", it.value()} + }); } else { - // found a key that is not in o -> remove it + common_keys_target_order.push_back(it.key()); + if (seen_new_key) + { + new_keys_form_suffix = false; + } + } + } + + if (common_keys_source_order == common_keys_target_order && new_keys_form_suffix) + { + // fast path: order of common keys already matches (or the + // object_t's iteration order does not depend on + // insertion history), so a plain per-key recursive diff + // is correct and minimal, as before. common_keys_source_order + // is, by construction, the subsequence of source's keys + // that are common to both objects, in source's iteration + // order -- so it can be walked in lockstep with `source` + // using a cheap key comparison instead of another lookup. + // Deleted keys (those source keys not in common_keys_source_order) + // are interleaved here too, in source's original order, to + // match the historical (pre-reordering-aware) output order. + auto common_it = common_keys_source_order.cbegin(); + for (auto it = source.cbegin(); it != source.cend(); ++it) + { + if (common_it != common_keys_source_order.cend() && it.key() == *common_it) + { + const auto path_key = detail::concat(path, '/', detail::escape(it.key())); + auto temp_diff = diff(it.value(), target[it.key()], path_key); + result.insert(result.end(), temp_diff.begin(), temp_diff.end()); + ++common_it; + } + else + { + // found a key that is not in target -> remove it + const auto path_key = detail::concat(path, '/', detail::escape(it.key())); + result.push_back(object( + { + {"op", "remove"}, {"path", path_key} + })); + } + } + + // append the "add" ops for brand-new keys collected above + // during the pass over target -- no second source.find() + // per target key needed + result.insert(result.end(), added_ops.begin(), added_ops.end()); + } + else + { + // slow path: the common keys are in a different relative + // order in source and target (only possible for a + // reorderable object_t like ordered_map). Building a + // minimal reordering patch is a nontrivial (LCS-like) + // problem; instead, remove every source key -- both + // deleted keys (which must be removed regardless) and + // common keys (removed so they can be re-added in + // target's order) -- and re-add every key that should + // remain, with its final target value, in target's + // order. basic_json::patch()'s "add" operation on an + // object uses operator[], which appends at the end for a + // vector-backed insertion-ordered map when the key does + // not already exist -- so removing a key and then adding + // it moves it to the end, fixing its position. + for (auto it = source.cbegin(); it != source.cend(); ++it) + { + const auto path_key = detail::concat(path, '/', detail::escape(it.key())); result.push_back(object( { {"op", "remove"}, {"path", path_key} })); } - } - // second pass: traverse other object's elements - for (auto it = target.cbegin(); it != target.cend(); ++it) - { - if (source.find(it.key()) == source.end()) + // add every key that is either common (just removed + // above) or brand new, in target's iteration order, so + // that the final order after applying the patch matches + // target exactly + for (auto it = target.cbegin(); it != target.cend(); ++it) { - // found a key that is not in this -> add it const auto path_key = detail::concat(path, '/', detail::escape(it.key())); result.push_back( { diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 213236b51..7222dbb93 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -29257,34 +29257,139 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec case value_t::object: { - // first pass: traverse this object's elements + // first pass: record, for every source key, whether it is + // common to both objects (in source's iteration order) or + // was deleted (i.e., in source but not in target) -- this is + // a by-product of the target.find() call already needed to + // tell the two cases apart, so it adds no extra lookups. The + // "remove" ops themselves are emitted later, interleaved + // with the recursive per-key diffs in the fast path below, + // to match source's original iteration order (as the + // original, pre-reordering-aware implementation did) instead + // of grouping all removes before all recursive diffs. + std::vector common_keys_source_order; for (auto it = source.cbegin(); it != source.cend(); ++it) { - // escape the key name to be used in a JSON patch - const auto path_key = detail::concat(path, '/', detail::escape(it.key())); - if (target.find(it.key()) != target.end()) { - // recursive call to compare object values at key it - auto temp_diff = diff(it.value(), target[it.key()], path_key); - result.insert(result.end(), temp_diff.begin(), temp_diff.end()); + common_keys_source_order.push_back(it.key()); + } + } + + // second pass: find keys that were added (i.e., in target but + // not in source), and record the keys common to both, in + // target's iteration order -- again a by-product of the + // source.find() call already needed to detect added keys. At + // the same time, determine whether every added key comes + // after every common key in target's order (a precondition + // for the fast path below, which only ever appends new keys + // at the very end): for an object_t whose iteration order is + // a pure function of the key set (e.g. the default std::map, + // which always iterates in sorted key order), the order + // check further below is always true and this whole + // mechanism is effectively a no-op; it only matters for a + // reorderable object_t such as the one backing `ordered_json`. + // patch ops for keys that were added (i.e., in target but not + // in source); built here so the fast path below can reuse + // them without a second source.find() per target key. Only + // used by the fast path -- the slow (reordering) path + // rebuilds "add" ops for every key itself. + std::vector common_keys_target_order; + basic_json added_ops(value_t::array); + bool new_keys_form_suffix = true; + bool seen_new_key = false; + for (auto it = target.cbegin(); it != target.cend(); ++it) + { + if (source.find(it.key()) == source.end()) + { + seen_new_key = true; + const auto path_key = detail::concat(path, '/', detail::escape(it.key())); + added_ops.push_back( + { + {"op", "add"}, {"path", path_key}, + {"value", it.value()} + }); } else { - // found a key that is not in o -> remove it + common_keys_target_order.push_back(it.key()); + if (seen_new_key) + { + new_keys_form_suffix = false; + } + } + } + + if (common_keys_source_order == common_keys_target_order && new_keys_form_suffix) + { + // fast path: order of common keys already matches (or the + // object_t's iteration order does not depend on + // insertion history), so a plain per-key recursive diff + // is correct and minimal, as before. common_keys_source_order + // is, by construction, the subsequence of source's keys + // that are common to both objects, in source's iteration + // order -- so it can be walked in lockstep with `source` + // using a cheap key comparison instead of another lookup. + // Deleted keys (those source keys not in common_keys_source_order) + // are interleaved here too, in source's original order, to + // match the historical (pre-reordering-aware) output order. + auto common_it = common_keys_source_order.cbegin(); + for (auto it = source.cbegin(); it != source.cend(); ++it) + { + if (common_it != common_keys_source_order.cend() && it.key() == *common_it) + { + const auto path_key = detail::concat(path, '/', detail::escape(it.key())); + auto temp_diff = diff(it.value(), target[it.key()], path_key); + result.insert(result.end(), temp_diff.begin(), temp_diff.end()); + ++common_it; + } + else + { + // found a key that is not in target -> remove it + const auto path_key = detail::concat(path, '/', detail::escape(it.key())); + result.push_back(object( + { + {"op", "remove"}, {"path", path_key} + })); + } + } + + // append the "add" ops for brand-new keys collected above + // during the pass over target -- no second source.find() + // per target key needed + result.insert(result.end(), added_ops.begin(), added_ops.end()); + } + else + { + // slow path: the common keys are in a different relative + // order in source and target (only possible for a + // reorderable object_t like ordered_map). Building a + // minimal reordering patch is a nontrivial (LCS-like) + // problem; instead, remove every source key -- both + // deleted keys (which must be removed regardless) and + // common keys (removed so they can be re-added in + // target's order) -- and re-add every key that should + // remain, with its final target value, in target's + // order. basic_json::patch()'s "add" operation on an + // object uses operator[], which appends at the end for a + // vector-backed insertion-ordered map when the key does + // not already exist -- so removing a key and then adding + // it moves it to the end, fixing its position. + for (auto it = source.cbegin(); it != source.cend(); ++it) + { + const auto path_key = detail::concat(path, '/', detail::escape(it.key())); result.push_back(object( { {"op", "remove"}, {"path", path_key} })); } - } - // second pass: traverse other object's elements - for (auto it = target.cbegin(); it != target.cend(); ++it) - { - if (source.find(it.key()) == source.end()) + // add every key that is either common (just removed + // above) or brand new, in target's iteration order, so + // that the final order after applying the patch matches + // target exactly + for (auto it = target.cbegin(); it != target.cend(); ++it) { - // found a key that is not in this -> add it const auto path_key = detail::concat(path, '/', detail::escape(it.key())); result.push_back( { diff --git a/tests/src/unit-ordered_json.cpp b/tests/src/unit-ordered_json.cpp index a38a1a2b8..62a949a7f 100644 --- a/tests/src/unit-ordered_json.cpp +++ b/tests/src/unit-ordered_json.cpp @@ -81,3 +81,84 @@ TEST_CASE("regression test for issue #3732 - iteration_proxy_value(fn); } + +TEST_CASE("regression test - diff() must account for ordered_json member order") +{ + SECTION("pure reorder, no value changes") + { + ordered_json a = {{"a", 1}, {"b", 2}}; + ordered_json b = {{"b", 2}, {"a", 1}}; + CHECK(a != b); // order-sensitive equality + CHECK(a.patch(ordered_json::diff(a, b)) == b); + } + + SECTION("new key must land at the front") + { + ordered_json c = {{"b", 2}}; + ordered_json e = {{"a", 1}, {"b", 2}}; + CHECK(c.patch(ordered_json::diff(c, e)) == e); + } + + SECTION("reorder plus a value change on one of the reordered keys") + { + ordered_json a = {{"a", 1}, {"b", 2}}; + ordered_json b = {{"b", 20}, {"a", 1}}; + CHECK(a != b); + CHECK(a.patch(ordered_json::diff(a, b)) == b); + } + + SECTION("reorder plus a deleted key") + { + ordered_json a = {{"a", 1}, {"b", 2}, {"c", 3}}; + ordered_json b = {{"b", 2}, {"a", 1}}; + CHECK(a != b); + CHECK(a.patch(ordered_json::diff(a, b)) == b); + } + + SECTION("reorder plus a nested value that itself needs a recursive diff") + { + ordered_json a = {{"a", {{"x", 1}, {"y", 2}}}, {"b", 2}}; + ordered_json b = {{"b", 2}, {"a", {{"x", 1}, {"y", 99}}}}; + CHECK(a != b); + CHECK(a.patch(ordered_json::diff(a, b)) == b); + } + + SECTION("three or more keys shuffled into a different order") + { + ordered_json a = {{"a", 1}, {"b", 2}, {"c", 3}, {"d", 4}}; + ordered_json b = {{"d", 4}, {"b", 2}, {"a", 1}, {"c", 3}}; + CHECK(a != b); + CHECK(a.patch(ordered_json::diff(a, b)) == b); + } + + SECTION("matching order still produces a minimal patch (fast path unaffected)") + { + ordered_json a = {{"a", 1}, {"b", 2}, {"c", 3}}; + ordered_json b = {{"a", 1}, {"b", 20}, {"c", 3}}; + auto p = ordered_json::diff(a, b); + // only the changed value should be touched, not a wholesale remove+add + CHECK(p.size() == 1); + CHECK(p[0]["op"] == "replace"); + CHECK(p[0]["path"] == "/b"); + CHECK(a.patch(p) == b); + } + + SECTION("plain json (std::map-backed) is unaffected by same-key-different-insertion-order") + { + json a; + a["b"] = 2; + a["a"] = 1; + + json b; + b["a"] = 1; + b["b"] = 2; + + // std::map iteration is always sorted by key, so a == b regardless of + // insertion order, and diff() must still produce the same minimal + // (empty) result as before this fix + CHECK(a == b); + auto p = json::diff(a, b); + CHECK(p.empty()); + CHECK(a.patch(p) == b); + } +} From e485441123b8510aabd61cf63e1f4672e49004c1 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Tue, 22 Sep 2026 21:52:30 +0200 Subject: [PATCH 2/7] Restore a duplicate key's prior value when the callback rejects its new value (#5466) * Restore a duplicate key's prior value when the callback rejects its new value json_sax_dom_callback_parser::key() unconditionally overwrote the object slot for a key with a `discarded` placeholder as soon as the key was accepted by the parser callback. For a duplicate key (legal JSON), this destroyed the pre-existing value from an earlier occurrence of the same key before the new value was even parsed. If the new value was then rejected by the callback, remove_discarded_value() erased the member entirely instead of leaving the original value in place, contradicting the documented behavior that a discarded value behaves as if it was never read. Add a small stash of (slot pointer, previous value) pairs so that when key() overwrites an existing member with the discarded placeholder, the previous value can be restored later if the corresponding value (scalar, object, or array) is rejected, instead of being erased. The stash entry is dropped without restoring once the new value is definitively accepted (in handle_value() for scalars, end_object()/end_array() for containers), so a duplicate key whose new value is accepted still keeps the last value as before. Non-duplicate keys are unaffected: rejecting their value still removes the member entirely, since there is nothing to restore. Signed-off-by: Niels Lohmann * Mark parser-callback test lambdas noexcept to fix GCC -Wnoexcept -Werror GCC's libstdc++ std::function move assignment evaluates a noexcept check that invokes a wrapped callable in an unevaluated context; a non-noexcept parser_callback_t lambda then trips -Wnoexcept ("noexcept- expression evaluates to 'false'"), which CI's ci_test_gcc job builds with -Werror. The pre-existing parser_callback_t test lambdas in this file already work around this by declaring themselves noexcept; apply the same fix to the three added lambdas that didn't. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- include/nlohmann/detail/input/json_sax.hpp | 108 ++++++++++++++++++--- single_include/nlohmann/json.hpp | 108 ++++++++++++++++++--- tests/src/unit-regression2.cpp | 71 ++++++++++++++ 3 files changed, 259 insertions(+), 28 deletions(-) diff --git a/include/nlohmann/detail/input/json_sax.hpp b/include/nlohmann/detail/input/json_sax.hpp index 37d0ab270..f2c1584b4 100644 --- a/include/nlohmann/detail/input/json_sax.hpp +++ b/include/nlohmann/detail/input/json_sax.hpp @@ -8,11 +8,11 @@ #pragma once -#include // min +#include // find_if, min #include #include // string #include // enable_if_t -#include // move +#include // move, pair #include // vector #include @@ -631,7 +631,17 @@ class json_sax_dom_callback_parser // add discarded value at the given key and store the reference for later if (keep && ref_stack.back()) { - object_element = &(ref_stack.back()->m_data.m_value.object->operator[](val) = discarded); + auto& obj = *ref_stack.back()->m_data.m_value.object; + const auto it = obj.find(val); + if (it != obj.end()) + { + // this is a duplicate key (legal in JSON); remember its + // current value so it can be restored later if the new + // value is rejected by the callback, instead of being + // erased together with the discarded placeholder + duplicate_key_stash.emplace_back(&(it->second), it->second); + } + object_element = &(obj[val] = discarded); } return true; @@ -643,13 +653,18 @@ class json_sax_dom_callback_parser { if (!callback(static_cast(ref_stack.size()) - 1, parse_event_t::object_end, *ref_stack.back())) { - // discard object - *ref_stack.back() = discarded; + // discard object, unless this slot holds a duplicate key's + // previous value pending restoration, in which case that + // value is restored instead of being discarded + if (!resolve_duplicate_key_stash(ref_stack.back(), true)) + { + *ref_stack.back() = discarded; #if JSON_DIAGNOSTIC_POSITIONS - // Set start/end positions for discarded object. - handle_diagnostic_positions_for_json_value(*ref_stack.back()); + // Set start/end positions for discarded object. + handle_diagnostic_positions_for_json_value(*ref_stack.back()); #endif + } } else { @@ -663,6 +678,10 @@ class json_sax_dom_callback_parser #endif ref_stack.back()->set_parents(); + // this object is finally, definitively kept; drop any + // pending duplicate-key stash entry for its slot since it + // can no longer be restored + resolve_duplicate_key_stash(ref_stack.back(), false); } } @@ -743,16 +762,25 @@ class json_sax_dom_callback_parser #endif ref_stack.back()->set_parents(); + // this array is finally, definitively kept; drop any + // pending duplicate-key stash entry for its slot since it + // can no longer be restored + resolve_duplicate_key_stash(ref_stack.back(), false); } else { - // discard array - *ref_stack.back() = discarded; + // discard array, unless this slot holds a duplicate key's + // previous value pending restoration, in which case that + // value is restored instead of being discarded + if (!resolve_duplicate_key_stash(ref_stack.back(), true)) + { + *ref_stack.back() = discarded; #if JSON_DIAGNOSTIC_POSITIONS - // Set start/end positions for discarded array. - handle_diagnostic_positions_for_json_value(*ref_stack.back()); + // Set start/end positions for discarded array. + handle_diagnostic_positions_for_json_value(*ref_stack.back()); #endif + } } } @@ -869,6 +897,35 @@ class json_sax_dom_callback_parser } #endif + /// if there is a pending duplicate-key stash entry for this exact slot, + /// remove it from the stash; if restore_value is true, the stashed + /// previous value is moved back into the slot first (use this when the + /// new value at that slot was rejected); otherwise the stash entry is + /// simply dropped (use this when the new value was accepted, so it + /// correctly supersedes the old one and no restore should ever happen + /// for this slot again) + /// @return whether a matching stash entry was found (and processed) + bool resolve_duplicate_key_stash(BasicJsonType* slot, bool restore_value) + { + const auto it = std::find_if(duplicate_key_stash.begin(), duplicate_key_stash.end(), + [slot](const std::pair& entry) + { + return entry.first == slot; + }); + + if (it == duplicate_key_stash.end()) + { + return false; + } + + if (restore_value) + { + *slot = std::move(it->second); + } + duplicate_key_stash.erase(it); + return true; + } + /*! @brief the key the value now being handled will be stored under @@ -887,7 +944,9 @@ class json_sax_dom_callback_parser } /*! - @brief remove the discarded value the callback rejected from its parent + @brief remove the discarded value the callback rejected from its parent, + unless it is a duplicate key's slot with a stashed previous value, in + which case that previous value is restored instead A rejected value can only ever be the one most recently added to @a parent: the last element of an array, or the placeholder key() stored under @a key @@ -902,7 +961,7 @@ class json_sax_dom_callback_parser @param[in,out] parent the container to remove the rejected value from @param[in] key the key the value was stored under; unused for arrays */ - static void remove_discarded_value(BasicJsonType& parent, const string_t& key) + void remove_discarded_value(BasicJsonType& parent, const string_t& key) { if (parent.is_array()) { @@ -918,7 +977,12 @@ class json_sax_dom_callback_parser const auto it = object.find(key); if (it != object.end() && it->second.is_discarded()) { - object.erase(it); + // a duplicate key's slot has a stashed previous value that + // must be restored instead of being erased + if (!resolve_duplicate_key_stash(&it->second, true)) + { + object.erase(it); + } } } } @@ -1020,6 +1084,16 @@ class json_sax_dom_callback_parser JSON_ASSERT(object_element); *object_element = std::move(value); + if (!skip_callback) + { + // this scalar value finally, definitively replaces whatever was + // at this slot; drop any pending duplicate-key stash entry for + // it since it can no longer be restored (a container value at + // this slot is resolved later, in end_object()/end_array(), + // since skip_callback is true for the placeholder handling that + // happens here for those) + resolve_duplicate_key_stash(object_element, false); + } return {true, object_element}; } @@ -1039,6 +1113,12 @@ class json_sax_dom_callback_parser std::vector container_key_stack {}; // NOLINT(readability-redundant-member-init) /// helper to hold the reference for the next object element BasicJsonType* object_element = nullptr; + /// stash of (slot pointer, previous value) for object members that + /// already existed when key() was called again for the same key + /// (duplicate keys); used to restore the previous value if the new + /// value is later rejected by the callback, instead of erasing the + /// member entirely + std::vector> duplicate_key_stash {}; /// whether a syntax error occurred bool errored = false; /// callback function diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 7222dbb93..291a44f22 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -7951,11 +7951,11 @@ NLOHMANN_JSON_NAMESPACE_END -#include // min +#include // find_if, min #include #include // string #include // enable_if_t -#include // move +#include // move, pair #include // vector // #include @@ -11404,7 +11404,17 @@ class json_sax_dom_callback_parser // add discarded value at the given key and store the reference for later if (keep && ref_stack.back()) { - object_element = &(ref_stack.back()->m_data.m_value.object->operator[](val) = discarded); + auto& obj = *ref_stack.back()->m_data.m_value.object; + const auto it = obj.find(val); + if (it != obj.end()) + { + // this is a duplicate key (legal in JSON); remember its + // current value so it can be restored later if the new + // value is rejected by the callback, instead of being + // erased together with the discarded placeholder + duplicate_key_stash.emplace_back(&(it->second), it->second); + } + object_element = &(obj[val] = discarded); } return true; @@ -11416,13 +11426,18 @@ class json_sax_dom_callback_parser { if (!callback(static_cast(ref_stack.size()) - 1, parse_event_t::object_end, *ref_stack.back())) { - // discard object - *ref_stack.back() = discarded; + // discard object, unless this slot holds a duplicate key's + // previous value pending restoration, in which case that + // value is restored instead of being discarded + if (!resolve_duplicate_key_stash(ref_stack.back(), true)) + { + *ref_stack.back() = discarded; #if JSON_DIAGNOSTIC_POSITIONS - // Set start/end positions for discarded object. - handle_diagnostic_positions_for_json_value(*ref_stack.back()); + // Set start/end positions for discarded object. + handle_diagnostic_positions_for_json_value(*ref_stack.back()); #endif + } } else { @@ -11436,6 +11451,10 @@ class json_sax_dom_callback_parser #endif ref_stack.back()->set_parents(); + // this object is finally, definitively kept; drop any + // pending duplicate-key stash entry for its slot since it + // can no longer be restored + resolve_duplicate_key_stash(ref_stack.back(), false); } } @@ -11516,16 +11535,25 @@ class json_sax_dom_callback_parser #endif ref_stack.back()->set_parents(); + // this array is finally, definitively kept; drop any + // pending duplicate-key stash entry for its slot since it + // can no longer be restored + resolve_duplicate_key_stash(ref_stack.back(), false); } else { - // discard array - *ref_stack.back() = discarded; + // discard array, unless this slot holds a duplicate key's + // previous value pending restoration, in which case that + // value is restored instead of being discarded + if (!resolve_duplicate_key_stash(ref_stack.back(), true)) + { + *ref_stack.back() = discarded; #if JSON_DIAGNOSTIC_POSITIONS - // Set start/end positions for discarded array. - handle_diagnostic_positions_for_json_value(*ref_stack.back()); + // Set start/end positions for discarded array. + handle_diagnostic_positions_for_json_value(*ref_stack.back()); #endif + } } } @@ -11642,6 +11670,35 @@ class json_sax_dom_callback_parser } #endif + /// if there is a pending duplicate-key stash entry for this exact slot, + /// remove it from the stash; if restore_value is true, the stashed + /// previous value is moved back into the slot first (use this when the + /// new value at that slot was rejected); otherwise the stash entry is + /// simply dropped (use this when the new value was accepted, so it + /// correctly supersedes the old one and no restore should ever happen + /// for this slot again) + /// @return whether a matching stash entry was found (and processed) + bool resolve_duplicate_key_stash(BasicJsonType* slot, bool restore_value) + { + const auto it = std::find_if(duplicate_key_stash.begin(), duplicate_key_stash.end(), + [slot](const std::pair& entry) + { + return entry.first == slot; + }); + + if (it == duplicate_key_stash.end()) + { + return false; + } + + if (restore_value) + { + *slot = std::move(it->second); + } + duplicate_key_stash.erase(it); + return true; + } + /*! @brief the key the value now being handled will be stored under @@ -11660,7 +11717,9 @@ class json_sax_dom_callback_parser } /*! - @brief remove the discarded value the callback rejected from its parent + @brief remove the discarded value the callback rejected from its parent, + unless it is a duplicate key's slot with a stashed previous value, in + which case that previous value is restored instead A rejected value can only ever be the one most recently added to @a parent: the last element of an array, or the placeholder key() stored under @a key @@ -11675,7 +11734,7 @@ class json_sax_dom_callback_parser @param[in,out] parent the container to remove the rejected value from @param[in] key the key the value was stored under; unused for arrays */ - static void remove_discarded_value(BasicJsonType& parent, const string_t& key) + void remove_discarded_value(BasicJsonType& parent, const string_t& key) { if (parent.is_array()) { @@ -11691,7 +11750,12 @@ class json_sax_dom_callback_parser const auto it = object.find(key); if (it != object.end() && it->second.is_discarded()) { - object.erase(it); + // a duplicate key's slot has a stashed previous value that + // must be restored instead of being erased + if (!resolve_duplicate_key_stash(&it->second, true)) + { + object.erase(it); + } } } } @@ -11793,6 +11857,16 @@ class json_sax_dom_callback_parser JSON_ASSERT(object_element); *object_element = std::move(value); + if (!skip_callback) + { + // this scalar value finally, definitively replaces whatever was + // at this slot; drop any pending duplicate-key stash entry for + // it since it can no longer be restored (a container value at + // this slot is resolved later, in end_object()/end_array(), + // since skip_callback is true for the placeholder handling that + // happens here for those) + resolve_duplicate_key_stash(object_element, false); + } return {true, object_element}; } @@ -11812,6 +11886,12 @@ class json_sax_dom_callback_parser std::vector container_key_stack {}; // NOLINT(readability-redundant-member-init) /// helper to hold the reference for the next object element BasicJsonType* object_element = nullptr; + /// stash of (slot pointer, previous value) for object members that + /// already existed when key() was called again for the same key + /// (duplicate keys); used to restore the previous value if the new + /// value is later rejected by the callback, instead of erasing the + /// member entirely + std::vector> duplicate_key_stash {}; /// whether a syntax error occurred bool errored = false; /// callback function diff --git a/tests/src/unit-regression2.cpp b/tests/src/unit-regression2.cpp index 4280ec361..2ae5666ff 100644 --- a/tests/src/unit-regression2.cpp +++ b/tests/src/unit-regression2.cpp @@ -763,4 +763,75 @@ TEST_CASE("regression tests 2") } +TEST_CASE("regression test - parser callback must not lose a duplicate key's prior value") +{ + // a callback that rejects only the scalar value 2 + const json::parser_callback_t drop_value_2 = [](int /*depth*/, json::parse_event_t ev, json & v) noexcept + { + return !(ev == json::parse_event_t::value && v == 2); + }; + + SECTION("duplicate key, second (scalar) value rejected - prior value is restored") + { + const json j = json::parse(R"({"a":1,"a":2})", drop_value_2); + CHECK(j.dump() == "{\"a\":1}"); + } + + SECTION("duplicate key, second value is an object rejected at object_end - prior value is restored") + { + const json j = json::parse(R"({"a":1,"a":{"x":2}})", + [](int depth, json::parse_event_t ev, json& /*parsed*/) noexcept + { + return !(ev == json::parse_event_t::object_end && depth == 1); + }); + CHECK(j.dump() == "{\"a\":1}"); + } + + SECTION("duplicate key, second value is an array rejected at array_end - prior value is restored") + { + const json j = json::parse(R"({"a":1,"a":[9,9]})", + [](int depth, json::parse_event_t ev, json& /*parsed*/) noexcept + { + return !(ev == json::parse_event_t::array_end && depth == 1); + }); + CHECK(j.dump() == "{\"a\":1}"); + } + + SECTION("duplicate key, second value accepted (scalar) - last value wins") + { + const json j = json::parse(R"({"a":1,"a":2})", [](int, json::parse_event_t, json&) noexcept + { + return true; + }); + CHECK(j.dump() == "{\"a\":2}"); + } + + SECTION("duplicate key, second value accepted (object) - last value wins") + { + const json j = json::parse(R"({"a":1,"a":{"x":2}})", [](int, json::parse_event_t, json&) noexcept + { + return true; + }); + CHECK(j.dump() == "{\"a\":{\"x\":2}}"); + } + + SECTION("brand new (non-duplicate) key, value rejected - member is fully absent") + { + const json j = json::parse(R"({"a":1,"b":2})", drop_value_2); + CHECK(j.dump() == "{\"a\":1}"); + } + + SECTION("duplicate key nested two levels deep") + { + const json j = json::parse(R"({"outer":{"a":1,"a":2}})", drop_value_2); + CHECK(j.dump() == "{\"outer\":{\"a\":1}}"); + } + + SECTION("three occurrences of the same key - middle rejected, last accepted") + { + const json j = json::parse(R"({"k":1,"k":2,"k":3})", drop_value_2); + CHECK(j.dump() == "{\"k\":3}"); + } +} + DOCTEST_CLANG_SUPPRESS_WARNING_POP From a2b19d6158d4346bb3b02b4351d21cf7584debba Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Tue, 22 Sep 2026 21:52:31 +0200 Subject: [PATCH 3/7] Honor allow_exceptions=false for excessive array/object size (out_of_range.408) (#5467) * Honor allow_exceptions=false for excessive array/object size (out_of_range.408) The SAX DOM parsers' start_object()/start_array() threw out_of_range.408 directly via JSON_THROW when a binary format (CBOR/UBJSON/BJData) declared a container size exceeding max_size(), bypassing the allow_exceptions flag that every other malformed-input error path in these classes honors via parse_error(). This meant that json::from_cbor(data, true, false) etc. could still throw (or abort under JSON_NOEXCEPTION) instead of returning a discarded value, contrary to the allow_exceptions=false contract. Route all four call sites (two in json_sax_dom_parser, two in json_sax_dom_callback_parser) through parse_error() instead, matching the existing error-handling pattern used elsewhere in this file. Behavior is unchanged when allow_exceptions is true (the default); the exception message and type are identical. Signed-off-by: Niels Lohmann * Drop a non-portable exact exception message check in the 408 test The allow_exceptions=false regression test checked the exact message text produced when allow_exceptions=true (the default). On platforms where std::size_t is 32-bit (e.g. mingw x86, MSVC Win32 builds), a declared CBOR length of 2^63 is intercepted earlier, by get_cbor_container_size()'s own (pre-existing, already correct) length-narrowing check, with different wording than this fix's start_array()/start_object() size check -- same error code, same "still throws when allow_exceptions=true" guarantee, different text. CHECK_THROWS_AS already verifies the behavior this test cares about (still throws json::out_of_range, unchanged); drop the exact-message assertion since it isn't portable across size_t widths and doesn't add coverage of this fix specifically. Signed-off-by: Niels Lohmann * Fix -Werror=unused-result on json::from_cbor() in the 408 regression test from_cbor() is [[nodiscard]]; CHECK_THROWS_AS() otherwise discards its result, which GCC flags under -Werror. Assign to a throwaway json, as the rest of the suite already does for from_cbor()/from_msgpack(). Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- include/nlohmann/detail/input/json_sax.hpp | 8 +++--- single_include/nlohmann/json.hpp | 8 +++--- tests/src/unit-regression2.cpp | 31 ++++++++++++++++++++++ 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/include/nlohmann/detail/input/json_sax.hpp b/include/nlohmann/detail/input/json_sax.hpp index f2c1584b4..962913610 100644 --- a/include/nlohmann/detail/input/json_sax.hpp +++ b/include/nlohmann/detail/input/json_sax.hpp @@ -278,7 +278,7 @@ class json_sax_dom_parser if (JSON_HEDLEY_UNLIKELY(len != detail::unknown_size() && len > ref_stack.back()->max_size())) { - JSON_THROW(out_of_range::create(408, concat("excessive object size: ", std::to_string(len)), ref_stack.back())); + return parse_error(0, "", out_of_range::create(408, concat("excessive object size: ", std::to_string(len)), ref_stack.back())); } return true; @@ -327,7 +327,7 @@ class json_sax_dom_parser if (JSON_HEDLEY_UNLIKELY(len != detail::unknown_size() && len > ref_stack.back()->max_size())) { - JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back())); + return parse_error(0, "", out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back())); } if (len != detail::unknown_size()) @@ -611,7 +611,7 @@ class json_sax_dom_callback_parser // check object limit if (JSON_HEDLEY_UNLIKELY(len != detail::unknown_size() && len > ref_stack.back()->max_size())) { - JSON_THROW(out_of_range::create(408, concat("excessive object size: ", std::to_string(len)), ref_stack.back())); + return parse_error(0, "", out_of_range::create(408, concat("excessive object size: ", std::to_string(len)), ref_stack.back())); } } return true; @@ -730,7 +730,7 @@ class json_sax_dom_callback_parser // check array limit if (JSON_HEDLEY_UNLIKELY(len != detail::unknown_size() && len > ref_stack.back()->max_size())) { - JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back())); + return parse_error(0, "", out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back())); } if (len != detail::unknown_size()) diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 291a44f22..9b61ac0db 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -11051,7 +11051,7 @@ class json_sax_dom_parser if (JSON_HEDLEY_UNLIKELY(len != detail::unknown_size() && len > ref_stack.back()->max_size())) { - JSON_THROW(out_of_range::create(408, concat("excessive object size: ", std::to_string(len)), ref_stack.back())); + return parse_error(0, "", out_of_range::create(408, concat("excessive object size: ", std::to_string(len)), ref_stack.back())); } return true; @@ -11100,7 +11100,7 @@ class json_sax_dom_parser if (JSON_HEDLEY_UNLIKELY(len != detail::unknown_size() && len > ref_stack.back()->max_size())) { - JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back())); + return parse_error(0, "", out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back())); } if (len != detail::unknown_size()) @@ -11384,7 +11384,7 @@ class json_sax_dom_callback_parser // check object limit if (JSON_HEDLEY_UNLIKELY(len != detail::unknown_size() && len > ref_stack.back()->max_size())) { - JSON_THROW(out_of_range::create(408, concat("excessive object size: ", std::to_string(len)), ref_stack.back())); + return parse_error(0, "", out_of_range::create(408, concat("excessive object size: ", std::to_string(len)), ref_stack.back())); } } return true; @@ -11503,7 +11503,7 @@ class json_sax_dom_callback_parser // check array limit if (JSON_HEDLEY_UNLIKELY(len != detail::unknown_size() && len > ref_stack.back()->max_size())) { - JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back())); + return parse_error(0, "", out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back())); } if (len != detail::unknown_size()) diff --git a/tests/src/unit-regression2.cpp b/tests/src/unit-regression2.cpp index 2ae5666ff..2c0cf6549 100644 --- a/tests/src/unit-regression2.cpp +++ b/tests/src/unit-regression2.cpp @@ -834,4 +834,35 @@ TEST_CASE("regression test - parser callback must not lose a duplicate key's pri } } +TEST_CASE("regression test - excessive binary container size honors allow_exceptions=false") +{ + // CBOR array with declared length 2^63 + const std::vector cbor = {0x9b, 0x80, 0, 0, 0, 0, 0, 0, 0}; + // CBOR map with declared length 2^63 + const std::vector cbor_m = {0xbb, 0x80, 0, 0, 0, 0, 0, 0, 0}; + // UBJSON array with declared length 2^63-1 + const std::vector ubj = {'[', '#', 'L', 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; + // BJData array with declared length 2^63-1 (little endian) + const std::vector bjd = {'[', '#', 'L', 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f}; + + // allow_exceptions=false must report failure instead of throwing/aborting + CHECK(json::from_cbor(cbor, true, false).is_discarded()); + CHECK(json::from_cbor(cbor_m, true, false).is_discarded()); + CHECK(json::from_ubjson(ubj, true, false).is_discarded()); + CHECK(json::from_bjdata(bjd, true, false).is_discarded()); + + // allow_exceptions=true (the default) must still throw exactly as before. + // The exact message text is not checked here: on platforms where + // std::size_t is 32-bit, the CBOR reader's own length-narrowing check + // (get_cbor_container_size(), unrelated to this fix) intercepts a + // declared length of 2^63 before it ever reaches the check this test + // targets, with different (but equally valid, and already correct) + // wording -- see unit-cbor.cpp for coverage of that message. + json _; + CHECK_THROWS_AS(_ = json::from_cbor(cbor), json::out_of_range); + + // regression guard: a genuinely truncated CBOR input must remain discarded + CHECK(json::from_cbor(std::vector {0x9b, 0, 0, 0, 0, 0, 0, 0, 0x02}, true, false).is_discarded()); +} + DOCTEST_CLANG_SUPPRESS_WARNING_POP From d2c1a6a272d28965a066b9a19bbf7662a55b54d1 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Tue, 22 Sep 2026 21:52:31 +0200 Subject: [PATCH 4/7] Reject array insert(pos, first, last) iterators not pointing into an array (#5468) The array-range insert() overload checked that pos fits the current value and that first/last share the same owning value, but never verified that value is itself an array. Passing iterators from an object, a primitive, or null handed value-initialized (singular) std::vector iterators straight to array_t::insert(), which is undefined behavior. Add the missing is_array() check, mirroring the equivalent check already present in the object-range insert() overload. Signed-off-by: Niels Lohmann --- docs/mkdocs/docs/api/basic_json/insert.md | 2 ++ include/nlohmann/json.hpp | 6 ++++++ single_include/nlohmann/json.hpp | 6 ++++++ tests/src/unit-modifiers.cpp | 14 ++++++++++++++ 4 files changed, 28 insertions(+) diff --git a/docs/mkdocs/docs/api/basic_json/insert.md b/docs/mkdocs/docs/api/basic_json/insert.md index 14d5823c1..fcb1e6e44 100644 --- a/docs/mkdocs/docs/api/basic_json/insert.md +++ b/docs/mkdocs/docs/api/basic_json/insert.md @@ -88,6 +88,8 @@ Strong exception safety: if an exception occurs, the original value stays intact do not belong to the same JSON value; example: `"iterators do not fit"` - Throws [`invalid_iterator.211`](../../home/exceptions.md#jsonexceptioninvalid_iterator211) if `first` or `last` are iterators into container for which insert is called; example: `"passed iterators may not belong to container"` + - Throws [`invalid_iterator.202`](../../home/exceptions.md#jsonexceptioninvalid_iterator202) if `first` or `last` + do not point to an array; example: `"iterators first and last must point to arrays"` 4. The function can throw the following exceptions: - Throws [`type_error.309`](../../home/exceptions.md#jsonexceptiontype_error309) if called on JSON values other than arrays; example: `"cannot use insert() with string"` diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 0294a268d..d55efacc2 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -3511,6 +3511,12 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_THROW(invalid_iterator::create(211, "passed iterators may not belong to container", this)); } + // passed iterators must belong to arrays + if (JSON_HEDLEY_UNLIKELY(!first.m_object->is_array())) + { + JSON_THROW(invalid_iterator::create(202, "iterators first and last must point to arrays", this)); + } + // insert to array and return iterator return insert_iterator(pos, first.m_it.array_iterator, last.m_it.array_iterator); } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 9b61ac0db..2c73944c0 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -27516,6 +27516,12 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_THROW(invalid_iterator::create(211, "passed iterators may not belong to container", this)); } + // passed iterators must belong to arrays + if (JSON_HEDLEY_UNLIKELY(!first.m_object->is_array())) + { + JSON_THROW(invalid_iterator::create(202, "iterators first and last must point to arrays", this)); + } + // insert to array and return iterator return insert_iterator(pos, first.m_it.array_iterator, last.m_it.array_iterator); } diff --git a/tests/src/unit-modifiers.cpp b/tests/src/unit-modifiers.cpp index de14b3f70..369162772 100644 --- a/tests/src/unit-modifiers.cpp +++ b/tests/src/unit-modifiers.cpp @@ -641,6 +641,20 @@ TEST_CASE("modifiers") CHECK_THROWS_WITH_AS(j_array.insert(j_array.end(), j_other_array.begin(), j_other_array2.end()), "[json.exception.invalid_iterator.210] iterators do not fit", json::invalid_iterator&); } + + SECTION("iterators not pointing into an array") + { + json j_object2 = {{"k", 1}, {"l", 2}}; + json j_primitive = 5; + json j_null; + + CHECK_THROWS_WITH_AS(j_array.insert(j_array.begin(), j_object2.begin(), j_object2.end()), "[json.exception.invalid_iterator.202] iterators first and last must point to arrays", + json::invalid_iterator&); + CHECK_THROWS_WITH_AS(j_array.insert(j_array.begin(), j_primitive.begin(), j_primitive.end()), "[json.exception.invalid_iterator.202] iterators first and last must point to arrays", + json::invalid_iterator&); + CHECK_THROWS_WITH_AS(j_array.insert(j_array.begin(), j_null.begin(), j_null.end()), "[json.exception.invalid_iterator.202] iterators first and last must point to arrays", + json::invalid_iterator&); + } } SECTION("range for object") From 0b20b7e62211be63522b986bd8eef7d7c68f0cf5 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Tue, 22 Sep 2026 21:52:32 +0200 Subject: [PATCH 5/7] Reject MessagePack/BSON binary subtypes that don't fit their wire format (#5469) * Reject MessagePack/BSON binary subtypes that don't fit their wire format Both formats store byte_container_with_subtype's subtype (a uint64_t) in a single byte. The writers cast to std::int8_t/std::uint8_t without a range check, so subtypes above 255 were silently truncated modulo 256 instead of raising an error. Throw out_of_range.413 instead when the subtype exceeds the representable range of 0-255. Signed-off-by: Niels Lohmann * Move the new binary-subtype regression test out of unit-regression2.cpp unit-regression2.cpp is already at the edge of what the MinGW linker can relocate; adding this test's ~26 lines tips test-regression2_cpp20 (clang, Windows) over into "relocation truncated to fit: IMAGE_REL_AMD64_REL32 against `.rdata'" (see 8ce64b9c1 / b82717c8a for the same failure mode). Split the test along format lines instead: MessagePack assertions move to unit-msgpack.cpp, BSON assertions to unit-bson.cpp. The CBOR round-trip guard is dropped as redundant -- unit-cbor.cpp's "Tagged values" section already round-trips subtypes up to 8589934590, far past the 70000 checked here. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- docs/mkdocs/docs/home/exceptions.md | 15 +++++++++++++++ include/nlohmann/detail/output/binary_writer.hpp | 11 +++++++++++ single_include/nlohmann/json.hpp | 11 +++++++++++ tests/src/unit-bson.cpp | 9 +++++++++ tests/src/unit-msgpack.cpp | 15 +++++++++++++++ 5 files changed, 61 insertions(+) diff --git a/docs/mkdocs/docs/home/exceptions.md b/docs/mkdocs/docs/home/exceptions.md index 09cc8e178..9a7698b2f 100644 --- a/docs/mkdocs/docs/home/exceptions.md +++ b/docs/mkdocs/docs/home/exceptions.md @@ -970,6 +970,21 @@ A JSON Patch `move` operation's `"from"` location is a proper prefix of its `"pa This exception was added in version 3.13.0. Before that, this situation could succeed with a corrupted result: for an array target, removing the "from" element before the "add" step shifted subsequent indices, so "path" silently re-resolved to a different element than intended. +### json.exception.out_of_range.415 + +MessagePack's ext type and BSON's binary subtype are each stored in a single byte. This exception is thrown when serializing a +[`byte_container_with_subtype`](../api/byte_container_with_subtype/index.md) whose subtype exceeds 255. + +!!! failure "Example message" + + ``` + [json.exception.out_of_range.415] subtype 70000 is too large for the MessagePack ext type (max 255) + ``` + +!!! note + + This exception was added in version 3.13.0. Before that, subtypes above 255 were silently truncated modulo 256 instead of raising an error. + ## Further exceptions This exception is thrown in case of errors that cannot be classified with the diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 4bd173257..fe88e7f27 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -688,6 +688,11 @@ class binary_writer // step 1.5: if this is an ext type, write the subtype if (use_ext) { + if (JSON_HEDLEY_UNLIKELY(j.m_data.m_value.binary->subtype() > (std::numeric_limits::max)())) + { + JSON_THROW(out_of_range::create(415, concat("subtype ", std::to_string(j.m_data.m_value.binary->subtype()), " is too large for the MessagePack ext type (max 255)"), &j)); + } + write_number(static_cast(j.m_data.m_value.binary->subtype())); } @@ -1213,6 +1218,12 @@ class binary_writer write_bson_entry_header(name, 0x05); write_number(to_bson_length(value.size()), true); + + if (value.has_subtype() && JSON_HEDLEY_UNLIKELY(value.subtype() > (std::numeric_limits::max)())) + { + JSON_THROW(out_of_range::create(415, concat("subtype ", std::to_string(value.subtype()), " is too large for the BSON binary subtype (max 255)"), nullptr)); + } + write_number(value.has_subtype() ? static_cast(value.subtype()) : static_cast(0x00)); oa->write_characters(reinterpret_cast(value.data()), value.size()); diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 2c73944c0..f300cbdb1 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -19445,6 +19445,11 @@ class binary_writer // step 1.5: if this is an ext type, write the subtype if (use_ext) { + if (JSON_HEDLEY_UNLIKELY(j.m_data.m_value.binary->subtype() > (std::numeric_limits::max)())) + { + JSON_THROW(out_of_range::create(415, concat("subtype ", std::to_string(j.m_data.m_value.binary->subtype()), " is too large for the MessagePack ext type (max 255)"), &j)); + } + write_number(static_cast(j.m_data.m_value.binary->subtype())); } @@ -19970,6 +19975,12 @@ class binary_writer write_bson_entry_header(name, 0x05); write_number(to_bson_length(value.size()), true); + + if (value.has_subtype() && JSON_HEDLEY_UNLIKELY(value.subtype() > (std::numeric_limits::max)())) + { + JSON_THROW(out_of_range::create(415, concat("subtype ", std::to_string(value.subtype()), " is too large for the BSON binary subtype (max 255)"), nullptr)); + } + write_number(value.has_subtype() ? static_cast(value.subtype()) : static_cast(0x00)); oa->write_characters(reinterpret_cast(value.data()), value.size()); diff --git a/tests/src/unit-bson.cpp b/tests/src/unit-bson.cpp index 153e12d30..669a4bfe1 100644 --- a/tests/src/unit-bson.cpp +++ b/tests/src/unit-bson.cpp @@ -791,6 +791,15 @@ TEST_CASE("BSON") } } +TEST_CASE("regression test - BSON binary subtype rejects a value that doesn't fit a single byte") +{ + json const doc255 = {{"b", json::binary({1, 2}, 255)}}; + CHECK(json::from_bson(json::to_bson(doc255))["b"].get_binary().subtype() == 255); + + CHECK_THROWS_AS(json::to_bson(json{{"b", json::binary({1, 2}, 256)}}), json::out_of_range); + CHECK_THROWS_WITH_AS(json::to_bson(json{{"b", json::binary({1, 2}, 300)}}), "[json.exception.out_of_range.415] subtype 300 is too large for the BSON binary subtype (max 255)", json::out_of_range); +} + TEST_CASE("BSON input/output_adapters") { const json json_representation = diff --git a/tests/src/unit-msgpack.cpp b/tests/src/unit-msgpack.cpp index 74f7f4969..a8892081d 100644 --- a/tests/src/unit-msgpack.cpp +++ b/tests/src/unit-msgpack.cpp @@ -1682,6 +1682,21 @@ TEST_CASE("issue #5405 - array reserve for definite-length MessagePack arrays") } } +TEST_CASE("regression test - MessagePack ext type rejects a subtype that doesn't fit a single byte") +{ + // subtype 0-255 must still round-trip correctly (regression guard, pre-existing behavior) + CHECK(json::from_msgpack(json::to_msgpack(json::binary({1, 2}, 0))).get_binary().subtype() == 0); + CHECK(json::from_msgpack(json::to_msgpack(json::binary({1, 2}, 200))).get_binary().subtype() == 200); + CHECK(json::from_msgpack(json::to_msgpack(json::binary({1, 2}, 255))).get_binary().subtype() == 255); + + // a subtype > 255 must throw instead of silently truncating + CHECK_THROWS_AS(json::to_msgpack(json::binary({1, 2}, 256)), json::out_of_range); + CHECK_THROWS_WITH_AS(json::to_msgpack(json::binary({1, 2}, 70000)), "[json.exception.out_of_range.415] subtype 70000 is too large for the MessagePack ext type (max 255)", json::out_of_range); + + // a binary value with no subtype at all must be unaffected + CHECK(json::from_msgpack(json::to_msgpack(json::binary({1, 2}))).get_binary().has_subtype() == false); +} + // use this testcase outside [hide] to run it with Valgrind TEST_CASE("MessagePack nesting does not consume the call stack") { From f92024b31771395af1e828a757760847e11a26d9 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 23 Sep 2026 07:48:21 +0200 Subject: [PATCH 6/7] De-duplicate the swap() diagnostic-positions characterization test (#5540) --- tests/src/unit-class_parser.cpp | 95 ++++++--------------------------- 1 file changed, 15 insertions(+), 80 deletions(-) diff --git a/tests/src/unit-class_parser.cpp b/tests/src/unit-class_parser.cpp index e22c4cacf..df4e7270d 100644 --- a/tests/src/unit-class_parser.cpp +++ b/tests/src/unit-class_parser.cpp @@ -2261,86 +2261,6 @@ TEST_CASE("parser class") #endif } -#if JSON_DIAGNOSTIC_POSITIONS - -TEST_CASE("diagnostic positions: value lifetime") -{ - SECTION("copy constructor copies positions, recursively") - { - const std::string s = R"({"a":1,"b":[1,2,3]})"; - const json a = json::parse(s); - const json b = a; // NOLINT(performance-unnecessary-copy-initialization) - - CHECK(b.start_pos() == a.start_pos()); - CHECK(b.end_pos() == a.end_pos()); - CHECK(b["b"].start_pos() == a["b"].start_pos()); - CHECK(b["b"].end_pos() == a["b"].end_pos()); - } - - SECTION("move constructor resets the moved-from value to npos") - { - const std::string s = R"({"a":1,"b":[1,2,3]})"; - json a = json::parse(s); - const auto a_start = a.start_pos(); - const auto a_end = a.end_pos(); - - const json b(std::move(a)); - - CHECK(b.start_pos() == a_start); - CHECK(b.end_pos() == a_end); - - CHECK(a.start_pos() == std::string::npos); // NOLINT(bugprone-use-after-move,clang-analyzer-cplusplus.Move) - CHECK(a.end_pos() == std::string::npos); // NOLINT(bugprone-use-after-move,clang-analyzer-cplusplus.Move) - } - - SECTION("swap() exchanges positions along with the values") - { - // basic_json::swap() (and the friend swap() that forwards to it) used - // to swap only m_data.m_type/m_data.m_value, leaving - // start_position/end_position untouched -- unlike copy-assignment's - // operator=(basic_json), which swaps positions as part of its - // copy-and-swap implementation. After swap(a, b), each value ended up - // with the *other* value's content but its *own* original position. - // This is now fixed so that swap() is consistent with copy-assignment. - json a = json::parse(R"({"a":1})"); - json b = json::parse(R"([1,2,3,4,5])"); - const auto a_start = a.start_pos(); - const auto a_end = a.end_pos(); - const auto b_start = b.start_pos(); - const auto b_end = b.end_pos(); - // lengths (and thus end positions) differ, which is enough to tell - // after the swap whether positions actually moved with the values - CHECK(a_end != b_end); - - using std::swap; - swap(a, b); - - CHECK(a == json::parse(R"([1,2,3,4,5])")); - CHECK(b == json::parse(R"({"a":1})")); - - CHECK(a.start_pos() == b_start); - CHECK(a.end_pos() == b_end); - CHECK(b.start_pos() == a_start); - CHECK(b.end_pos() == a_end); - - // member swap() behaves the same as the free function - json c = json::parse(R"({"a":1})"); - json d = json::parse(R"([1,2,3,4,5])"); - const auto c_start = c.start_pos(); - const auto c_end = c.end_pos(); - const auto d_start = d.start_pos(); - const auto d_end = d.end_pos(); - - c.swap(d); - - CHECK(c.start_pos() == d_start); - CHECK(c.end_pos() == d_end); - CHECK(d.start_pos() == c_start); - CHECK(d.end_pos() == c_end); - } -} -#endif - // this test relies on parse errors being thrown, so it is skipped when // exceptions are disabled (json::parse aborts instead of throwing there) #if !defined(JSON_NOEXCEPTION) @@ -2546,6 +2466,21 @@ TEST_CASE("diagnostic positions: value lifetime, input adapters, and SAX") CHECK(a.end_pos() == b_end); CHECK(b.start_pos() == a_start); CHECK(b.end_pos() == a_end); + + // member swap() behaves the same as the free function + json c = json::parse(R"({"a":1})"); + json d = json::parse(R"([1,2,3,4,5])"); + const auto c_start = c.start_pos(); + const auto c_end = c.end_pos(); + const auto d_start = d.start_pos(); + const auto d_end = d.end_pos(); + + c.swap(d); + + CHECK(c.start_pos() == d_start); + CHECK(c.end_pos() == d_end); + CHECK(d.start_pos() == c_start); + CHECK(d.end_pos() == c_end); } SECTION("mutating a parsed document leaves positions of unrelated values untouched") From 1054b2097e721e8a455285a1b528cc72caf859b7 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 23 Sep 2026 08:59:46 +0200 Subject: [PATCH 7/7] Speed up binary writing: value-type output sink + byte-swap number encoding (#5286) * Devirtualize binary_writer via a value-type output sink to_cbor/to_msgpack/to_ubjson/to_bjdata/to_bson wrote every byte through output_adapter_t, a shared_ptr whose write_character/write_characters are virtual. Unlike the lexer (templated on a concrete InputAdapterType), the binary writer never got that treatment, so binary output paid a vtable lookup per byte and a make_shared per call. Template binary_writer on an OutputSinkType and give it two concrete, non-virtual sinks: - output_vector_sink: appends straight into a std::vector (push_back / insert), used by the vector-returning to_* convenience functions. No vtable, no shared_ptr; the writes inline. - output_adapter_sink: forwards to a type-erased output_adapter_t, so the existing to_*(j, output_adapter) overloads (streams, strings, custom adapters) keep working exactly as before -- one virtual call each, unchanged. binary_writer keeps a convenience constructor taking output_adapter_t (building the default output_adapter_sink), so the adapter overloads are untouched; only the convenience functions switch to the vector sink. The friend declaration and the basic_json binary_writer alias gain the new (defaulted) template parameter. Output is byte-for-byte identical: verified across ~3000 randomized values plus curated edge cases (all scalar widths, strings with invalid UTF-8, binary, nested arrays/objects) for CBOR, MessagePack, UBJSON (both size/type settings), BJData, and BSON, plus the output_adapter path, in C++11/17/20. Warning-clean under clang -Weverything and the gcc pedantic set; clang-tidy clean on the changed headers; make check-amalgamation clean. Throughput (g++ -O3, vs develop): scalar-dense binary output such as integer arrays ~1.4x; many small to_cbor calls ~1.04x (DOM traversal bound); string/blob-heavy output unchanged (already bulk-bound). No workload regressed. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01XAYM1qhSA2FDaDcGfPW3fG Signed-off-by: Niels Lohmann * Fix CI failures from binary_writer output-sink change Four CI jobs failed on the initial commit; all are addressed here without changing any output (binary encodings remain byte-for-byte identical to develop across the differential corpus): 1. ci_test_gcc / cuda (-Werror=duplicated-branches): for number_float_t == float, static_cast(n) is the identity, so write_compact_float's two branches are intentionally identical. Once the concrete vector sink is inlined, GCC constant-folds and diagnoses this (the type-erased path hid it behind a non-inlined virtual call). Silence -Wduplicated-branches for GCC (clang has no such warning) alongside the existing -Wfloat-equal pragma. 2. ci_static_analysis_clang (UBSan nonnull-attribute): binary_writer passes a null pointer with length 0 for empty strings/binary. output_vector_sink / output_adapter_sink declared write_characters JSON_HEDLEY_NON_NULL, so the sanitizer flagged the (harmless) zero-length call once the sink was called directly rather than through the attribute-free virtual base. Drop the attribute from both sinks, matching the pre-existing behavior. 3. ci_cpplint (build/include_what_you_use): output_adapter_sink uses std::move; add #include . 4. ci_cuda_example (nvcc 11.8): NVCC's front end rejects the default template argument on the binary_writer alias template. Revert the alias to its original single-parameter form (relying on binary_writer's own defaulted OutputSinkType) and spell out the full type in the vector-sink convenience functions. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01XAYM1qhSA2FDaDcGfPW3fG Signed-off-by: Niels Lohmann * Encode big-endian numbers with a byte swap instead of std::reverse write_number() reordered multi-byte numbers for the big-endian formats (CBOR/MessagePack/UBJSON) with std::reverse over the byte array. GCC lowered only some sizes to a bswap; clang kept a scalar byte shuffle (0 bswap instructions in the CBOR number path). Replace the reverse with size-dispatched __builtin_bswap16/32/64 helpers (portable shift fallback for other compilers; std::reverse retained for exotic sizes such as a long double number_float_t). Codegen: the CBOR number path now emits bswap on both compilers (gcc 2 -> 16, clang 0 -> 4). Output is byte-for-byte identical to the previous implementation across the binary differential corpus. Throughput (isolated vs the std::reverse version, best of 9): CBOR int64 array gcc +7% clang +10% CBOR uint16 array gcc +27% clang flat Modest but consistent on number-dense encodings; negligible on string/blob-heavy output, as expected. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01XAYM1qhSA2FDaDcGfPW3fG Signed-off-by: Niels Lohmann * Reserve output capacity up front for binary serialization The vector-returning to_cbor/to_msgpack/to_ubjson/to_bjdata/to_bson grew the output buffer purely by geometric reallocation. Reserving an estimate up front avoids the early reallocations, which is the dominant per-byte cost for array/object-heavy output. The estimate (binary_reserve_hint) is deliberately conservative and safe against untrusted input: it consults only the top-level element count (O(1), no walk of the DOM), guards the multiplication against overflow, and clamps the result to a fixed 1 MiB ceiling, so a large or hostile DOM can never force an oversized allocation here. The buffer still grows geometrically past the hint, so an underestimate only costs a few later reallocations; scalars/strings/binary are written in one shot and get no hint. Reserving capacity does not change the bytes produced. Throughput (g++/clang -O3, vs the previous commit): cbor int array +10% / +13% cbor object array +20% / +38% Output is byte-for-byte identical to develop across the binary differential corpus. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01XAYM1qhSA2FDaDcGfPW3fG Signed-off-by: Niels Lohmann * Address review findings on the binary writer output sinks - binary_reserve_hint(): the 4-bytes-per-element estimate over-reserved by up to 4x for arrays of small scalars (CBOR encodes 0..23 in one byte), and the returned vector kept that capacity. Make the hint a strict lower bound on the encoded size instead, which also removes the 1 MiB clamp whose branch no test could reach (the largest container in the suite has 65793 elements). - Guard the -Wduplicated-branches pragma with __GNUC__ >= 7. The warning does not exist before GCC 7, so naming it made GCC 4.8/4.9/5/6 - which the CI matrix still builds - warn under -Wpragmas on every including translation unit, breaking downstream -Werror builds. - Constrain the adapter constructor of binary_writer with the enable_if its documentation already claimed, so a writer over some other sink type is no longer advertised as constructible from an output adapter. - Let output_vector_adapter wrap output_vector_sink rather than duplicating the append logic, so the type-erased and templated paths share one implementation. - Collapse the three copies of the memcpy/byte_swap/memcpy dance into a single byte_swap_buffer() helper, and add the MSVC _byteswap_* intrinsics so MSVC no longer falls back to the scalar shuffle this change exists to eliminate. - Add a vector_writer() helper for the five vector-returning to_* overloads instead of spelling out the writer type at each call site, and drop a dead default member initializer on output_adapter_sink. - New tests: the vector sink and the adapter sink must produce identical bytes for every format (the two to_* overloads no longer delegate to each other and could otherwise drift), and binary_reserve_hint() must never exceed the size actually written. Signed-off-by: Niels Lohmann * Route the -Wduplicated-branches pragma through Hedley Match #5485, which moved the binary writer's hand-rolled diagnostic pragmas onto JSON_HEDLEY_PRAGMA (merged into develop while this branch was open). The devirtualization's -Wduplicated-branches suppression in write_compact_float was the one raw '#pragma GCC diagnostic' left; it now uses JSON_HEDLEY_PRAGMA like the adjacent -Wfloat-equal line, still guarded to GCC >= 7 and non-clang (the warning exists only there). Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01XAYM1qhSA2FDaDcGfPW3fG Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann Co-authored-by: Claude Opus 4.8 --- .../nlohmann/detail/output/binary_writer.hpp | 458 +++++++++----- .../detail/output/output_adapters.hpp | 84 ++- include/nlohmann/json.hpp | 25 +- single_include/nlohmann/json.hpp | 567 ++++++++++++------ tests/src/unit-binary_writer_sinks.cpp | 198 ++++++ 5 files changed, 992 insertions(+), 340 deletions(-) create mode 100644 tests/src/unit-binary_writer_sinks.cpp diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index fe88e7f27..a355f1c15 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -16,9 +16,14 @@ #include // memcpy #include // numeric_limits #include // string +#include // enable_if, is_constructible #include // move #include // vector +#ifdef _MSC_VER + #include // _byteswap_ushort, _byteswap_ulong, _byteswap_uint64 +#endif + #include #include #include @@ -39,10 +44,41 @@ enum class bjdata_version_t // binary writer // /////////////////// +/*! +@brief capacity hint for binary serialization into a std::vector + +Returns a *lower* bound on the number of bytes the serialization will produce, +so that writing an array/object of many elements does not start reallocating +from an empty buffer. Every array element occupies at least one byte in every +supported binary format, and every object entry at least two (a key of at least +one byte plus a value of at least one), plus one byte for the container header, +so the hint can never exceed the final size and the returned vector is never +left holding capacity the caller did not ask for. The buffer still grows +geometrically past the hint, so under-reserving only costs a few later +reallocations. Only the top-level element count is consulted (O(1), no walk of +the DOM); a single scalar, string, or binary value is written in one shot and +needs no hint. +*/ +template +std::size_t binary_reserve_hint(const BasicJsonType& j) +{ + if (j.is_array()) + { + return j.size() + 1; + } + + if (j.is_object()) + { + return (j.size() * 2) + 1; + } + + return 0; +} + /*! @brief serialization to CBOR and MessagePack values */ -template +template> class binary_writer { using string_t = typename BasicJsonType::string_t; @@ -53,12 +89,28 @@ class binary_writer /*! @brief create a binary writer + @param[in] sink output sink to write to (a value-type sink such as + output_vector_sink, or output_adapter_sink wrapping a + type-erased output adapter) + */ + explicit binary_writer(OutputSinkType sink) : oa(std::move(sink)) + {} + + /*! + @brief create a binary writer from a type-erased output adapter + + Convenience constructor for the default (output_adapter_sink) sink so the + `output_adapter`-based overloads keep constructing the writer directly from + an adapter. Constrained to sinks that can actually be built from an adapter, + so that a writer over some other sink type is not advertised as constructible + from one. + @param[in] adapter output adapter to write to */ - explicit binary_writer(output_adapter_t adapter) : oa(std::move(adapter)) - { - JSON_ASSERT(oa); - } + template < typename SinkType = OutputSinkType, + typename std::enable_if < std::is_constructible>::value, int >::type = 0 > + explicit binary_writer(output_adapter_t adapter) : oa(SinkType(std::move(adapter))) + {} /*! @param[in] j JSON value to serialize @@ -99,15 +151,15 @@ class binary_writer { case value_t::null: { - oa->write_character(to_char_type(0xF6)); + oa.write_character(to_char_type(0xF6)); break; } case value_t::boolean: { - oa->write_character(j.m_data.m_value.boolean - ? to_char_type(0xF5) - : to_char_type(0xF4)); + oa.write_character(j.m_data.m_value.boolean + ? to_char_type(0xF5) + : to_char_type(0xF4)); break; } @@ -124,22 +176,22 @@ class binary_writer } else if (j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x18)); + oa.write_character(to_char_type(0x18)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x19)); + oa.write_character(to_char_type(0x19)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x1A)); + oa.write_character(to_char_type(0x1A)); write_number(static_cast(j.m_data.m_value.number_integer)); } else { - oa->write_character(to_char_type(0x1B)); + oa.write_character(to_char_type(0x1B)); write_number(static_cast(j.m_data.m_value.number_integer)); } } @@ -154,22 +206,22 @@ class binary_writer } else if (positive_number <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x38)); + oa.write_character(to_char_type(0x38)); write_number(static_cast(positive_number)); } else if (positive_number <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x39)); + oa.write_character(to_char_type(0x39)); write_number(static_cast(positive_number)); } else if (positive_number <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x3A)); + oa.write_character(to_char_type(0x3A)); write_number(static_cast(positive_number)); } else { - oa->write_character(to_char_type(0x3B)); + oa.write_character(to_char_type(0x3B)); write_number(static_cast(positive_number)); } } @@ -184,22 +236,22 @@ class binary_writer } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x18)); + oa.write_character(to_char_type(0x18)); write_number(static_cast(j.m_data.m_value.number_unsigned)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x19)); + oa.write_character(to_char_type(0x19)); write_number(static_cast(j.m_data.m_value.number_unsigned)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x1A)); + oa.write_character(to_char_type(0x1A)); write_number(static_cast(j.m_data.m_value.number_unsigned)); } else { - oa->write_character(to_char_type(0x1B)); + oa.write_character(to_char_type(0x1B)); write_number(static_cast(j.m_data.m_value.number_unsigned)); } break; @@ -210,16 +262,16 @@ class binary_writer if (std::isnan(j.m_data.m_value.number_float)) { // NaN is 0xf97e00 in CBOR - oa->write_character(to_char_type(0xF9)); - oa->write_character(to_char_type(0x7E)); - oa->write_character(to_char_type(0x00)); + oa.write_character(to_char_type(0xF9)); + oa.write_character(to_char_type(0x7E)); + oa.write_character(to_char_type(0x00)); } else if (std::isinf(j.m_data.m_value.number_float)) { // Infinity is 0xf97c00, -Infinity is 0xf9fc00 - oa->write_character(to_char_type(0xf9)); - oa->write_character(j.m_data.m_value.number_float > 0 ? to_char_type(0x7C) : to_char_type(0xFC)); - oa->write_character(to_char_type(0x00)); + oa.write_character(to_char_type(0xf9)); + oa.write_character(j.m_data.m_value.number_float > 0 ? to_char_type(0x7C) : to_char_type(0xFC)); + oa.write_character(to_char_type(0x00)); } else { @@ -238,31 +290,31 @@ class binary_writer } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x78)); + oa.write_character(to_char_type(0x78)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x79)); + oa.write_character(to_char_type(0x79)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x7A)); + oa.write_character(to_char_type(0x7A)); write_number(static_cast(N)); } // LCOV_EXCL_START else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x7B)); + oa.write_character(to_char_type(0x7B)); write_number(static_cast(N)); } // LCOV_EXCL_STOP // step 2: write the string - oa->write_characters( - reinterpret_cast(j.m_data.m_value.string->data()), - j.m_data.m_value.string->size()); + oa.write_characters( + reinterpret_cast(j.m_data.m_value.string->data()), + j.m_data.m_value.string->size()); break; } @@ -276,23 +328,23 @@ class binary_writer } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x98)); + oa.write_character(to_char_type(0x98)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x99)); + oa.write_character(to_char_type(0x99)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x9A)); + oa.write_character(to_char_type(0x9A)); write_number(static_cast(N)); } // LCOV_EXCL_START else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x9B)); + oa.write_character(to_char_type(0x9B)); write_number(static_cast(N)); } // LCOV_EXCL_STOP @@ -339,31 +391,31 @@ class binary_writer } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x58)); + oa.write_character(to_char_type(0x58)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x59)); + oa.write_character(to_char_type(0x59)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x5A)); + oa.write_character(to_char_type(0x5A)); write_number(static_cast(N)); } // LCOV_EXCL_START else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x5B)); + oa.write_character(to_char_type(0x5B)); write_number(static_cast(N)); } // LCOV_EXCL_STOP // step 2: write each element - oa->write_characters( - reinterpret_cast(j.m_data.m_value.binary->data()), - N); + oa.write_characters( + reinterpret_cast(j.m_data.m_value.binary->data()), + N); break; } @@ -378,23 +430,23 @@ class binary_writer } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0xB8)); + oa.write_character(to_char_type(0xB8)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0xB9)); + oa.write_character(to_char_type(0xB9)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0xBA)); + oa.write_character(to_char_type(0xBA)); write_number(static_cast(N)); } // LCOV_EXCL_START else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0xBB)); + oa.write_character(to_char_type(0xBB)); write_number(static_cast(N)); } // LCOV_EXCL_STOP @@ -423,15 +475,15 @@ class binary_writer { case value_t::null: // nil { - oa->write_character(to_char_type(0xC0)); + oa.write_character(to_char_type(0xC0)); break; } case value_t::boolean: // true and false { - oa->write_character(j.m_data.m_value.boolean - ? to_char_type(0xC3) - : to_char_type(0xC2)); + oa.write_character(j.m_data.m_value.boolean + ? to_char_type(0xC3) + : to_char_type(0xC2)); break; } @@ -450,25 +502,25 @@ class binary_writer else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 8 - oa->write_character(to_char_type(0xCC)); + oa.write_character(to_char_type(0xCC)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 16 - oa->write_character(to_char_type(0xCD)); + oa.write_character(to_char_type(0xCD)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 32 - oa->write_character(to_char_type(0xCE)); + oa.write_character(to_char_type(0xCE)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 64 - oa->write_character(to_char_type(0xCF)); + oa.write_character(to_char_type(0xCF)); write_number(static_cast(j.m_data.m_value.number_integer)); } } @@ -483,28 +535,28 @@ class binary_writer j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { // int 8 - oa->write_character(to_char_type(0xD0)); + oa.write_character(to_char_type(0xD0)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { // int 16 - oa->write_character(to_char_type(0xD1)); + oa.write_character(to_char_type(0xD1)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { // int 32 - oa->write_character(to_char_type(0xD2)); + oa.write_character(to_char_type(0xD2)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { // int 64 - oa->write_character(to_char_type(0xD3)); + oa.write_character(to_char_type(0xD3)); write_number(static_cast(j.m_data.m_value.number_integer)); } } @@ -521,25 +573,25 @@ class binary_writer else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 8 - oa->write_character(to_char_type(0xCC)); + oa.write_character(to_char_type(0xCC)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 16 - oa->write_character(to_char_type(0xCD)); + oa.write_character(to_char_type(0xCD)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 32 - oa->write_character(to_char_type(0xCE)); + oa.write_character(to_char_type(0xCE)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 64 - oa->write_character(to_char_type(0xCF)); + oa.write_character(to_char_type(0xCF)); write_number(static_cast(j.m_data.m_value.number_integer)); } break; @@ -563,26 +615,26 @@ class binary_writer else if (N <= (std::numeric_limits::max)()) { // str 8 - oa->write_character(to_char_type(0xD9)); + oa.write_character(to_char_type(0xD9)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { // str 16 - oa->write_character(to_char_type(0xDA)); + oa.write_character(to_char_type(0xDA)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { // str 32 - oa->write_character(to_char_type(0xDB)); + oa.write_character(to_char_type(0xDB)); write_number(static_cast(N)); } // step 2: write the string - oa->write_characters( - reinterpret_cast(j.m_data.m_value.string->data()), - j.m_data.m_value.string->size()); + oa.write_characters( + reinterpret_cast(j.m_data.m_value.string->data()), + j.m_data.m_value.string->size()); break; } @@ -598,13 +650,13 @@ class binary_writer else if (N <= (std::numeric_limits::max)()) { // array 16 - oa->write_character(to_char_type(0xDC)); + oa.write_character(to_char_type(0xDC)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { // array 32 - oa->write_character(to_char_type(0xDD)); + oa.write_character(to_char_type(0xDD)); write_number(static_cast(N)); } @@ -660,7 +712,7 @@ class binary_writer fixed = false; } - oa->write_character(to_char_type(output_type)); + oa.write_character(to_char_type(output_type)); if (!fixed) { write_number(static_cast(N)); @@ -672,7 +724,7 @@ class binary_writer ? 0xC8 // ext 16 : 0xC5; // bin 16 - oa->write_character(to_char_type(output_type)); + oa.write_character(to_char_type(output_type)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) @@ -681,7 +733,7 @@ class binary_writer ? 0xC9 // ext 32 : 0xC6; // bin 32 - oa->write_character(to_char_type(output_type)); + oa.write_character(to_char_type(output_type)); write_number(static_cast(N)); } @@ -697,9 +749,9 @@ class binary_writer } // step 2: write the byte string - oa->write_characters( - reinterpret_cast(j.m_data.m_value.binary->data()), - N); + oa.write_characters( + reinterpret_cast(j.m_data.m_value.binary->data()), + N); break; } @@ -716,13 +768,13 @@ class binary_writer else if (N <= (std::numeric_limits::max)()) { // map 16 - oa->write_character(to_char_type(0xDE)); + oa.write_character(to_char_type(0xDE)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { // map 32 - oa->write_character(to_char_type(0xDF)); + oa.write_character(to_char_type(0xDF)); write_number(static_cast(N)); } @@ -761,7 +813,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('Z')); + oa.write_character(to_char_type('Z')); } break; } @@ -770,9 +822,9 @@ class binary_writer { if (add_prefix) { - oa->write_character(j.m_data.m_value.boolean - ? to_char_type('T') - : to_char_type('F')); + oa.write_character(j.m_data.m_value.boolean + ? to_char_type('T') + : to_char_type('F')); } break; } @@ -799,12 +851,12 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('S')); + oa.write_character(to_char_type('S')); } write_number_with_ubjson_prefix(j.m_data.m_value.string->size(), true, use_bjdata); - oa->write_characters( - reinterpret_cast(j.m_data.m_value.string->data()), - j.m_data.m_value.string->size()); + oa.write_characters( + reinterpret_cast(j.m_data.m_value.string->data()), + j.m_data.m_value.string->size()); break; } @@ -812,7 +864,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('[')); + oa.write_character(to_char_type('[')); } bool prefix_required = true; @@ -844,14 +896,14 @@ class binary_writer && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) { prefix_required = false; - oa->write_character(to_char_type('$')); - oa->write_character(first_prefix); + oa.write_character(to_char_type('$')); + oa.write_character(first_prefix); } } if (use_count) { - oa->write_character(to_char_type('#')); + oa.write_character(to_char_type('#')); write_number_with_ubjson_prefix(j.m_data.m_value.array->size(), true, use_bjdata); } @@ -862,7 +914,7 @@ class binary_writer if (!use_count) { - oa->write_character(to_char_type(']')); + oa.write_character(to_char_type(']')); } break; @@ -872,7 +924,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('[')); + oa.write_character(to_char_type('[')); } if (use_type && (bjdata_draft3 || !j.m_data.m_value.binary->empty())) @@ -881,36 +933,36 @@ class binary_writer { JSON_THROW(other_error::create(502, "use_type requires use_size = true", &j)); } - oa->write_character(to_char_type('$')); - oa->write_character(bjdata_draft3 ? 'B' : 'U'); + oa.write_character(to_char_type('$')); + oa.write_character(bjdata_draft3 ? 'B' : 'U'); } if (use_count) { - oa->write_character(to_char_type('#')); + oa.write_character(to_char_type('#')); write_number_with_ubjson_prefix(j.m_data.m_value.binary->size(), true, use_bjdata); } if (use_type) { - oa->write_characters( - reinterpret_cast(j.m_data.m_value.binary->data()), - j.m_data.m_value.binary->size()); + oa.write_characters( + reinterpret_cast(j.m_data.m_value.binary->data()), + j.m_data.m_value.binary->size()); } else { for (size_t i = 0; i < j.m_data.m_value.binary->size(); ++i) { - oa->write_character(to_char_type(bjdata_draft3 ? 'B' : 'U')); + oa.write_character(to_char_type(bjdata_draft3 ? 'B' : 'U')); // the cast is needed for binary types whose value type // is not an integer (e.g., std::byte) - oa->write_character(to_char_type(static_cast(j.m_data.m_value.binary->data()[i]))); + oa.write_character(to_char_type(static_cast(j.m_data.m_value.binary->data()[i]))); } } if (!use_count) { - oa->write_character(to_char_type(']')); + oa.write_character(to_char_type(']')); } break; @@ -928,7 +980,7 @@ class binary_writer if (add_prefix) { - oa->write_character(to_char_type('{')); + oa.write_character(to_char_type('{')); } bool prefix_required = true; @@ -950,29 +1002,29 @@ class binary_writer if (same_prefix && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) { prefix_required = false; - oa->write_character(to_char_type('$')); - oa->write_character(first_prefix); + oa.write_character(to_char_type('$')); + oa.write_character(first_prefix); } } if (use_count) { - oa->write_character(to_char_type('#')); + oa.write_character(to_char_type('#')); write_number_with_ubjson_prefix(j.m_data.m_value.object->size(), true, use_bjdata); } for (const auto& el : *j.m_data.m_value.object) { write_number_with_ubjson_prefix(el.first.size(), true, use_bjdata); - oa->write_characters( - reinterpret_cast(el.first.data()), - el.first.size()); + oa.write_characters( + reinterpret_cast(el.first.data()), + el.first.size()); write_ubjson(el.second, use_count, use_type, prefix_required, use_bjdata, bjdata_version); } if (!use_count) { - oa->write_character(to_char_type('}')); + oa.write_character(to_char_type('}')); } break; @@ -1026,13 +1078,13 @@ class binary_writer void write_bson_entry_header(const string_t& name, const std::uint8_t element_type) { - oa->write_character(to_char_type(element_type)); - oa->write_characters( - reinterpret_cast(name.data()), - name.size()); + oa.write_character(to_char_type(element_type)); + oa.write_characters( + reinterpret_cast(name.data()), + name.size()); // the terminating null byte is written explicitly rather than taken // from the buffer, so that string_t::data() need not be null-terminated - oa->write_character(to_char_type(0x00)); + oa.write_character(to_char_type(0x00)); } /*! @@ -1042,7 +1094,7 @@ class binary_writer const bool value) { write_bson_entry_header(name, 0x08); - oa->write_character(value ? to_char_type(0x01) : to_char_type(0x00)); + oa.write_character(value ? to_char_type(0x01) : to_char_type(0x00)); } /*! @@ -1072,12 +1124,12 @@ class binary_writer write_bson_entry_header(name, 0x02); write_number(to_bson_length(value.size() + 1ul), true); - oa->write_characters( - reinterpret_cast(value.data()), - value.size()); + oa.write_characters( + reinterpret_cast(value.data()), + value.size()); // the terminating null byte is written explicitly rather than taken // from the buffer, so that string_t::data() need not be null-terminated - oa->write_character(to_char_type(0x00)); + oa.write_character(to_char_type(0x00)); } /*! @@ -1206,7 +1258,7 @@ class binary_writer write_bson_element(string_t(key.data(), key.size()), el); } - oa->write_character(to_char_type(0x00)); + oa.write_character(to_char_type(0x00)); } /*! @@ -1226,7 +1278,7 @@ class binary_writer write_number(value.has_subtype() ? static_cast(value.subtype()) : static_cast(0x00)); - oa->write_characters(reinterpret_cast(value.data()), value.size()); + oa.write_characters(reinterpret_cast(value.data()), value.size()); } /*! @@ -1352,7 +1404,7 @@ class binary_writer write_bson_element(el.first, el.second); } - oa->write_character(to_char_type(0x00)); + oa.write_character(to_char_type(0x00)); } ////////// @@ -1396,7 +1448,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(get_ubjson_float_prefix(n)); + oa.write_character(get_ubjson_float_prefix(n)); } write_number(n, use_bjdata); } @@ -1412,7 +1464,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('i')); // int8 + oa.write_character(to_char_type('i')); // int8 } write_number(static_cast(n), use_bjdata); } @@ -1420,7 +1472,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('U')); // uint8 + oa.write_character(to_char_type('U')); // uint8 } write_number(static_cast(n), use_bjdata); } @@ -1428,7 +1480,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('I')); // int16 + oa.write_character(to_char_type('I')); // int16 } write_number(static_cast(n), use_bjdata); } @@ -1436,7 +1488,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('u')); // uint16 - bjdata only + oa.write_character(to_char_type('u')); // uint16 - bjdata only } write_number(static_cast(n), use_bjdata); } @@ -1444,7 +1496,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('l')); // int32 + oa.write_character(to_char_type('l')); // int32 } write_number(static_cast(n), use_bjdata); } @@ -1452,7 +1504,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('m')); // uint32 - bjdata only + oa.write_character(to_char_type('m')); // uint32 - bjdata only } write_number(static_cast(n), use_bjdata); } @@ -1460,7 +1512,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('L')); // int64 + oa.write_character(to_char_type('L')); // int64 } write_number(static_cast(n), use_bjdata); } @@ -1468,7 +1520,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('M')); // uint64 - bjdata only + oa.write_character(to_char_type('M')); // uint64 - bjdata only } write_number(static_cast(n), use_bjdata); } @@ -1476,14 +1528,14 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('H')); // high-precision number + oa.write_character(to_char_type('H')); // high-precision number } const auto number = BasicJsonType(n).dump(); write_number_with_ubjson_prefix(number.size(), true, use_bjdata); for (std::size_t i = 0; i < number.size(); ++i) { - oa->write_character(to_char_type(static_cast(number[i]))); + oa.write_character(to_char_type(static_cast(number[i]))); } } } @@ -1500,7 +1552,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('i')); // int8 + oa.write_character(to_char_type('i')); // int8 } write_number(static_cast(n), use_bjdata); } @@ -1508,7 +1560,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('U')); // uint8 + oa.write_character(to_char_type('U')); // uint8 } write_number(static_cast(n), use_bjdata); } @@ -1516,7 +1568,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('I')); // int16 + oa.write_character(to_char_type('I')); // int16 } write_number(static_cast(n), use_bjdata); } @@ -1524,7 +1576,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('u')); // uint16 - bjdata only + oa.write_character(to_char_type('u')); // uint16 - bjdata only } write_number(static_cast(n), use_bjdata); } @@ -1532,7 +1584,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('l')); // int32 + oa.write_character(to_char_type('l')); // int32 } write_number(static_cast(n), use_bjdata); } @@ -1540,7 +1592,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('m')); // uint32 - bjdata only + oa.write_character(to_char_type('m')); // uint32 - bjdata only } write_number(static_cast(n), use_bjdata); } @@ -1548,7 +1600,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('L')); // int64 + oa.write_character(to_char_type('L')); // int64 } write_number(static_cast(n), use_bjdata); } @@ -1557,14 +1609,14 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('H')); // high-precision number + oa.write_character(to_char_type('H')); // high-precision number } const auto number = BasicJsonType(n).dump(); write_number_with_ubjson_prefix(number.size(), true, use_bjdata); for (std::size_t i = 0; i < number.size(); ++i) { - oa->write_character(to_char_type(static_cast(number[i]))); + oa.write_character(to_char_type(static_cast(number[i]))); } } // LCOV_EXCL_STOP @@ -1856,10 +1908,10 @@ class binary_writer } } - oa->write_character('['); - oa->write_character('$'); - oa->write_character(dtype); - oa->write_character('#'); + oa.write_character('['); + oa.write_character('$'); + oa.write_character(dtype); + oa.write_character('#'); key = "_ArraySize_"; write_ubjson(value.at(key), use_count, use_type, true, true, bjdata_version); @@ -1955,6 +2007,87 @@ class binary_writer On the other hand, BSON and BJData use little endian and should reorder on big endian systems. */ + // single-instruction byte swaps (compilers lower these to bswap/rev/movbe); + // used to emit big-endian numbers without a per-byte std::reverse loop + static std::uint16_t byte_swap(std::uint16_t x) noexcept + { +#if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap16(x); +#elif defined(_MSC_VER) + return _byteswap_ushort(x); +#else + return static_cast((x >> 8) | (x << 8)); +#endif + } + + static std::uint32_t byte_swap(std::uint32_t x) noexcept + { +#if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap32(x); +#elif defined(_MSC_VER) + return _byteswap_ulong(x); +#else + return ((x & 0x000000FFu) << 24) | ((x & 0x0000FF00u) << 8) + | ((x & 0x00FF0000u) >> 8) | ((x & 0xFF000000u) >> 24); +#endif + } + + static std::uint64_t byte_swap(std::uint64_t x) noexcept + { +#if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap64(x); +#elif defined(_MSC_VER) + return _byteswap_uint64(x); +#else + x = ((x & 0x00000000FFFFFFFFull) << 32) | ((x & 0xFFFFFFFF00000000ull) >> 32); + x = ((x & 0x0000FFFF0000FFFFull) << 16) | ((x & 0xFFFF0000FFFF0000ull) >> 16); + x = ((x & 0x00FF00FF00FF00FFull) << 8) | ((x & 0xFF00FF00FF00FF00ull) >> 8); + return x; +#endif + } + + /*! + @brief reverse the bytes of a buffer by byte-swapping it as UIntType + + Loading the buffer into an unsigned integer of the same width and swapping + that is what lets the compiler emit a single bswap/rev/movbe; reversing the + buffer element by element does not reliably get there (clang keeps a scalar + shuffle). The two memcpy calls are the only portable way to reinterpret the + bytes and are folded away by every optimizer. + */ + template + static void byte_swap_buffer(std::array& a) noexcept + { + static_assert(sizeof(UIntType) == N, "swap width must match the buffer size"); + UIntType v{}; + std::memcpy(&v, a.data(), sizeof(v)); + v = byte_swap(v); + std::memcpy(a.data(), &v, sizeof(v)); + } + + // reverse the bytes of a fixed-size buffer; a single byte_swap() for the + // common 2/4/8-byte number payloads, std::reverse for any other size + static void reverse_bytes(std::array& a) noexcept + { + byte_swap_buffer(a); + } + + static void reverse_bytes(std::array& a) noexcept + { + byte_swap_buffer(a); + } + + static void reverse_bytes(std::array& a) noexcept + { + byte_swap_buffer(a); + } + + template + static void reverse_bytes(std::array& a) noexcept + { + std::reverse(a.begin(), a.end()); + } + template void write_number(const NumberType n, const bool OutputIsLittleEndian = false) { @@ -1966,10 +2099,10 @@ class binary_writer if (is_little_endian != OutputIsLittleEndian) { // reverse byte order prior to conversion if necessary - std::reverse(vec.begin(), vec.end()); + reverse_bytes(vec); } - oa->write_characters(vec.data(), sizeof(NumberType)); + oa.write_characters(vec.data(), sizeof(NumberType)); } void write_compact_float(const number_float_t n, detail::input_format_t format) @@ -1977,21 +2110,30 @@ class binary_writer #ifdef __GNUC__ JSON_HEDLEY_DIAGNOSTIC_PUSH JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") +#endif + // When number_float_t is float, static_cast(n) is the identity and + // both branches below are intentionally identical (the "compact" float + // representation is the value itself). Only GCC diagnoses this, and only + // when the sink calls are inlined; clang has no such warning. + // (-Wduplicated-branches only exists from GCC 7 on; naming it on an older + // GCC would itself warn under -Wpragmas) +#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ >= 7) + JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wduplicated-branches") #endif if (!std::isfinite(n) || ((static_cast(n) >= static_cast(std::numeric_limits::lowest()) && static_cast(n) <= static_cast((std::numeric_limits::max)()) && static_cast(static_cast(n)) == static_cast(n)))) { - oa->write_character(format == detail::input_format_t::cbor - ? get_cbor_float_prefix(static_cast(n)) - : get_msgpack_float_prefix(static_cast(n))); + oa.write_character(format == detail::input_format_t::cbor + ? get_cbor_float_prefix(static_cast(n)) + : get_msgpack_float_prefix(static_cast(n))); write_number(static_cast(n)); } else { - oa->write_character(format == detail::input_format_t::cbor - ? get_cbor_float_prefix(n) - : get_msgpack_float_prefix(n)); + oa.write_character(format == detail::input_format_t::cbor + ? get_cbor_float_prefix(n) + : get_msgpack_float_prefix(n)); write_number(n); } #ifdef __GNUC__ @@ -2058,7 +2200,7 @@ class binary_writer const bool is_little_endian = little_endianness(); /// the output - output_adapter_t oa = nullptr; + OutputSinkType oa; }; } // namespace detail diff --git a/include/nlohmann/detail/output/output_adapters.hpp b/include/nlohmann/detail/output/output_adapters.hpp index 94763e64a..7eb73121c 100644 --- a/include/nlohmann/detail/output/output_adapters.hpp +++ b/include/nlohmann/detail/output/output_adapters.hpp @@ -13,6 +13,7 @@ #include // back_inserter #include // shared_ptr, make_shared #include // basic_string +#include // move #include // vector #ifndef JSON_NO_IO @@ -44,22 +45,32 @@ template struct output_adapter_protocol template using output_adapter_t = std::shared_ptr>; -/// output adapter for byte vectors +/// @brief non-virtual output sink writing into a std::vector +/// +/// This sink is not part of the virtual output_adapter_protocol hierarchy: it is +/// passed to binary_writer by value as a template parameter, so +/// write_character()/write_characters() are ordinary (inlinable) calls with no +/// vtable lookup and no shared_ptr. It is used for the common +/// `to_cbor`/`to_msgpack`/... into a std::vector. output_vector_adapter below +/// wraps this same sink to provide the virtual interface. template> -class output_vector_adapter : public output_adapter_protocol +class output_vector_sink { public: - explicit output_vector_adapter(std::vector& vec) noexcept + explicit output_vector_sink(std::vector& vec) noexcept : v(vec) {} - void write_character(CharType c) override + void write_character(CharType c) { v.push_back(c); } - JSON_HEDLEY_NON_NULL(2) - void write_characters(const CharType* s, std::size_t length) override + // no JSON_HEDLEY_NON_NULL here: binary_writer legitimately passes a null + // pointer with length 0 for empty strings/binary values. Appending an empty + // range is a no-op; the type-erased path tolerates this via the (unattributed) + // virtual base, and the concrete sink must do the same. + void write_characters(const CharType* s, std::size_t length) { v.insert(v.end(), s, s + length); } @@ -68,6 +79,34 @@ class output_vector_adapter : public output_adapter_protocol std::vector& v; }; +/// output adapter for byte vectors +/// +/// The appending itself lives in output_vector_sink; this class only adds the +/// virtual output_adapter_protocol interface on top of it, so both the +/// type-erased and the templated path share one implementation. +template> +class output_vector_adapter : public output_adapter_protocol +{ + public: + explicit output_vector_adapter(std::vector& vec) noexcept + : sink(vec) + {} + + void write_character(CharType c) override + { + sink.write_character(c); + } + + JSON_HEDLEY_NON_NULL(2) + void write_characters(const CharType* s, std::size_t length) override + { + sink.write_characters(s, length); + } + + private: + output_vector_sink sink; +}; + #ifndef JSON_NO_IO /// output adapter for output streams template @@ -118,6 +157,39 @@ class output_string_adapter : public output_adapter_protocol StringType& str; }; +/// @brief output sink forwarding to a type-erased output adapter +/// +/// Wraps the polymorphic output_adapter_t so the same binary_writer template can +/// also target arbitrary adapters (output streams, strings, user-provided +/// adapters) via the `output_adapter`-based overloads. Each write still goes +/// through one virtual call, exactly as before; only the concrete sinks above +/// avoid it. +template +class output_adapter_sink +{ + public: + explicit output_adapter_sink(output_adapter_t adapter) + : oa(std::move(adapter)) + { + JSON_ASSERT(oa); + } + + void write_character(CharType c) + { + oa->write_character(c); + } + + // no JSON_HEDLEY_NON_NULL: forwards (null, 0) for empty payloads, exactly as + // the type-erased path already did before this sink existed + void write_characters(const CharType* s, std::size_t length) + { + oa->write_characters(s, length); + } + + private: + output_adapter_t oa; +}; + template> class output_adapter { diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index d55efacc2..a63363fbf 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -140,7 +140,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec friend ::nlohmann::detail::serializer; template friend class ::nlohmann::detail::iter_impl; - template + template friend class ::nlohmann::detail::binary_writer; template friend class ::nlohmann::detail::binary_reader; @@ -188,6 +188,14 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec template using binary_reader = ::nlohmann::detail::binary_reader; template using binary_writer = ::nlohmann::detail::binary_writer; + // binary_writer over a concrete (non-virtual) sink appending into a std::vector, + // used by the vector-returning to_* overloads + template using vector_binary_writer = + ::nlohmann::detail::binary_writer>; + template static vector_binary_writer vector_writer(std::vector& v) + { + return vector_binary_writer(::nlohmann::detail::output_vector_sink(v)); + } JSON_PRIVATE_UNLESS_TESTED: using serializer = ::nlohmann::detail::serializer; @@ -4444,7 +4452,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec static std::vector to_cbor(const basic_json& j) { std::vector result; - to_cbor(j, result); + result.reserve(detail::binary_reserve_hint(j)); + vector_writer(result).write_cbor(j); return result; } @@ -4467,7 +4476,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec static std::vector to_msgpack(const basic_json& j) { std::vector result; - to_msgpack(j, result); + result.reserve(detail::binary_reserve_hint(j)); + vector_writer(result).write_msgpack(j); return result; } @@ -4492,7 +4502,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool use_type = false) { std::vector result; - to_ubjson(j, result, use_size, use_type); + result.reserve(detail::binary_reserve_hint(j)); + vector_writer(result).write_ubjson(j, use_size, use_type); return result; } @@ -4520,7 +4531,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bjdata_version_t version = bjdata_version_t::draft2) { std::vector result; - to_bjdata(j, result, use_size, use_type, version); + result.reserve(detail::binary_reserve_hint(j)); + vector_writer(result).write_ubjson(j, use_size, use_type, true, true, version); return result; } @@ -4547,7 +4559,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec static std::vector to_bson(const basic_json& j) { std::vector result; - to_bson(j, result); + result.reserve(detail::binary_reserve_hint(j)); + vector_writer(result).write_bson(j); return result; } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index f300cbdb1..4af4af1e0 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -18621,9 +18621,14 @@ NLOHMANN_JSON_NAMESPACE_END #include // memcpy #include // numeric_limits #include // string +#include // enable_if, is_constructible #include // move #include // vector +#ifdef _MSC_VER + #include // _byteswap_ushort, _byteswap_ulong, _byteswap_uint64 +#endif + // #include // #include @@ -18644,6 +18649,7 @@ NLOHMANN_JSON_NAMESPACE_END #include // back_inserter #include // shared_ptr, make_shared #include // basic_string +#include // move #include // vector #ifndef JSON_NO_IO @@ -18676,22 +18682,32 @@ template struct output_adapter_protocol template using output_adapter_t = std::shared_ptr>; -/// output adapter for byte vectors +/// @brief non-virtual output sink writing into a std::vector +/// +/// This sink is not part of the virtual output_adapter_protocol hierarchy: it is +/// passed to binary_writer by value as a template parameter, so +/// write_character()/write_characters() are ordinary (inlinable) calls with no +/// vtable lookup and no shared_ptr. It is used for the common +/// `to_cbor`/`to_msgpack`/... into a std::vector. output_vector_adapter below +/// wraps this same sink to provide the virtual interface. template> -class output_vector_adapter : public output_adapter_protocol +class output_vector_sink { public: - explicit output_vector_adapter(std::vector& vec) noexcept + explicit output_vector_sink(std::vector& vec) noexcept : v(vec) {} - void write_character(CharType c) override + void write_character(CharType c) { v.push_back(c); } - JSON_HEDLEY_NON_NULL(2) - void write_characters(const CharType* s, std::size_t length) override + // no JSON_HEDLEY_NON_NULL here: binary_writer legitimately passes a null + // pointer with length 0 for empty strings/binary values. Appending an empty + // range is a no-op; the type-erased path tolerates this via the (unattributed) + // virtual base, and the concrete sink must do the same. + void write_characters(const CharType* s, std::size_t length) { v.insert(v.end(), s, s + length); } @@ -18700,6 +18716,34 @@ class output_vector_adapter : public output_adapter_protocol std::vector& v; }; +/// output adapter for byte vectors +/// +/// The appending itself lives in output_vector_sink; this class only adds the +/// virtual output_adapter_protocol interface on top of it, so both the +/// type-erased and the templated path share one implementation. +template> +class output_vector_adapter : public output_adapter_protocol +{ + public: + explicit output_vector_adapter(std::vector& vec) noexcept + : sink(vec) + {} + + void write_character(CharType c) override + { + sink.write_character(c); + } + + JSON_HEDLEY_NON_NULL(2) + void write_characters(const CharType* s, std::size_t length) override + { + sink.write_characters(s, length); + } + + private: + output_vector_sink sink; +}; + #ifndef JSON_NO_IO /// output adapter for output streams template @@ -18750,6 +18794,39 @@ class output_string_adapter : public output_adapter_protocol StringType& str; }; +/// @brief output sink forwarding to a type-erased output adapter +/// +/// Wraps the polymorphic output_adapter_t so the same binary_writer template can +/// also target arbitrary adapters (output streams, strings, user-provided +/// adapters) via the `output_adapter`-based overloads. Each write still goes +/// through one virtual call, exactly as before; only the concrete sinks above +/// avoid it. +template +class output_adapter_sink +{ + public: + explicit output_adapter_sink(output_adapter_t adapter) + : oa(std::move(adapter)) + { + JSON_ASSERT(oa); + } + + void write_character(CharType c) + { + oa->write_character(c); + } + + // no JSON_HEDLEY_NON_NULL: forwards (null, 0) for empty payloads, exactly as + // the type-erased path already did before this sink existed + void write_characters(const CharType* s, std::size_t length) + { + oa->write_characters(s, length); + } + + private: + output_adapter_t oa; +}; + template> class output_adapter { @@ -18796,10 +18873,41 @@ enum class bjdata_version_t // binary writer // /////////////////// +/*! +@brief capacity hint for binary serialization into a std::vector + +Returns a *lower* bound on the number of bytes the serialization will produce, +so that writing an array/object of many elements does not start reallocating +from an empty buffer. Every array element occupies at least one byte in every +supported binary format, and every object entry at least two (a key of at least +one byte plus a value of at least one), plus one byte for the container header, +so the hint can never exceed the final size and the returned vector is never +left holding capacity the caller did not ask for. The buffer still grows +geometrically past the hint, so under-reserving only costs a few later +reallocations. Only the top-level element count is consulted (O(1), no walk of +the DOM); a single scalar, string, or binary value is written in one shot and +needs no hint. +*/ +template +std::size_t binary_reserve_hint(const BasicJsonType& j) +{ + if (j.is_array()) + { + return j.size() + 1; + } + + if (j.is_object()) + { + return (j.size() * 2) + 1; + } + + return 0; +} + /*! @brief serialization to CBOR and MessagePack values */ -template +template> class binary_writer { using string_t = typename BasicJsonType::string_t; @@ -18810,12 +18918,28 @@ class binary_writer /*! @brief create a binary writer + @param[in] sink output sink to write to (a value-type sink such as + output_vector_sink, or output_adapter_sink wrapping a + type-erased output adapter) + */ + explicit binary_writer(OutputSinkType sink) : oa(std::move(sink)) + {} + + /*! + @brief create a binary writer from a type-erased output adapter + + Convenience constructor for the default (output_adapter_sink) sink so the + `output_adapter`-based overloads keep constructing the writer directly from + an adapter. Constrained to sinks that can actually be built from an adapter, + so that a writer over some other sink type is not advertised as constructible + from one. + @param[in] adapter output adapter to write to */ - explicit binary_writer(output_adapter_t adapter) : oa(std::move(adapter)) - { - JSON_ASSERT(oa); - } + template < typename SinkType = OutputSinkType, + typename std::enable_if < std::is_constructible>::value, int >::type = 0 > + explicit binary_writer(output_adapter_t adapter) : oa(SinkType(std::move(adapter))) + {} /*! @param[in] j JSON value to serialize @@ -18856,15 +18980,15 @@ class binary_writer { case value_t::null: { - oa->write_character(to_char_type(0xF6)); + oa.write_character(to_char_type(0xF6)); break; } case value_t::boolean: { - oa->write_character(j.m_data.m_value.boolean - ? to_char_type(0xF5) - : to_char_type(0xF4)); + oa.write_character(j.m_data.m_value.boolean + ? to_char_type(0xF5) + : to_char_type(0xF4)); break; } @@ -18881,22 +19005,22 @@ class binary_writer } else if (j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x18)); + oa.write_character(to_char_type(0x18)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x19)); + oa.write_character(to_char_type(0x19)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x1A)); + oa.write_character(to_char_type(0x1A)); write_number(static_cast(j.m_data.m_value.number_integer)); } else { - oa->write_character(to_char_type(0x1B)); + oa.write_character(to_char_type(0x1B)); write_number(static_cast(j.m_data.m_value.number_integer)); } } @@ -18911,22 +19035,22 @@ class binary_writer } else if (positive_number <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x38)); + oa.write_character(to_char_type(0x38)); write_number(static_cast(positive_number)); } else if (positive_number <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x39)); + oa.write_character(to_char_type(0x39)); write_number(static_cast(positive_number)); } else if (positive_number <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x3A)); + oa.write_character(to_char_type(0x3A)); write_number(static_cast(positive_number)); } else { - oa->write_character(to_char_type(0x3B)); + oa.write_character(to_char_type(0x3B)); write_number(static_cast(positive_number)); } } @@ -18941,22 +19065,22 @@ class binary_writer } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x18)); + oa.write_character(to_char_type(0x18)); write_number(static_cast(j.m_data.m_value.number_unsigned)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x19)); + oa.write_character(to_char_type(0x19)); write_number(static_cast(j.m_data.m_value.number_unsigned)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x1A)); + oa.write_character(to_char_type(0x1A)); write_number(static_cast(j.m_data.m_value.number_unsigned)); } else { - oa->write_character(to_char_type(0x1B)); + oa.write_character(to_char_type(0x1B)); write_number(static_cast(j.m_data.m_value.number_unsigned)); } break; @@ -18967,16 +19091,16 @@ class binary_writer if (std::isnan(j.m_data.m_value.number_float)) { // NaN is 0xf97e00 in CBOR - oa->write_character(to_char_type(0xF9)); - oa->write_character(to_char_type(0x7E)); - oa->write_character(to_char_type(0x00)); + oa.write_character(to_char_type(0xF9)); + oa.write_character(to_char_type(0x7E)); + oa.write_character(to_char_type(0x00)); } else if (std::isinf(j.m_data.m_value.number_float)) { // Infinity is 0xf97c00, -Infinity is 0xf9fc00 - oa->write_character(to_char_type(0xf9)); - oa->write_character(j.m_data.m_value.number_float > 0 ? to_char_type(0x7C) : to_char_type(0xFC)); - oa->write_character(to_char_type(0x00)); + oa.write_character(to_char_type(0xf9)); + oa.write_character(j.m_data.m_value.number_float > 0 ? to_char_type(0x7C) : to_char_type(0xFC)); + oa.write_character(to_char_type(0x00)); } else { @@ -18995,31 +19119,31 @@ class binary_writer } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x78)); + oa.write_character(to_char_type(0x78)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x79)); + oa.write_character(to_char_type(0x79)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x7A)); + oa.write_character(to_char_type(0x7A)); write_number(static_cast(N)); } // LCOV_EXCL_START else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x7B)); + oa.write_character(to_char_type(0x7B)); write_number(static_cast(N)); } // LCOV_EXCL_STOP // step 2: write the string - oa->write_characters( - reinterpret_cast(j.m_data.m_value.string->data()), - j.m_data.m_value.string->size()); + oa.write_characters( + reinterpret_cast(j.m_data.m_value.string->data()), + j.m_data.m_value.string->size()); break; } @@ -19033,23 +19157,23 @@ class binary_writer } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x98)); + oa.write_character(to_char_type(0x98)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x99)); + oa.write_character(to_char_type(0x99)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x9A)); + oa.write_character(to_char_type(0x9A)); write_number(static_cast(N)); } // LCOV_EXCL_START else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x9B)); + oa.write_character(to_char_type(0x9B)); write_number(static_cast(N)); } // LCOV_EXCL_STOP @@ -19096,31 +19220,31 @@ class binary_writer } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x58)); + oa.write_character(to_char_type(0x58)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x59)); + oa.write_character(to_char_type(0x59)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x5A)); + oa.write_character(to_char_type(0x5A)); write_number(static_cast(N)); } // LCOV_EXCL_START else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x5B)); + oa.write_character(to_char_type(0x5B)); write_number(static_cast(N)); } // LCOV_EXCL_STOP // step 2: write each element - oa->write_characters( - reinterpret_cast(j.m_data.m_value.binary->data()), - N); + oa.write_characters( + reinterpret_cast(j.m_data.m_value.binary->data()), + N); break; } @@ -19135,23 +19259,23 @@ class binary_writer } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0xB8)); + oa.write_character(to_char_type(0xB8)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0xB9)); + oa.write_character(to_char_type(0xB9)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0xBA)); + oa.write_character(to_char_type(0xBA)); write_number(static_cast(N)); } // LCOV_EXCL_START else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0xBB)); + oa.write_character(to_char_type(0xBB)); write_number(static_cast(N)); } // LCOV_EXCL_STOP @@ -19180,15 +19304,15 @@ class binary_writer { case value_t::null: // nil { - oa->write_character(to_char_type(0xC0)); + oa.write_character(to_char_type(0xC0)); break; } case value_t::boolean: // true and false { - oa->write_character(j.m_data.m_value.boolean - ? to_char_type(0xC3) - : to_char_type(0xC2)); + oa.write_character(j.m_data.m_value.boolean + ? to_char_type(0xC3) + : to_char_type(0xC2)); break; } @@ -19207,25 +19331,25 @@ class binary_writer else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 8 - oa->write_character(to_char_type(0xCC)); + oa.write_character(to_char_type(0xCC)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 16 - oa->write_character(to_char_type(0xCD)); + oa.write_character(to_char_type(0xCD)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 32 - oa->write_character(to_char_type(0xCE)); + oa.write_character(to_char_type(0xCE)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 64 - oa->write_character(to_char_type(0xCF)); + oa.write_character(to_char_type(0xCF)); write_number(static_cast(j.m_data.m_value.number_integer)); } } @@ -19240,28 +19364,28 @@ class binary_writer j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { // int 8 - oa->write_character(to_char_type(0xD0)); + oa.write_character(to_char_type(0xD0)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { // int 16 - oa->write_character(to_char_type(0xD1)); + oa.write_character(to_char_type(0xD1)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { // int 32 - oa->write_character(to_char_type(0xD2)); + oa.write_character(to_char_type(0xD2)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { // int 64 - oa->write_character(to_char_type(0xD3)); + oa.write_character(to_char_type(0xD3)); write_number(static_cast(j.m_data.m_value.number_integer)); } } @@ -19278,25 +19402,25 @@ class binary_writer else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 8 - oa->write_character(to_char_type(0xCC)); + oa.write_character(to_char_type(0xCC)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 16 - oa->write_character(to_char_type(0xCD)); + oa.write_character(to_char_type(0xCD)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 32 - oa->write_character(to_char_type(0xCE)); + oa.write_character(to_char_type(0xCE)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 64 - oa->write_character(to_char_type(0xCF)); + oa.write_character(to_char_type(0xCF)); write_number(static_cast(j.m_data.m_value.number_integer)); } break; @@ -19320,26 +19444,26 @@ class binary_writer else if (N <= (std::numeric_limits::max)()) { // str 8 - oa->write_character(to_char_type(0xD9)); + oa.write_character(to_char_type(0xD9)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { // str 16 - oa->write_character(to_char_type(0xDA)); + oa.write_character(to_char_type(0xDA)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { // str 32 - oa->write_character(to_char_type(0xDB)); + oa.write_character(to_char_type(0xDB)); write_number(static_cast(N)); } // step 2: write the string - oa->write_characters( - reinterpret_cast(j.m_data.m_value.string->data()), - j.m_data.m_value.string->size()); + oa.write_characters( + reinterpret_cast(j.m_data.m_value.string->data()), + j.m_data.m_value.string->size()); break; } @@ -19355,13 +19479,13 @@ class binary_writer else if (N <= (std::numeric_limits::max)()) { // array 16 - oa->write_character(to_char_type(0xDC)); + oa.write_character(to_char_type(0xDC)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { // array 32 - oa->write_character(to_char_type(0xDD)); + oa.write_character(to_char_type(0xDD)); write_number(static_cast(N)); } @@ -19417,7 +19541,7 @@ class binary_writer fixed = false; } - oa->write_character(to_char_type(output_type)); + oa.write_character(to_char_type(output_type)); if (!fixed) { write_number(static_cast(N)); @@ -19429,7 +19553,7 @@ class binary_writer ? 0xC8 // ext 16 : 0xC5; // bin 16 - oa->write_character(to_char_type(output_type)); + oa.write_character(to_char_type(output_type)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) @@ -19438,7 +19562,7 @@ class binary_writer ? 0xC9 // ext 32 : 0xC6; // bin 32 - oa->write_character(to_char_type(output_type)); + oa.write_character(to_char_type(output_type)); write_number(static_cast(N)); } @@ -19454,9 +19578,9 @@ class binary_writer } // step 2: write the byte string - oa->write_characters( - reinterpret_cast(j.m_data.m_value.binary->data()), - N); + oa.write_characters( + reinterpret_cast(j.m_data.m_value.binary->data()), + N); break; } @@ -19473,13 +19597,13 @@ class binary_writer else if (N <= (std::numeric_limits::max)()) { // map 16 - oa->write_character(to_char_type(0xDE)); + oa.write_character(to_char_type(0xDE)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { // map 32 - oa->write_character(to_char_type(0xDF)); + oa.write_character(to_char_type(0xDF)); write_number(static_cast(N)); } @@ -19518,7 +19642,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('Z')); + oa.write_character(to_char_type('Z')); } break; } @@ -19527,9 +19651,9 @@ class binary_writer { if (add_prefix) { - oa->write_character(j.m_data.m_value.boolean - ? to_char_type('T') - : to_char_type('F')); + oa.write_character(j.m_data.m_value.boolean + ? to_char_type('T') + : to_char_type('F')); } break; } @@ -19556,12 +19680,12 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('S')); + oa.write_character(to_char_type('S')); } write_number_with_ubjson_prefix(j.m_data.m_value.string->size(), true, use_bjdata); - oa->write_characters( - reinterpret_cast(j.m_data.m_value.string->data()), - j.m_data.m_value.string->size()); + oa.write_characters( + reinterpret_cast(j.m_data.m_value.string->data()), + j.m_data.m_value.string->size()); break; } @@ -19569,7 +19693,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('[')); + oa.write_character(to_char_type('[')); } bool prefix_required = true; @@ -19601,14 +19725,14 @@ class binary_writer && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) { prefix_required = false; - oa->write_character(to_char_type('$')); - oa->write_character(first_prefix); + oa.write_character(to_char_type('$')); + oa.write_character(first_prefix); } } if (use_count) { - oa->write_character(to_char_type('#')); + oa.write_character(to_char_type('#')); write_number_with_ubjson_prefix(j.m_data.m_value.array->size(), true, use_bjdata); } @@ -19619,7 +19743,7 @@ class binary_writer if (!use_count) { - oa->write_character(to_char_type(']')); + oa.write_character(to_char_type(']')); } break; @@ -19629,7 +19753,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('[')); + oa.write_character(to_char_type('[')); } if (use_type && (bjdata_draft3 || !j.m_data.m_value.binary->empty())) @@ -19638,36 +19762,36 @@ class binary_writer { JSON_THROW(other_error::create(502, "use_type requires use_size = true", &j)); } - oa->write_character(to_char_type('$')); - oa->write_character(bjdata_draft3 ? 'B' : 'U'); + oa.write_character(to_char_type('$')); + oa.write_character(bjdata_draft3 ? 'B' : 'U'); } if (use_count) { - oa->write_character(to_char_type('#')); + oa.write_character(to_char_type('#')); write_number_with_ubjson_prefix(j.m_data.m_value.binary->size(), true, use_bjdata); } if (use_type) { - oa->write_characters( - reinterpret_cast(j.m_data.m_value.binary->data()), - j.m_data.m_value.binary->size()); + oa.write_characters( + reinterpret_cast(j.m_data.m_value.binary->data()), + j.m_data.m_value.binary->size()); } else { for (size_t i = 0; i < j.m_data.m_value.binary->size(); ++i) { - oa->write_character(to_char_type(bjdata_draft3 ? 'B' : 'U')); + oa.write_character(to_char_type(bjdata_draft3 ? 'B' : 'U')); // the cast is needed for binary types whose value type // is not an integer (e.g., std::byte) - oa->write_character(to_char_type(static_cast(j.m_data.m_value.binary->data()[i]))); + oa.write_character(to_char_type(static_cast(j.m_data.m_value.binary->data()[i]))); } } if (!use_count) { - oa->write_character(to_char_type(']')); + oa.write_character(to_char_type(']')); } break; @@ -19685,7 +19809,7 @@ class binary_writer if (add_prefix) { - oa->write_character(to_char_type('{')); + oa.write_character(to_char_type('{')); } bool prefix_required = true; @@ -19707,29 +19831,29 @@ class binary_writer if (same_prefix && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) { prefix_required = false; - oa->write_character(to_char_type('$')); - oa->write_character(first_prefix); + oa.write_character(to_char_type('$')); + oa.write_character(first_prefix); } } if (use_count) { - oa->write_character(to_char_type('#')); + oa.write_character(to_char_type('#')); write_number_with_ubjson_prefix(j.m_data.m_value.object->size(), true, use_bjdata); } for (const auto& el : *j.m_data.m_value.object) { write_number_with_ubjson_prefix(el.first.size(), true, use_bjdata); - oa->write_characters( - reinterpret_cast(el.first.data()), - el.first.size()); + oa.write_characters( + reinterpret_cast(el.first.data()), + el.first.size()); write_ubjson(el.second, use_count, use_type, prefix_required, use_bjdata, bjdata_version); } if (!use_count) { - oa->write_character(to_char_type('}')); + oa.write_character(to_char_type('}')); } break; @@ -19783,13 +19907,13 @@ class binary_writer void write_bson_entry_header(const string_t& name, const std::uint8_t element_type) { - oa->write_character(to_char_type(element_type)); - oa->write_characters( - reinterpret_cast(name.data()), - name.size()); + oa.write_character(to_char_type(element_type)); + oa.write_characters( + reinterpret_cast(name.data()), + name.size()); // the terminating null byte is written explicitly rather than taken // from the buffer, so that string_t::data() need not be null-terminated - oa->write_character(to_char_type(0x00)); + oa.write_character(to_char_type(0x00)); } /*! @@ -19799,7 +19923,7 @@ class binary_writer const bool value) { write_bson_entry_header(name, 0x08); - oa->write_character(value ? to_char_type(0x01) : to_char_type(0x00)); + oa.write_character(value ? to_char_type(0x01) : to_char_type(0x00)); } /*! @@ -19829,12 +19953,12 @@ class binary_writer write_bson_entry_header(name, 0x02); write_number(to_bson_length(value.size() + 1ul), true); - oa->write_characters( - reinterpret_cast(value.data()), - value.size()); + oa.write_characters( + reinterpret_cast(value.data()), + value.size()); // the terminating null byte is written explicitly rather than taken // from the buffer, so that string_t::data() need not be null-terminated - oa->write_character(to_char_type(0x00)); + oa.write_character(to_char_type(0x00)); } /*! @@ -19963,7 +20087,7 @@ class binary_writer write_bson_element(string_t(key.data(), key.size()), el); } - oa->write_character(to_char_type(0x00)); + oa.write_character(to_char_type(0x00)); } /*! @@ -19983,7 +20107,7 @@ class binary_writer write_number(value.has_subtype() ? static_cast(value.subtype()) : static_cast(0x00)); - oa->write_characters(reinterpret_cast(value.data()), value.size()); + oa.write_characters(reinterpret_cast(value.data()), value.size()); } /*! @@ -20109,7 +20233,7 @@ class binary_writer write_bson_element(el.first, el.second); } - oa->write_character(to_char_type(0x00)); + oa.write_character(to_char_type(0x00)); } ////////// @@ -20153,7 +20277,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(get_ubjson_float_prefix(n)); + oa.write_character(get_ubjson_float_prefix(n)); } write_number(n, use_bjdata); } @@ -20169,7 +20293,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('i')); // int8 + oa.write_character(to_char_type('i')); // int8 } write_number(static_cast(n), use_bjdata); } @@ -20177,7 +20301,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('U')); // uint8 + oa.write_character(to_char_type('U')); // uint8 } write_number(static_cast(n), use_bjdata); } @@ -20185,7 +20309,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('I')); // int16 + oa.write_character(to_char_type('I')); // int16 } write_number(static_cast(n), use_bjdata); } @@ -20193,7 +20317,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('u')); // uint16 - bjdata only + oa.write_character(to_char_type('u')); // uint16 - bjdata only } write_number(static_cast(n), use_bjdata); } @@ -20201,7 +20325,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('l')); // int32 + oa.write_character(to_char_type('l')); // int32 } write_number(static_cast(n), use_bjdata); } @@ -20209,7 +20333,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('m')); // uint32 - bjdata only + oa.write_character(to_char_type('m')); // uint32 - bjdata only } write_number(static_cast(n), use_bjdata); } @@ -20217,7 +20341,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('L')); // int64 + oa.write_character(to_char_type('L')); // int64 } write_number(static_cast(n), use_bjdata); } @@ -20225,7 +20349,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('M')); // uint64 - bjdata only + oa.write_character(to_char_type('M')); // uint64 - bjdata only } write_number(static_cast(n), use_bjdata); } @@ -20233,14 +20357,14 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('H')); // high-precision number + oa.write_character(to_char_type('H')); // high-precision number } const auto number = BasicJsonType(n).dump(); write_number_with_ubjson_prefix(number.size(), true, use_bjdata); for (std::size_t i = 0; i < number.size(); ++i) { - oa->write_character(to_char_type(static_cast(number[i]))); + oa.write_character(to_char_type(static_cast(number[i]))); } } } @@ -20257,7 +20381,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('i')); // int8 + oa.write_character(to_char_type('i')); // int8 } write_number(static_cast(n), use_bjdata); } @@ -20265,7 +20389,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('U')); // uint8 + oa.write_character(to_char_type('U')); // uint8 } write_number(static_cast(n), use_bjdata); } @@ -20273,7 +20397,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('I')); // int16 + oa.write_character(to_char_type('I')); // int16 } write_number(static_cast(n), use_bjdata); } @@ -20281,7 +20405,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('u')); // uint16 - bjdata only + oa.write_character(to_char_type('u')); // uint16 - bjdata only } write_number(static_cast(n), use_bjdata); } @@ -20289,7 +20413,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('l')); // int32 + oa.write_character(to_char_type('l')); // int32 } write_number(static_cast(n), use_bjdata); } @@ -20297,7 +20421,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('m')); // uint32 - bjdata only + oa.write_character(to_char_type('m')); // uint32 - bjdata only } write_number(static_cast(n), use_bjdata); } @@ -20305,7 +20429,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('L')); // int64 + oa.write_character(to_char_type('L')); // int64 } write_number(static_cast(n), use_bjdata); } @@ -20314,14 +20438,14 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('H')); // high-precision number + oa.write_character(to_char_type('H')); // high-precision number } const auto number = BasicJsonType(n).dump(); write_number_with_ubjson_prefix(number.size(), true, use_bjdata); for (std::size_t i = 0; i < number.size(); ++i) { - oa->write_character(to_char_type(static_cast(number[i]))); + oa.write_character(to_char_type(static_cast(number[i]))); } } // LCOV_EXCL_STOP @@ -20613,10 +20737,10 @@ class binary_writer } } - oa->write_character('['); - oa->write_character('$'); - oa->write_character(dtype); - oa->write_character('#'); + oa.write_character('['); + oa.write_character('$'); + oa.write_character(dtype); + oa.write_character('#'); key = "_ArraySize_"; write_ubjson(value.at(key), use_count, use_type, true, true, bjdata_version); @@ -20712,6 +20836,87 @@ class binary_writer On the other hand, BSON and BJData use little endian and should reorder on big endian systems. */ + // single-instruction byte swaps (compilers lower these to bswap/rev/movbe); + // used to emit big-endian numbers without a per-byte std::reverse loop + static std::uint16_t byte_swap(std::uint16_t x) noexcept + { +#if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap16(x); +#elif defined(_MSC_VER) + return _byteswap_ushort(x); +#else + return static_cast((x >> 8) | (x << 8)); +#endif + } + + static std::uint32_t byte_swap(std::uint32_t x) noexcept + { +#if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap32(x); +#elif defined(_MSC_VER) + return _byteswap_ulong(x); +#else + return ((x & 0x000000FFu) << 24) | ((x & 0x0000FF00u) << 8) + | ((x & 0x00FF0000u) >> 8) | ((x & 0xFF000000u) >> 24); +#endif + } + + static std::uint64_t byte_swap(std::uint64_t x) noexcept + { +#if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap64(x); +#elif defined(_MSC_VER) + return _byteswap_uint64(x); +#else + x = ((x & 0x00000000FFFFFFFFull) << 32) | ((x & 0xFFFFFFFF00000000ull) >> 32); + x = ((x & 0x0000FFFF0000FFFFull) << 16) | ((x & 0xFFFF0000FFFF0000ull) >> 16); + x = ((x & 0x00FF00FF00FF00FFull) << 8) | ((x & 0xFF00FF00FF00FF00ull) >> 8); + return x; +#endif + } + + /*! + @brief reverse the bytes of a buffer by byte-swapping it as UIntType + + Loading the buffer into an unsigned integer of the same width and swapping + that is what lets the compiler emit a single bswap/rev/movbe; reversing the + buffer element by element does not reliably get there (clang keeps a scalar + shuffle). The two memcpy calls are the only portable way to reinterpret the + bytes and are folded away by every optimizer. + */ + template + static void byte_swap_buffer(std::array& a) noexcept + { + static_assert(sizeof(UIntType) == N, "swap width must match the buffer size"); + UIntType v{}; + std::memcpy(&v, a.data(), sizeof(v)); + v = byte_swap(v); + std::memcpy(a.data(), &v, sizeof(v)); + } + + // reverse the bytes of a fixed-size buffer; a single byte_swap() for the + // common 2/4/8-byte number payloads, std::reverse for any other size + static void reverse_bytes(std::array& a) noexcept + { + byte_swap_buffer(a); + } + + static void reverse_bytes(std::array& a) noexcept + { + byte_swap_buffer(a); + } + + static void reverse_bytes(std::array& a) noexcept + { + byte_swap_buffer(a); + } + + template + static void reverse_bytes(std::array& a) noexcept + { + std::reverse(a.begin(), a.end()); + } + template void write_number(const NumberType n, const bool OutputIsLittleEndian = false) { @@ -20723,10 +20928,10 @@ class binary_writer if (is_little_endian != OutputIsLittleEndian) { // reverse byte order prior to conversion if necessary - std::reverse(vec.begin(), vec.end()); + reverse_bytes(vec); } - oa->write_characters(vec.data(), sizeof(NumberType)); + oa.write_characters(vec.data(), sizeof(NumberType)); } void write_compact_float(const number_float_t n, detail::input_format_t format) @@ -20734,21 +20939,30 @@ class binary_writer #ifdef __GNUC__ JSON_HEDLEY_DIAGNOSTIC_PUSH JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") +#endif + // When number_float_t is float, static_cast(n) is the identity and + // both branches below are intentionally identical (the "compact" float + // representation is the value itself). Only GCC diagnoses this, and only + // when the sink calls are inlined; clang has no such warning. + // (-Wduplicated-branches only exists from GCC 7 on; naming it on an older + // GCC would itself warn under -Wpragmas) +#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ >= 7) + JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wduplicated-branches") #endif if (!std::isfinite(n) || ((static_cast(n) >= static_cast(std::numeric_limits::lowest()) && static_cast(n) <= static_cast((std::numeric_limits::max)()) && static_cast(static_cast(n)) == static_cast(n)))) { - oa->write_character(format == detail::input_format_t::cbor - ? get_cbor_float_prefix(static_cast(n)) - : get_msgpack_float_prefix(static_cast(n))); + oa.write_character(format == detail::input_format_t::cbor + ? get_cbor_float_prefix(static_cast(n)) + : get_msgpack_float_prefix(static_cast(n))); write_number(static_cast(n)); } else { - oa->write_character(format == detail::input_format_t::cbor - ? get_cbor_float_prefix(n) - : get_msgpack_float_prefix(n)); + oa.write_character(format == detail::input_format_t::cbor + ? get_cbor_float_prefix(n) + : get_msgpack_float_prefix(n)); write_number(n); } #ifdef __GNUC__ @@ -20815,7 +21029,7 @@ class binary_writer const bool is_little_endian = little_endianness(); /// the output - output_adapter_t oa = nullptr; + OutputSinkType oa; }; } // namespace detail @@ -24156,7 +24370,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec friend ::nlohmann::detail::serializer; template friend class ::nlohmann::detail::iter_impl; - template + template friend class ::nlohmann::detail::binary_writer; template friend class ::nlohmann::detail::binary_reader; @@ -24204,6 +24418,14 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec template using binary_reader = ::nlohmann::detail::binary_reader; template using binary_writer = ::nlohmann::detail::binary_writer; + // binary_writer over a concrete (non-virtual) sink appending into a std::vector, + // used by the vector-returning to_* overloads + template using vector_binary_writer = + ::nlohmann::detail::binary_writer>; + template static vector_binary_writer vector_writer(std::vector& v) + { + return vector_binary_writer(::nlohmann::detail::output_vector_sink(v)); + } JSON_PRIVATE_UNLESS_TESTED: using serializer = ::nlohmann::detail::serializer; @@ -28460,7 +28682,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec static std::vector to_cbor(const basic_json& j) { std::vector result; - to_cbor(j, result); + result.reserve(detail::binary_reserve_hint(j)); + vector_writer(result).write_cbor(j); return result; } @@ -28483,7 +28706,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec static std::vector to_msgpack(const basic_json& j) { std::vector result; - to_msgpack(j, result); + result.reserve(detail::binary_reserve_hint(j)); + vector_writer(result).write_msgpack(j); return result; } @@ -28508,7 +28732,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool use_type = false) { std::vector result; - to_ubjson(j, result, use_size, use_type); + result.reserve(detail::binary_reserve_hint(j)); + vector_writer(result).write_ubjson(j, use_size, use_type); return result; } @@ -28536,7 +28761,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bjdata_version_t version = bjdata_version_t::draft2) { std::vector result; - to_bjdata(j, result, use_size, use_type, version); + result.reserve(detail::binary_reserve_hint(j)); + vector_writer(result).write_ubjson(j, use_size, use_type, true, true, version); return result; } @@ -28563,7 +28789,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec static std::vector to_bson(const basic_json& j) { std::vector result; - to_bson(j, result); + result.reserve(detail::binary_reserve_hint(j)); + vector_writer(result).write_bson(j); return result; } diff --git a/tests/src/unit-binary_writer_sinks.cpp b/tests/src/unit-binary_writer_sinks.cpp new file mode 100644 index 000000000..f60e1bf51 --- /dev/null +++ b/tests/src/unit-binary_writer_sinks.cpp @@ -0,0 +1,198 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; + +#include +#include +#include + +namespace +{ + +// a spread of values exercising every writer path: scalars of each width, the +// float paths, strings, binary, and containers big enough to reallocate +std::vector test_values() +{ + json big_array = json::array(); + for (int i = 0; i < 5000; ++i) + { + big_array.push_back(i); + } + + json big_object = json::object(); + for (int i = 0; i < 1000; ++i) + { + big_object[std::to_string(i)] = i; + } + + return + { + json(nullptr), json(true), json(false), + json(0), json(-1), json(255), json(-129), json(65535), json(-32769), + json(4294967295U), json(-2147483649LL), json(18446744073709551615ULL), + json(0.0), json(-0.5), json(3.1415926535897932), + json(""), json("hello"), json(std::string(1000, 'x')), + json::binary({0x00, 0x01, 0x02}, 42), + json::array(), json::object(), + json::array({1, 2, 3}), json({{"a", 1}, {"b", nullptr}}), + json({{"nested", {{"deep", json::array({1, "two", 3.0, nullptr})}}}}), + big_array, big_object + }; +} + +// values to_bson() accepts: the document must be an object +std::vector bson_values() +{ + json big_object = json::object(); + for (int i = 0; i < 1000; ++i) + { + big_object[std::to_string(i)] = i; + } + + return + { + json::object(), + json({{"a", 1}, {"b", nullptr}, {"c", true}, {"d", 2.5}, {"e", "text"}}), + json({{"arr", json::array({1, 2, 3})}, {"obj", {{"k", "v"}}}}), + big_object + }; +} + +} // namespace + +// The vector-returning to_*(j) overloads write through the non-virtual +// output_vector_sink, while to_*(j, adapter) goes through output_adapter_sink. +// The two are separate code paths that must stay byte-for-byte identical; these +// checks fail if either overload is ever changed without the other. +TEST_CASE("binary writer output sinks") +{ + SECTION("vector sink and adapter sink agree") + { + // note: no SUBCASE inside these loops - doctest keys subcases by + // name/file/line, so a subcase in a loop body would only ever run for + // the first iteration + for (const auto& j : test_values()) + { + CAPTURE(j.dump(-1, ' ', false, json::error_handler_t::replace)); + + std::vector cbor; + json::to_cbor(j, cbor); + CHECK(json::to_cbor(j) == cbor); + + std::vector msgpack; + json::to_msgpack(j, msgpack); + CHECK(json::to_msgpack(j) == msgpack); + + for (const bool use_size : + { + false, true + }) + { + for (const bool use_type : + { + false, true + }) + { + if (use_type && !use_size) + { + continue; // not a supported combination + } + CAPTURE(use_size); + CAPTURE(use_type); + std::vector ubjson; + json::to_ubjson(j, ubjson, use_size, use_type); + CHECK(json::to_ubjson(j, use_size, use_type) == ubjson); + } + } + + for (const auto version : + { + json::bjdata_version_t::draft2, json::bjdata_version_t::draft3 + }) + { + std::vector bjdata; + json::to_bjdata(j, bjdata, false, false, version); + CHECK(json::to_bjdata(j, false, false, version) == bjdata); + } + } + + for (const auto& j : bson_values()) + { + CAPTURE(j.dump()); + std::vector bson; + json::to_bson(j, bson); + CHECK(json::to_bson(j) == bson); + } + } + + SECTION("the char adapter produces the same bytes") + { + for (const auto& j : test_values()) + { + CAPTURE(j.dump(-1, ' ', false, json::error_handler_t::replace)); + + const std::vector expected = json::to_cbor(j); + std::vector as_char; + json::to_cbor(j, as_char); + + REQUIRE(as_char.size() == expected.size()); + std::vector as_bytes; + as_bytes.reserve(as_char.size()); + for (const char c : as_char) + { + as_bytes.push_back(static_cast(c)); + } + CHECK(as_bytes == expected); + } + } +} + +// binary_reserve_hint() is documented as a *lower* bound on the serialized size, +// so that reserving it up front can never leave the returned vector holding +// capacity beyond what the value actually needs. +TEST_CASE("binary_reserve_hint never over-reserves") +{ + for (const auto& j : test_values()) + { + CAPTURE(j.dump(-1, ' ', false, json::error_handler_t::replace)); + + const std::size_t hint = nlohmann::detail::binary_reserve_hint(j); + + CHECK(hint <= json::to_cbor(j).size()); + CHECK(hint <= json::to_msgpack(j).size()); + CHECK(hint <= json::to_ubjson(j).size()); + CHECK(hint <= json::to_ubjson(j, true, true).size()); + CHECK(hint <= json::to_bjdata(j).size()); + } + + for (const auto& j : bson_values()) + { + CAPTURE(j.dump()); + CHECK(nlohmann::detail::binary_reserve_hint(j) <= json::to_bson(j).size()); + } + + SECTION("scalars get no hint") + { + CHECK(nlohmann::detail::binary_reserve_hint(json(nullptr)) == 0); + CHECK(nlohmann::detail::binary_reserve_hint(json(42)) == 0); + CHECK(nlohmann::detail::binary_reserve_hint(json("a string")) == 0); + CHECK(nlohmann::detail::binary_reserve_hint(json::binary({0x01})) == 0); + } + + SECTION("containers are hinted from their element count") + { + CHECK(nlohmann::detail::binary_reserve_hint(json::array()) == 1); + CHECK(nlohmann::detail::binary_reserve_hint(json::array({1, 2, 3})) == 4); + CHECK(nlohmann::detail::binary_reserve_hint(json::object()) == 1); + CHECK(nlohmann::detail::binary_reserve_hint(json({{"a", 1}, {"b", 2}})) == 5); + } +}