diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 117391edd..d69c88137 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -6336,21 +6336,256 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec { // the patch basic_json result(value_t::array); + diff_recursively(result, source, target, path, 0); + return result; + } - // if the values are the same, return an empty patch + private: + /// @brief two arrays or two objects @ref diff_iteratively is diffing + struct diff_frame + { + diff_frame(const basic_json* source_, const basic_json* target_, const std::size_t path_length_) noexcept + : source(source_), target(target_), path_length(path_length_) + {} + + // declared for GCC's -Weffc++, which asks for them in a class with + // pointer members and a non-trivial destructor; the exception + // specifications are left implicit, as GCC 4.8 rejects explicit ones + // that differ from them + diff_frame(const diff_frame&) = default; + diff_frame(diff_frame&&) = default; + diff_frame& operator=(const diff_frame&) = default; + diff_frame& operator=(diff_frame&&) = default; + ~diff_frame() = default; + + /// the values being diffed, both arrays or both objects + const basic_json* source; + const basic_json* target; + /// the length of their path in `current_path` + std::size_t path_length; + /// arrays: the next index to diff + std::size_t index = 0; + /// objects: the next member of source to look at + const_iterator member{}; // NOLINT(readability-redundant-member-init) + /// objects: the keys common to both, in source's order + std::vector common_keys{}; // NOLINT(readability-redundant-member-init) + /// objects: the next entry of common_keys + std::size_t next_common = 0; + /// objects: the "add" operations for keys only target has + basic_json added_ops{}; // NOLINT(readability-redundant-member-init) + }; + + // The operations of a diff are built by the functions below rather than + // where they are needed: building one takes several temporaries, and + // unoptimized builds give each temporary a stack slot of its own in the + // function it appears in. In diff_recursively, which is on the call stack + // once per nesting level, that made every level cost kilobytes of stack. + + /// @brief append a "replace" operation for @a path with @a value to @a result + static void diff_replace(basic_json& result, const string_t& path, const basic_json& value) + { + result.push_back( + { + {"op", "replace"}, {"path", path}, {"value", value} + }); + } + + /// @brief append a "remove" operation for @a path to @a result + static void diff_remove(basic_json& result, const string_t& path) + { + result.push_back(object( + { + {"op", "remove"}, {"path", path} + })); + } + + /// @brief append an "add" operation for @a path with @a value to @a result + static void diff_add(basic_json& result, const string_t& path, const basic_json& value) + { + result.push_back( + { + {"op", "add"}, {"path", path}, {"value", value} + }); + } + + /// @brief append the "remove" operations for the elements of array + /// @a source from @a index on, and the "add" operations for the + /// elements of array @a target from source's size on, to @a result + static void diff_array_tails(basic_json& result, const basic_json& source, const basic_json& target, + const string_t& path, const std::size_t index) + { + // remove my remaining elements, highest index first; appending + // in that order avoids the quadratic reinsertion done before + for (std::size_t j = source.size(); j > index; --j) + { + diff_remove(result, detail::concat(path, '/', detail::to_string(j - 1))); + } + + // add other remaining elements + for (std::size_t i = source.size(); i < target.size(); ++i) + { + diff_add(result, detail::concat(path, "/-"), target[i]); + } + } + + /*! + @brief compare the keys of objects @a source and @a target + + If object_t does not keep its members in insertion order, or if the keys + both objects have are in the same order in both, and the keys only + @a target has come after them, stores the keys common to both in + source's order in @a common_keys, stores the "add" operations for the keys + only @a target has in @a added_ops, and returns true: the caller then diffs + the objects member by member. Otherwise, appends operations that remove + every member of @a source and add every member of @a target to @a result, + and returns false. + */ + static bool diff_object_keys(basic_json& result, const basic_json& source, const basic_json& target, + const string_t& path, std::vector& common_keys, + basic_json& added_ops) + { + // first pass: record, for every source key, whether it is + // common to both objects (in source's iteration order) or + // was deleted (i.e., in source but not in target) -- this is + // a by-product of the target.find() call already needed to + // tell the two cases apart, so it adds no extra lookups. The + // "remove" ops themselves are emitted later, interleaved + // with the per-key diffs in the caller's fast path, to match + // source's original iteration order (as the original, + // pre-reordering-aware implementation did) instead of + // grouping all removes before all per-key diffs. + std::vector common_keys_source_order; + for (auto it = source.cbegin(); it != source.cend(); ++it) + { + if (target.find(it.key()) != target.end()) + { + common_keys_source_order.push_back(it.key()); + } + } + + // second pass: find keys that were added (i.e., in target but + // not in source), and record the keys common to both, in + // target's iteration order -- again a by-product of the + // source.find() call already needed to detect added keys. At + // the same time, determine whether every added key comes + // after every common key in target's order (a precondition + // for the fast path, which only ever appends new keys + // at the very end). Both are only needed for an object_t that + // keeps its members in insertion order, such as the one + // backing `ordered_json`; for any other object_t, the fast + // path is always taken and they are not computed. + // The patch ops for keys that were added (i.e., in target but not + // in source) are built here so the fast path can reuse + // them without a second source.find() per target key. Only + // used by the fast path -- the slow (reordering) path + // rebuilds "add" ops for every key itself. + std::vector common_keys_target_order; + bool new_keys_form_suffix = true; + bool seen_new_key = false; + for (auto it = target.cbegin(); it != target.cend(); ++it) + { + if (source.find(it.key()) == source.end()) + { + seen_new_key = true; + diff_add(added_ops, detail::concat(path, '/', detail::escape(it.key())), it.value()); + } + else + { +#ifdef JSON_HEDLEY_MSVC_VERSION +#pragma warning(push ) +#pragma warning(disable : 4127) // ignore warning to replace if with if constexpr +#endif + if (detail::is_ordered_map::value) + { + common_keys_target_order.push_back(it.key()); + if (seen_new_key) + { + new_keys_form_suffix = false; + } + } +#ifdef JSON_HEDLEY_MSVC_VERSION +#pragma warning( pop ) +#endif + } + } + + // Only an object type that keeps its members in insertion + // order, such as nlohmann::ordered_map, can need reordering: + // patch() appends a new member at the end of such an object. + // Any other object type places its members itself - std::map + // in key order, a hash map in an order its operator== ignores - + // so a member-by-member diff always reproduces target there. + if (!detail::is_ordered_map::value + || (common_keys_source_order == common_keys_target_order && new_keys_form_suffix)) + { + // fast path: order of common keys already matches (or the + // object_t's iteration order does not depend on + // insertion history), so a plain per-key diff is correct + // and minimal, as before + common_keys = std::move(common_keys_source_order); + return true; + } + + // slow path: the common keys are in a different relative + // order in source and target (only possible for a + // reorderable object_t like ordered_map). Building a + // minimal reordering patch is a nontrivial (LCS-like) + // problem; instead, remove every source key -- both + // deleted keys (which must be removed regardless) and + // common keys (removed so they can be re-added in + // target's order) -- and re-add every key that should + // remain, with its final target value, in target's + // order. basic_json::patch()'s "add" operation on an + // object uses operator[], which appends at the end for a + // vector-backed insertion-ordered map when the key does + // not already exist -- so removing a key and then adding + // it moves it to the end, fixing its position. + for (auto it = source.cbegin(); it != source.cend(); ++it) + { + diff_remove(result, detail::concat(path, '/', detail::escape(it.key()))); + } + + // add every key that is either common (just removed + // above) or brand new, in target's iteration order, so + // that the final order after applying the patch matches + // target exactly + for (auto it = target.cbegin(); it != target.cend(); ++it) + { + diff_add(result, detail::concat(path, '/', detail::escape(it.key())), it.value()); + } + return false; + } + + /*! + @brief @ref diff, for values at nesting level @a depth, appending the + operations to @a result + + Diffing two arrays or objects calls this function again, once per nesting + level, so values nested deeply enough used to exhaust the call stack and + terminate the process. The descent is bounded here: once @ref + detail::recursion_depth_limit levels have been entered, @ref + diff_iteratively diffs what is left without the call stack. + */ + static void diff_recursively(basic_json& result, const basic_json& source, const basic_json& target, + const string_t& path, const std::size_t depth) + { + // if the values are the same, there is nothing to do if (source == target) { - return result; + return; + } + + if (JSON_HEDLEY_UNLIKELY(depth >= detail::recursion_depth_limit())) + { + diff_iteratively(result, source, target, path); + return; } if (source.type() != target.type()) { // different types: replace value - result.push_back( - { - {"op", "replace"}, {"path", path}, {"value", target} - }); - return result; + diff_replace(result, path, target); + return; } switch (source.type()) @@ -6362,200 +6597,50 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec while (i < source.size() && i < target.size()) { // recursive call to compare array values at index i - auto temp_diff = diff(source[i], target[i], detail::concat(path, '/', detail::to_string(i))); - result.insert(result.end(), temp_diff.begin(), temp_diff.end()); + diff_recursively(result, source[i], target[i], detail::concat(path, '/', detail::to_string(i)), depth + 1); ++i; } // We now reached the end of at least one array // in a second pass, traverse the remaining elements - - // remove my remaining elements, highest index first; appending - // in that order avoids the quadratic reinsertion done before - for (std::size_t j = source.size(); j > i; --j) - { - result.push_back(object( - { - {"op", "remove"}, - {"path", detail::concat(path, '/', detail::to_string(j - 1))} - })); - } - i = source.size(); - - // add other remaining elements - while (i < target.size()) - { - result.push_back( - { - {"op", "add"}, - {"path", detail::concat(path, "/-")}, - {"value", target[i]} - }); - ++i; - } - + diff_array_tails(result, source, target, path, i); break; } case value_t::object: { - // first pass: record, for every source key, whether it is - // common to both objects (in source's iteration order) or - // was deleted (i.e., in source but not in target) -- this is - // a by-product of the target.find() call already needed to - // tell the two cases apart, so it adds no extra lookups. The - // "remove" ops themselves are emitted later, interleaved - // with the recursive per-key diffs in the fast path below, - // to match source's original iteration order (as the - // original, pre-reordering-aware implementation did) instead - // of grouping all removes before all recursive diffs. - std::vector common_keys_source_order; - for (auto it = source.cbegin(); it != source.cend(); ++it) - { - if (target.find(it.key()) != target.end()) - { - common_keys_source_order.push_back(it.key()); - } - } - - // second pass: find keys that were added (i.e., in target but - // not in source), and record the keys common to both, in - // target's iteration order -- again a by-product of the - // source.find() call already needed to detect added keys. At - // the same time, determine whether every added key comes - // after every common key in target's order (a precondition - // for the fast path below, which only ever appends new keys - // at the very end). Both are only needed for an object_t that - // keeps its members in insertion order, such as the one - // backing `ordered_json`; for any other object_t, the fast - // path is always taken and they are not computed. - // patch ops for keys that were added (i.e., in target but not - // in source); built here so the fast path below can reuse - // them without a second source.find() per target key. Only - // used by the fast path -- the slow (reordering) path - // rebuilds "add" ops for every key itself. - std::vector common_keys_target_order; + std::vector common_keys; basic_json added_ops(value_t::array); - bool new_keys_form_suffix = true; - bool seen_new_key = false; - for (auto it = target.cbegin(); it != target.cend(); ++it) + if (diff_object_keys(result, source, target, path, common_keys, added_ops)) { - if (source.find(it.key()) == source.end()) - { - seen_new_key = true; - const auto path_key = detail::concat(path, '/', detail::escape(it.key())); - added_ops.push_back( - { - {"op", "add"}, {"path", path_key}, - {"value", it.value()} - }); - } - else - { -#ifdef JSON_HEDLEY_MSVC_VERSION -#pragma warning(push ) -#pragma warning(disable : 4127) // ignore warning to replace if with if constexpr -#endif - if (detail::is_ordered_map::value) - { - common_keys_target_order.push_back(it.key()); - if (seen_new_key) - { - new_keys_form_suffix = false; - } - } -#ifdef JSON_HEDLEY_MSVC_VERSION -#pragma warning( pop ) -#endif - } - } - - // Only an object type that keeps its members in insertion - // order, such as nlohmann::ordered_map, can need reordering: - // patch() appends a new member at the end of such an object. - // Any other object type places its members itself - std::map - // in key order, a hash map in an order its operator== ignores - - // so a member-by-member diff always reproduces target there. - if (!detail::is_ordered_map::value - || (common_keys_source_order == common_keys_target_order && new_keys_form_suffix)) - { - // fast path: order of common keys already matches (or the - // object_t's iteration order does not depend on - // insertion history), so a plain per-key recursive diff - // is correct and minimal, as before. common_keys_source_order - // is, by construction, the subsequence of source's keys - // that are common to both objects, in source's iteration - // order -- so it can be walked in lockstep with `source` - // using a cheap key comparison instead of another lookup. - // Deleted keys (those source keys not in common_keys_source_order) - // are interleaved here too, in source's original order, to - // match the historical (pre-reordering-aware) output order. - auto common_it = common_keys_source_order.cbegin(); + // fast path: common_keys is, by construction, the + // subsequence of source's keys that are common to both + // objects, in source's iteration order -- so it can be + // walked in lockstep with `source` using a cheap key + // comparison instead of another lookup. Deleted keys + // (those source keys not in common_keys) are interleaved + // here too, in source's original order, to match the + // historical (pre-reordering-aware) output order. + auto common_it = common_keys.cbegin(); for (auto it = source.cbegin(); it != source.cend(); ++it) { - if (common_it != common_keys_source_order.cend() && it.key() == *common_it) + if (common_it != common_keys.cend() && it.key() == *common_it) { - const auto path_key = detail::concat(path, '/', detail::escape(it.key())); - auto temp_diff = diff(it.value(), target[it.key()], path_key); - result.insert(result.end(), temp_diff.begin(), temp_diff.end()); + diff_recursively(result, it.value(), target[it.key()], detail::concat(path, '/', detail::escape(it.key())), depth + 1); ++common_it; } else { // found a key that is not in target -> remove it - const auto path_key = detail::concat(path, '/', detail::escape(it.key())); - result.push_back(object( - { - {"op", "remove"}, {"path", path_key} - })); + diff_remove(result, detail::concat(path, '/', detail::escape(it.key()))); } } - // append the "add" ops for brand-new keys collected above - // during the pass over target -- no second source.find() - // per target key needed + // append the "add" ops for brand-new keys collected by + // diff_object_keys -- no second source.find() per target + // key needed result.insert(result.end(), added_ops.begin(), added_ops.end()); } - else - { - // slow path: the common keys are in a different relative - // order in source and target (only possible for a - // reorderable object_t like ordered_map). Building a - // minimal reordering patch is a nontrivial (LCS-like) - // problem; instead, remove every source key -- both - // deleted keys (which must be removed regardless) and - // common keys (removed so they can be re-added in - // target's order) -- and re-add every key that should - // remain, with its final target value, in target's - // order. basic_json::patch()'s "add" operation on an - // object uses operator[], which appends at the end for a - // vector-backed insertion-ordered map when the key does - // not already exist -- so removing a key and then adding - // it moves it to the end, fixing its position. - for (auto it = source.cbegin(); it != source.cend(); ++it) - { - const auto path_key = detail::concat(path, '/', detail::escape(it.key())); - result.push_back(object( - { - {"op", "remove"}, {"path", path_key} - })); - } - - // add every key that is either common (just removed - // above) or brand new, in target's iteration order, so - // that the final order after applying the patch matches - // target exactly - for (auto it = target.cbegin(); it != target.cend(); ++it) - { - const auto path_key = detail::concat(path, '/', detail::escape(it.key())); - result.push_back( - { - {"op", "add"}, {"path", path_key}, - {"value", it.value()} - }); - } - } - break; } @@ -6570,16 +6655,170 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec default: { // both primitive types: replace value - result.push_back( - { - {"op", "replace"}, {"path", path}, {"value", target} - }); + diff_replace(result, path, target); break; } } - - return result; } + + /*! + @brief @ref diff without the call stack, appending the operations to + @a result + + Produces the same operations as @ref diff_recursively. Only reached for + values nested more deeply than @ref detail::recursion_depth_limit. + */ + static void diff_iteratively(basic_json& result, const basic_json& source, const basic_json& target, + const string_t& path) + { + // The arrays and objects being diffed are kept on an explicit stack, + // and every pair of elements is still diffed completely before the + // next one, so the operations come out in the same order as in + // diff_recursively. The path of the values being diffed is kept in + // one buffer that grows and shrinks with the stack, rather than in a + // new string per level. + std::vector stack; + string_t current_path = path; + + // diff `s` against `t`, whose path is current_path: primitives, + // values of different types, and objects whose members were reordered + // are handled right away; arrays and other objects get a frame + const auto enter = [&result, &stack, ¤t_path](const basic_json & s, const basic_json & t) + { + // if the values are the same, there is nothing to do. Arrays and + // objects are not compared up front: comparing them visits + // everything below them, so doing that at every level would take + // quadratic time in the nesting depth - equal ones yield no + // operations anyway. + if ((!s.is_structured() || !t.is_structured()) && s == t) + { + return; + } + + if (s.type() != t.type()) + { + // different types: replace value + diff_replace(result, current_path, t); + return; + } + + switch (s.type()) + { + case value_t::array: + { + stack.emplace_back(&s, &t, current_path.size()); + return; + } + + case value_t::object: + { + std::vector common_keys; + basic_json added_ops(value_t::array); + if (diff_object_keys(result, s, t, current_path, common_keys, added_ops)) + { + // fast path: the frame walks source in lockstep with + // common_keys, as diff_recursively does, and appends + // added_ops once all members are done + stack.emplace_back(&s, &t, current_path.size()); + stack.back().member = s.cbegin(); + stack.back().common_keys = std::move(common_keys); + stack.back().added_ops = std::move(added_ops); + } + return; + } + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + // both primitive types: replace value + diff_replace(result, current_path, t); + return; + } + } + }; + + enter(source, target); + while (!stack.empty()) + { + // the frame is copied out member by member and changed through + // stack.back(): enter() may push a frame and the end of the loop + // pops it, either of which would invalidate a reference to it + const basic_json* const s = stack.back().source; + const basic_json* const t = stack.back().target; + const std::size_t path_length = stack.back().path_length; + const std::size_t depth = stack.size(); + + if (s->is_array()) + { + const auto& source_array = *s->m_data.m_value.array; + const auto& target_array = *t->m_data.m_value.array; + + // first pass: traverse common elements + const std::size_t i = stack.back().index; + if (i < source_array.size() && i < target_array.size()) + { + ++stack.back().index; + detail::concat_into(current_path, '/', detail::to_string(i)); + enter(source_array[i], target_array[i]); + if (stack.size() == depth) + { + current_path.resize(path_length); + } + continue; + } + + // We now reached the end of at least one array + // in a second pass, traverse the remaining elements + diff_array_tails(result, *s, *t, current_path, i); + } + else + { + const const_iterator it = stack.back().member; + if (it != s->cend()) + { + ++stack.back().member; + const std::size_t next_common = stack.back().next_common; + if (next_common < stack.back().common_keys.size() && it.key() == stack.back().common_keys[next_common]) + { + ++stack.back().next_common; + const basic_json& target_value = (*t)[it.key()]; + detail::concat_into(current_path, '/', detail::escape(it.key())); + enter(it.value(), target_value); + if (stack.size() == depth) + { + current_path.resize(path_length); + } + } + else + { + // found a key that is not in target -> remove it + diff_remove(result, detail::concat(current_path, '/', detail::escape(it.key()))); + } + continue; + } + + // append the "add" ops for brand-new keys collected when the + // object was entered + result.insert(result.end(), stack.back().added_ops.begin(), stack.back().added_ops.end()); + } + + // this array or object is done: continue with the one it is in + stack.pop_back(); + if (!stack.empty()) + { + current_path.resize(stack.back().path_length); + } + } + } + + public: /// @} //////////////////////////////// diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index cacc8ea70..44e5201df 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -33083,21 +33083,256 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec { // the patch basic_json result(value_t::array); + diff_recursively(result, source, target, path, 0); + return result; + } - // if the values are the same, return an empty patch + private: + /// @brief two arrays or two objects @ref diff_iteratively is diffing + struct diff_frame + { + diff_frame(const basic_json* source_, const basic_json* target_, const std::size_t path_length_) noexcept + : source(source_), target(target_), path_length(path_length_) + {} + + // declared for GCC's -Weffc++, which asks for them in a class with + // pointer members and a non-trivial destructor; the exception + // specifications are left implicit, as GCC 4.8 rejects explicit ones + // that differ from them + diff_frame(const diff_frame&) = default; + diff_frame(diff_frame&&) = default; + diff_frame& operator=(const diff_frame&) = default; + diff_frame& operator=(diff_frame&&) = default; + ~diff_frame() = default; + + /// the values being diffed, both arrays or both objects + const basic_json* source; + const basic_json* target; + /// the length of their path in `current_path` + std::size_t path_length; + /// arrays: the next index to diff + std::size_t index = 0; + /// objects: the next member of source to look at + const_iterator member{}; // NOLINT(readability-redundant-member-init) + /// objects: the keys common to both, in source's order + std::vector common_keys{}; // NOLINT(readability-redundant-member-init) + /// objects: the next entry of common_keys + std::size_t next_common = 0; + /// objects: the "add" operations for keys only target has + basic_json added_ops{}; // NOLINT(readability-redundant-member-init) + }; + + // The operations of a diff are built by the functions below rather than + // where they are needed: building one takes several temporaries, and + // unoptimized builds give each temporary a stack slot of its own in the + // function it appears in. In diff_recursively, which is on the call stack + // once per nesting level, that made every level cost kilobytes of stack. + + /// @brief append a "replace" operation for @a path with @a value to @a result + static void diff_replace(basic_json& result, const string_t& path, const basic_json& value) + { + result.push_back( + { + {"op", "replace"}, {"path", path}, {"value", value} + }); + } + + /// @brief append a "remove" operation for @a path to @a result + static void diff_remove(basic_json& result, const string_t& path) + { + result.push_back(object( + { + {"op", "remove"}, {"path", path} + })); + } + + /// @brief append an "add" operation for @a path with @a value to @a result + static void diff_add(basic_json& result, const string_t& path, const basic_json& value) + { + result.push_back( + { + {"op", "add"}, {"path", path}, {"value", value} + }); + } + + /// @brief append the "remove" operations for the elements of array + /// @a source from @a index on, and the "add" operations for the + /// elements of array @a target from source's size on, to @a result + static void diff_array_tails(basic_json& result, const basic_json& source, const basic_json& target, + const string_t& path, const std::size_t index) + { + // remove my remaining elements, highest index first; appending + // in that order avoids the quadratic reinsertion done before + for (std::size_t j = source.size(); j > index; --j) + { + diff_remove(result, detail::concat(path, '/', detail::to_string(j - 1))); + } + + // add other remaining elements + for (std::size_t i = source.size(); i < target.size(); ++i) + { + diff_add(result, detail::concat(path, "/-"), target[i]); + } + } + + /*! + @brief compare the keys of objects @a source and @a target + + If object_t does not keep its members in insertion order, or if the keys + both objects have are in the same order in both, and the keys only + @a target has come after them, stores the keys common to both in + source's order in @a common_keys, stores the "add" operations for the keys + only @a target has in @a added_ops, and returns true: the caller then diffs + the objects member by member. Otherwise, appends operations that remove + every member of @a source and add every member of @a target to @a result, + and returns false. + */ + static bool diff_object_keys(basic_json& result, const basic_json& source, const basic_json& target, + const string_t& path, std::vector& common_keys, + basic_json& added_ops) + { + // first pass: record, for every source key, whether it is + // common to both objects (in source's iteration order) or + // was deleted (i.e., in source but not in target) -- this is + // a by-product of the target.find() call already needed to + // tell the two cases apart, so it adds no extra lookups. The + // "remove" ops themselves are emitted later, interleaved + // with the per-key diffs in the caller's fast path, to match + // source's original iteration order (as the original, + // pre-reordering-aware implementation did) instead of + // grouping all removes before all per-key diffs. + std::vector common_keys_source_order; + for (auto it = source.cbegin(); it != source.cend(); ++it) + { + if (target.find(it.key()) != target.end()) + { + common_keys_source_order.push_back(it.key()); + } + } + + // second pass: find keys that were added (i.e., in target but + // not in source), and record the keys common to both, in + // target's iteration order -- again a by-product of the + // source.find() call already needed to detect added keys. At + // the same time, determine whether every added key comes + // after every common key in target's order (a precondition + // for the fast path, which only ever appends new keys + // at the very end). Both are only needed for an object_t that + // keeps its members in insertion order, such as the one + // backing `ordered_json`; for any other object_t, the fast + // path is always taken and they are not computed. + // The patch ops for keys that were added (i.e., in target but not + // in source) are built here so the fast path can reuse + // them without a second source.find() per target key. Only + // used by the fast path -- the slow (reordering) path + // rebuilds "add" ops for every key itself. + std::vector common_keys_target_order; + bool new_keys_form_suffix = true; + bool seen_new_key = false; + for (auto it = target.cbegin(); it != target.cend(); ++it) + { + if (source.find(it.key()) == source.end()) + { + seen_new_key = true; + diff_add(added_ops, detail::concat(path, '/', detail::escape(it.key())), it.value()); + } + else + { +#ifdef JSON_HEDLEY_MSVC_VERSION +#pragma warning(push ) +#pragma warning(disable : 4127) // ignore warning to replace if with if constexpr +#endif + if (detail::is_ordered_map::value) + { + common_keys_target_order.push_back(it.key()); + if (seen_new_key) + { + new_keys_form_suffix = false; + } + } +#ifdef JSON_HEDLEY_MSVC_VERSION +#pragma warning( pop ) +#endif + } + } + + // Only an object type that keeps its members in insertion + // order, such as nlohmann::ordered_map, can need reordering: + // patch() appends a new member at the end of such an object. + // Any other object type places its members itself - std::map + // in key order, a hash map in an order its operator== ignores - + // so a member-by-member diff always reproduces target there. + if (!detail::is_ordered_map::value + || (common_keys_source_order == common_keys_target_order && new_keys_form_suffix)) + { + // fast path: order of common keys already matches (or the + // object_t's iteration order does not depend on + // insertion history), so a plain per-key diff is correct + // and minimal, as before + common_keys = std::move(common_keys_source_order); + return true; + } + + // slow path: the common keys are in a different relative + // order in source and target (only possible for a + // reorderable object_t like ordered_map). Building a + // minimal reordering patch is a nontrivial (LCS-like) + // problem; instead, remove every source key -- both + // deleted keys (which must be removed regardless) and + // common keys (removed so they can be re-added in + // target's order) -- and re-add every key that should + // remain, with its final target value, in target's + // order. basic_json::patch()'s "add" operation on an + // object uses operator[], which appends at the end for a + // vector-backed insertion-ordered map when the key does + // not already exist -- so removing a key and then adding + // it moves it to the end, fixing its position. + for (auto it = source.cbegin(); it != source.cend(); ++it) + { + diff_remove(result, detail::concat(path, '/', detail::escape(it.key()))); + } + + // add every key that is either common (just removed + // above) or brand new, in target's iteration order, so + // that the final order after applying the patch matches + // target exactly + for (auto it = target.cbegin(); it != target.cend(); ++it) + { + diff_add(result, detail::concat(path, '/', detail::escape(it.key())), it.value()); + } + return false; + } + + /*! + @brief @ref diff, for values at nesting level @a depth, appending the + operations to @a result + + Diffing two arrays or objects calls this function again, once per nesting + level, so values nested deeply enough used to exhaust the call stack and + terminate the process. The descent is bounded here: once @ref + detail::recursion_depth_limit levels have been entered, @ref + diff_iteratively diffs what is left without the call stack. + */ + static void diff_recursively(basic_json& result, const basic_json& source, const basic_json& target, + const string_t& path, const std::size_t depth) + { + // if the values are the same, there is nothing to do if (source == target) { - return result; + return; + } + + if (JSON_HEDLEY_UNLIKELY(depth >= detail::recursion_depth_limit())) + { + diff_iteratively(result, source, target, path); + return; } if (source.type() != target.type()) { // different types: replace value - result.push_back( - { - {"op", "replace"}, {"path", path}, {"value", target} - }); - return result; + diff_replace(result, path, target); + return; } switch (source.type()) @@ -33109,200 +33344,50 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec while (i < source.size() && i < target.size()) { // recursive call to compare array values at index i - auto temp_diff = diff(source[i], target[i], detail::concat(path, '/', detail::to_string(i))); - result.insert(result.end(), temp_diff.begin(), temp_diff.end()); + diff_recursively(result, source[i], target[i], detail::concat(path, '/', detail::to_string(i)), depth + 1); ++i; } // We now reached the end of at least one array // in a second pass, traverse the remaining elements - - // remove my remaining elements, highest index first; appending - // in that order avoids the quadratic reinsertion done before - for (std::size_t j = source.size(); j > i; --j) - { - result.push_back(object( - { - {"op", "remove"}, - {"path", detail::concat(path, '/', detail::to_string(j - 1))} - })); - } - i = source.size(); - - // add other remaining elements - while (i < target.size()) - { - result.push_back( - { - {"op", "add"}, - {"path", detail::concat(path, "/-")}, - {"value", target[i]} - }); - ++i; - } - + diff_array_tails(result, source, target, path, i); break; } case value_t::object: { - // first pass: record, for every source key, whether it is - // common to both objects (in source's iteration order) or - // was deleted (i.e., in source but not in target) -- this is - // a by-product of the target.find() call already needed to - // tell the two cases apart, so it adds no extra lookups. The - // "remove" ops themselves are emitted later, interleaved - // with the recursive per-key diffs in the fast path below, - // to match source's original iteration order (as the - // original, pre-reordering-aware implementation did) instead - // of grouping all removes before all recursive diffs. - std::vector common_keys_source_order; - for (auto it = source.cbegin(); it != source.cend(); ++it) - { - if (target.find(it.key()) != target.end()) - { - common_keys_source_order.push_back(it.key()); - } - } - - // second pass: find keys that were added (i.e., in target but - // not in source), and record the keys common to both, in - // target's iteration order -- again a by-product of the - // source.find() call already needed to detect added keys. At - // the same time, determine whether every added key comes - // after every common key in target's order (a precondition - // for the fast path below, which only ever appends new keys - // at the very end). Both are only needed for an object_t that - // keeps its members in insertion order, such as the one - // backing `ordered_json`; for any other object_t, the fast - // path is always taken and they are not computed. - // patch ops for keys that were added (i.e., in target but not - // in source); built here so the fast path below can reuse - // them without a second source.find() per target key. Only - // used by the fast path -- the slow (reordering) path - // rebuilds "add" ops for every key itself. - std::vector common_keys_target_order; + std::vector common_keys; basic_json added_ops(value_t::array); - bool new_keys_form_suffix = true; - bool seen_new_key = false; - for (auto it = target.cbegin(); it != target.cend(); ++it) + if (diff_object_keys(result, source, target, path, common_keys, added_ops)) { - if (source.find(it.key()) == source.end()) - { - seen_new_key = true; - const auto path_key = detail::concat(path, '/', detail::escape(it.key())); - added_ops.push_back( - { - {"op", "add"}, {"path", path_key}, - {"value", it.value()} - }); - } - else - { -#ifdef JSON_HEDLEY_MSVC_VERSION -#pragma warning(push ) -#pragma warning(disable : 4127) // ignore warning to replace if with if constexpr -#endif - if (detail::is_ordered_map::value) - { - common_keys_target_order.push_back(it.key()); - if (seen_new_key) - { - new_keys_form_suffix = false; - } - } -#ifdef JSON_HEDLEY_MSVC_VERSION -#pragma warning( pop ) -#endif - } - } - - // Only an object type that keeps its members in insertion - // order, such as nlohmann::ordered_map, can need reordering: - // patch() appends a new member at the end of such an object. - // Any other object type places its members itself - std::map - // in key order, a hash map in an order its operator== ignores - - // so a member-by-member diff always reproduces target there. - if (!detail::is_ordered_map::value - || (common_keys_source_order == common_keys_target_order && new_keys_form_suffix)) - { - // fast path: order of common keys already matches (or the - // object_t's iteration order does not depend on - // insertion history), so a plain per-key recursive diff - // is correct and minimal, as before. common_keys_source_order - // is, by construction, the subsequence of source's keys - // that are common to both objects, in source's iteration - // order -- so it can be walked in lockstep with `source` - // using a cheap key comparison instead of another lookup. - // Deleted keys (those source keys not in common_keys_source_order) - // are interleaved here too, in source's original order, to - // match the historical (pre-reordering-aware) output order. - auto common_it = common_keys_source_order.cbegin(); + // fast path: common_keys is, by construction, the + // subsequence of source's keys that are common to both + // objects, in source's iteration order -- so it can be + // walked in lockstep with `source` using a cheap key + // comparison instead of another lookup. Deleted keys + // (those source keys not in common_keys) are interleaved + // here too, in source's original order, to match the + // historical (pre-reordering-aware) output order. + auto common_it = common_keys.cbegin(); for (auto it = source.cbegin(); it != source.cend(); ++it) { - if (common_it != common_keys_source_order.cend() && it.key() == *common_it) + if (common_it != common_keys.cend() && it.key() == *common_it) { - const auto path_key = detail::concat(path, '/', detail::escape(it.key())); - auto temp_diff = diff(it.value(), target[it.key()], path_key); - result.insert(result.end(), temp_diff.begin(), temp_diff.end()); + diff_recursively(result, it.value(), target[it.key()], detail::concat(path, '/', detail::escape(it.key())), depth + 1); ++common_it; } else { // found a key that is not in target -> remove it - const auto path_key = detail::concat(path, '/', detail::escape(it.key())); - result.push_back(object( - { - {"op", "remove"}, {"path", path_key} - })); + diff_remove(result, detail::concat(path, '/', detail::escape(it.key()))); } } - // append the "add" ops for brand-new keys collected above - // during the pass over target -- no second source.find() - // per target key needed + // append the "add" ops for brand-new keys collected by + // diff_object_keys -- no second source.find() per target + // key needed result.insert(result.end(), added_ops.begin(), added_ops.end()); } - else - { - // slow path: the common keys are in a different relative - // order in source and target (only possible for a - // reorderable object_t like ordered_map). Building a - // minimal reordering patch is a nontrivial (LCS-like) - // problem; instead, remove every source key -- both - // deleted keys (which must be removed regardless) and - // common keys (removed so they can be re-added in - // target's order) -- and re-add every key that should - // remain, with its final target value, in target's - // order. basic_json::patch()'s "add" operation on an - // object uses operator[], which appends at the end for a - // vector-backed insertion-ordered map when the key does - // not already exist -- so removing a key and then adding - // it moves it to the end, fixing its position. - for (auto it = source.cbegin(); it != source.cend(); ++it) - { - const auto path_key = detail::concat(path, '/', detail::escape(it.key())); - result.push_back(object( - { - {"op", "remove"}, {"path", path_key} - })); - } - - // add every key that is either common (just removed - // above) or brand new, in target's iteration order, so - // that the final order after applying the patch matches - // target exactly - for (auto it = target.cbegin(); it != target.cend(); ++it) - { - const auto path_key = detail::concat(path, '/', detail::escape(it.key())); - result.push_back( - { - {"op", "add"}, {"path", path_key}, - {"value", it.value()} - }); - } - } - break; } @@ -33317,16 +33402,170 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec default: { // both primitive types: replace value - result.push_back( - { - {"op", "replace"}, {"path", path}, {"value", target} - }); + diff_replace(result, path, target); break; } } - - return result; } + + /*! + @brief @ref diff without the call stack, appending the operations to + @a result + + Produces the same operations as @ref diff_recursively. Only reached for + values nested more deeply than @ref detail::recursion_depth_limit. + */ + static void diff_iteratively(basic_json& result, const basic_json& source, const basic_json& target, + const string_t& path) + { + // The arrays and objects being diffed are kept on an explicit stack, + // and every pair of elements is still diffed completely before the + // next one, so the operations come out in the same order as in + // diff_recursively. The path of the values being diffed is kept in + // one buffer that grows and shrinks with the stack, rather than in a + // new string per level. + std::vector stack; + string_t current_path = path; + + // diff `s` against `t`, whose path is current_path: primitives, + // values of different types, and objects whose members were reordered + // are handled right away; arrays and other objects get a frame + const auto enter = [&result, &stack, ¤t_path](const basic_json & s, const basic_json & t) + { + // if the values are the same, there is nothing to do. Arrays and + // objects are not compared up front: comparing them visits + // everything below them, so doing that at every level would take + // quadratic time in the nesting depth - equal ones yield no + // operations anyway. + if ((!s.is_structured() || !t.is_structured()) && s == t) + { + return; + } + + if (s.type() != t.type()) + { + // different types: replace value + diff_replace(result, current_path, t); + return; + } + + switch (s.type()) + { + case value_t::array: + { + stack.emplace_back(&s, &t, current_path.size()); + return; + } + + case value_t::object: + { + std::vector common_keys; + basic_json added_ops(value_t::array); + if (diff_object_keys(result, s, t, current_path, common_keys, added_ops)) + { + // fast path: the frame walks source in lockstep with + // common_keys, as diff_recursively does, and appends + // added_ops once all members are done + stack.emplace_back(&s, &t, current_path.size()); + stack.back().member = s.cbegin(); + stack.back().common_keys = std::move(common_keys); + stack.back().added_ops = std::move(added_ops); + } + return; + } + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + // both primitive types: replace value + diff_replace(result, current_path, t); + return; + } + } + }; + + enter(source, target); + while (!stack.empty()) + { + // the frame is copied out member by member and changed through + // stack.back(): enter() may push a frame and the end of the loop + // pops it, either of which would invalidate a reference to it + const basic_json* const s = stack.back().source; + const basic_json* const t = stack.back().target; + const std::size_t path_length = stack.back().path_length; + const std::size_t depth = stack.size(); + + if (s->is_array()) + { + const auto& source_array = *s->m_data.m_value.array; + const auto& target_array = *t->m_data.m_value.array; + + // first pass: traverse common elements + const std::size_t i = stack.back().index; + if (i < source_array.size() && i < target_array.size()) + { + ++stack.back().index; + detail::concat_into(current_path, '/', detail::to_string(i)); + enter(source_array[i], target_array[i]); + if (stack.size() == depth) + { + current_path.resize(path_length); + } + continue; + } + + // We now reached the end of at least one array + // in a second pass, traverse the remaining elements + diff_array_tails(result, *s, *t, current_path, i); + } + else + { + const const_iterator it = stack.back().member; + if (it != s->cend()) + { + ++stack.back().member; + const std::size_t next_common = stack.back().next_common; + if (next_common < stack.back().common_keys.size() && it.key() == stack.back().common_keys[next_common]) + { + ++stack.back().next_common; + const basic_json& target_value = (*t)[it.key()]; + detail::concat_into(current_path, '/', detail::escape(it.key())); + enter(it.value(), target_value); + if (stack.size() == depth) + { + current_path.resize(path_length); + } + } + else + { + // found a key that is not in target -> remove it + diff_remove(result, detail::concat(current_path, '/', detail::escape(it.key()))); + } + continue; + } + + // append the "add" ops for brand-new keys collected when the + // object was entered + result.insert(result.end(), stack.back().added_ops.begin(), stack.back().added_ops.end()); + } + + // this array or object is done: continue with the one it is in + stack.pop_back(); + if (!stack.empty()) + { + current_path.resize(stack.back().path_length); + } + } + } + + public: /// @} //////////////////////////////// diff --git a/tests/src/unit-json_patch.cpp b/tests/src/unit-json_patch.cpp index ef239e9f4..a78412b23 100644 --- a/tests/src/unit-json_patch.cpp +++ b/tests/src/unit-json_patch.cpp @@ -15,8 +15,65 @@ using nlohmann::json; #endif #include +#include +#include #include "make_test_data_available.hpp" +namespace +{ +// alternating objects and arrays nested `depth` levels deep, with members that +// depend on `variant` at some levels, so diffing two variants yields +// operations on many levels: replacing the innermost value, adding, removing, +// and (for ordered_json) reordering members, and changing array lengths +template +BasicJsonType nested(const std::size_t depth, const int variant) +{ + BasicJsonType value = variant; + for (std::size_t i = 0; i < depth; ++i) + { + if (i % 2 == 0) + { + BasicJsonType object = BasicJsonType::object(); + if ((i + static_cast(variant)) % 7 == 0) + { + object["x"] = i; + } + if (variant == 2 && i % 11 == 0) + { + object["z"] = "z"; + } + object["a"] = std::move(value); + if (variant == 1 && i % 5 == 0) + { + object["y"] = 1; + } + value = std::move(object); + } + else + { + BasicJsonType array = BasicJsonType::array({std::move(value)}); + if ((i + static_cast(variant)) % 3 == 0) + { + array.push_back(i); + } + value = std::move(array); + } + } + return value; +} + +// a path of `depth` reference tokens, as nested() nests its values +std::string nested_path(const std::size_t depth) +{ + std::string path; + for (std::size_t i = depth; i > 0; --i) + { + path += (i - 1) % 2 == 0 ? "/a" : "/0"; + } + return path; +} +} // namespace + TEST_CASE("JSON patch") { SECTION("examples from RFC 6902") @@ -1752,6 +1809,102 @@ TEST_CASE("JSON patch - diff emits array removals in descending index order") } } +TEST_CASE("JSON patch: diff of deeply nested values") +{ + SECTION("the diff reproduces the target at every depth") + { + // depths on either side of the nesting depth up to which diff() + // recurses (detail::recursion_depth_limit(), 128); not every depth up + // to 300, as the test would then time out under Valgrind + std::vector depths; + for (std::size_t depth = 0; depth <= 16; ++depth) + { + depths.push_back(depth); + } + for (std::size_t depth = 120; depth <= 136; ++depth) + { + depths.push_back(depth); + } + depths.push_back(300); + + for (const auto depth : depths) + { + CAPTURE(depth); + for (int from = 0; from < 3; ++from) + { + for (int to = 0; to < 3; ++to) + { + CAPTURE(from); + CAPTURE(to); + const auto source = nested(depth, from); + const auto target = nested(depth, to); + const auto patch = json::diff(source, target); + CHECK(source.patch(patch) == target); + CHECK(patch.empty() == (from == to)); + + const auto ordered_source = nested(depth, from); + const auto ordered_target = nested(depth, to); + CHECK(ordered_source.patch(nlohmann::ordered_json::diff(ordered_source, ordered_target)) == ordered_target); + } + } + } + } + + SECTION("a difference only in the innermost value is one replace operation") + { + for (std::size_t depth = 0; depth <= 300; ++depth) + { + CAPTURE(depth); + json source = 1; + json target = 2; + for (std::size_t i = 0; i < depth; ++i) + { + source = i % 2 == 0 ? json::object({{"a", std::move(source)}}) : json::array({std::move(source)}); + target = i % 2 == 0 ? json::object({{"a", std::move(target)}}) : json::array({std::move(target)}); + } + CHECK(json::diff(source, target, "/root") == json::array({{{"op", "replace"}, {"path", "/root" + nested_path(depth)}, {"value", 2}}})); + } + } + + SECTION("values nested too deeply for the call stack (#5393)") + { + // diff() used to recurse once per nesting level, and compared the + // values with operator== on every level. The values are only + // parsed and diffed, never copied or compared, since those recurse + // too. + const std::size_t depth = 100000; + for (const bool objects : + { + false, true + }) + { + CAPTURE(objects); + std::string source_text; + std::string target_text; + std::string equal_text; + std::string path; + for (std::size_t i = 0; i < depth; ++i) + { + source_text += objects ? "{\"a\":" : "["; + path += objects ? "/a" : "/0"; + } + target_text = source_text + "2"; + equal_text = source_text + "1"; + source_text += "1"; + const std::string closing(depth, objects ? '}' : ']'); + const auto source = json::parse(source_text + closing); + + const auto patch = json::diff(source, json::parse(target_text + closing)); + REQUIRE(patch.size() == 1); + CHECK(patch[0]["op"] == "replace"); + CHECK(patch[0]["path"] == path); + CHECK(patch[0]["value"] == 2); + + CHECK(json::diff(source, json::parse(equal_text + closing)).empty()); + } + } +} + TEST_CASE("JSON patch - diff() takes the fast path for non-reorderable object types (regression #5639)") { // #5465 added an order check to diff()'s object handling so a