From 25affee45e0fa3ce2cec4d96416bb9f119ab0fd7 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 23 Sep 2026 22:29:44 +0200 Subject: [PATCH] Write BSON in linear time, without recursing per nesting level to_bson() had two problems with nested values: - It recursed once per nesting level, so a value nested deeply enough - 100,000 levels on an 8 MiB stack - exhausted the call stack and terminated the process, although parse() accepts such values without complaint. - BSON prefixes every document and array with its length. The writer computed that length by walking the entire value below it, again for every nested document it wrote, which made serializing O(size x depth). A 200-level document took 30 ms instead of 1. Both passes are now iterative, and each length is computed exactly once: - calc_bson_sizes() computes the length of every document and array in one pass, each from the lengths of its entries, into a table ordered the way they are written. - write_bson_document() then writes the document, taking each length from the table. Everything observable is unchanged, as a differential test against develop confirms byte for byte: - The same bytes are written. - A key containing U+0000 still throws out_of_range.409 for the same first key, with the same diagnostics path, before anything is written. - A document too large for BSON still throws out_of_range.412 before anything is written. - A binary subtype above 255 still throws out_of_range.415 after the same partial output. Only the enclosing objects and arrays are kept on a stack, so a flat document allocates nothing for it. Measured against develop (clang -O3, median of 201 runs): flat objects unchanged, flat arrays 37% faster (the array length was computed twice), a nested 3,000-object document 2x faster, a 200-level document 33x faster. to_bson.md documented the quadratic complexity since #5334; it is linear again. Fixes #5392 for BSON, and #5308. Signed-off-by: Niels Lohmann --- docs/mkdocs/docs/api/basic_json/to_bson.md | 6 +- .../nlohmann/detail/output/binary_writer.hpp | 323 ++++++++++++------ single_include/nlohmann/json.hpp | 323 ++++++++++++------ tests/src/unit-bson.cpp | 66 +++- 4 files changed, 508 insertions(+), 210 deletions(-) diff --git a/docs/mkdocs/docs/api/basic_json/to_bson.md b/docs/mkdocs/docs/api/basic_json/to_bson.md index 786fbc9e0..4cd45a57d 100644 --- a/docs/mkdocs/docs/api/basic_json/to_bson.md +++ b/docs/mkdocs/docs/api/basic_json/to_bson.md @@ -46,9 +46,8 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va ## Complexity -Proportional to the size of the JSON value `j` multiplied by its maximum nesting -depth, `O(n × d)`. BSON length prefixes are computed recursively before nested -values are written. +Linear in the size of the JSON value `j`. The length prefixes of all nested documents and arrays are computed in one +pass before anything is written. ## Examples @@ -77,3 +76,4 @@ values are written. ## Version history - Added in version 3.4.0. +- Linear in the size of `j`, and no longer limited by the call stack for deeply nested values, since version 3.13.0. diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index a355f1c15..a9291bcbd 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -122,7 +122,7 @@ class binary_writer { case value_t::object: { - write_bson_object(*j.m_data.m_value.object); + write_bson_document(j); break; } @@ -1201,35 +1201,6 @@ class binary_writer } } - /*! - @brief Writes a BSON element with key @a name and object @a value - */ - void write_bson_object_entry(const string_t& name, - const typename BasicJsonType::object_t& value) - { - write_bson_entry_header(name, 0x03); // object - write_bson_object(value); - } - - /*! - @return The size of the BSON-encoded array @a value - */ - static std::size_t calc_bson_array_size(const typename BasicJsonType::array_t& value) - { - std::size_t array_index = 0ul; - - const std::size_t embedded_document_size = std::accumulate(std::begin(value), std::end(value), static_cast(0), [&array_index](std::size_t result, const typename BasicJsonType::array_t::value_type & el) - { - // the index is built as a std::string, while calc_bson_element_size - // takes a string_t; convert explicitly, as the two are only - // implicitly convertible for some string types - const auto key = std::to_string(array_index++); - return result + calc_bson_element_size(string_t(key.data(), key.size()), el); - }); - - return sizeof(std::int32_t) + embedded_document_size + 1ul; - } - /*! @return The size of the BSON-encoded binary array @a value */ @@ -1238,29 +1209,6 @@ class binary_writer return sizeof(std::int32_t) + value.size() + 1ul; } - /*! - @brief Writes a BSON element with key @a name and array @a value - */ - void write_bson_array(const string_t& name, - const typename BasicJsonType::array_t& value) - { - write_bson_entry_header(name, 0x04); // array - write_number(to_bson_length(calc_bson_array_size(value)), true); - - std::size_t array_index = 0ul; - - for (const auto& el : value) - { - // the index is built as a std::string, while write_bson_element takes - // a string_t; convert explicitly, as the two are only implicitly - // convertible for some string types - const auto key = std::to_string(array_index++); - write_bson_element(string_t(key.data(), key.size()), el); - } - - oa.write_character(to_char_type(0x00)); - } - /*! @brief Writes a BSON element with key @a name and binary value @a value */ @@ -1282,43 +1230,37 @@ class binary_writer } /*! - @brief Calculates the size necessary to serialize the JSON value @a j with its @a name - @return The calculated size for the BSON document entry for @a j with the given @a name. + @return The size of the value of the BSON document entry for @a j, which + is neither an object nor an array */ - static std::size_t calc_bson_element_size(const string_t& name, - const BasicJsonType& j) + static std::size_t calc_bson_value_size(const BasicJsonType& j) { - const auto header_size = calc_bson_entry_header_size(name, j); switch (j.type()) { - case value_t::object: - return header_size + calc_bson_object_size(*j.m_data.m_value.object); - - case value_t::array: - return header_size + calc_bson_array_size(*j.m_data.m_value.array); - case value_t::binary: - return header_size + calc_bson_binary_size(*j.m_data.m_value.binary); + return calc_bson_binary_size(*j.m_data.m_value.binary); case value_t::boolean: - return header_size + 1ul; + return 1ul; case value_t::number_float: - return header_size + 8ul; + return 8ul; case value_t::number_integer: - return header_size + calc_bson_integer_size(j.m_data.m_value.number_integer); + return calc_bson_integer_size(j.m_data.m_value.number_integer); case value_t::number_unsigned: - return header_size + calc_bson_unsigned_size(j.m_data.m_value.number_unsigned); + return calc_bson_unsigned_size(j.m_data.m_value.number_unsigned); case value_t::string: - return header_size + calc_bson_string_size(*j.m_data.m_value.string); + return calc_bson_string_size(*j.m_data.m_value.string); case value_t::null: - return header_size + 0ul; + return 0ul; // LCOV_EXCL_START + case value_t::object: + case value_t::array: case value_t::discarded: default: JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) @@ -1328,22 +1270,13 @@ class binary_writer } /*! - @brief Serializes the JSON value @a j to BSON and associates it with the - key @a name. - @param name The name to associate with the JSON entity @a j within the - current BSON document + @brief Writes the BSON document entry with key @a name for @a j, which is + neither an object nor an array */ - void write_bson_element(const string_t& name, - const BasicJsonType& j) + void write_bson_value(const string_t& name, const BasicJsonType& j) { switch (j.type()) { - case value_t::object: - return write_bson_object_entry(name, *j.m_data.m_value.object); - - case value_t::array: - return write_bson_array(name, *j.m_data.m_value.array); - case value_t::binary: return write_bson_binary(name, *j.m_data.m_value.binary); @@ -1366,6 +1299,8 @@ class binary_writer return write_bson_null(name); // LCOV_EXCL_START + case value_t::object: + case value_t::array: case value_t::discarded: default: JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) @@ -1374,37 +1309,219 @@ class binary_writer } } - /*! - @brief Calculates the size of the BSON serialization of the given - JSON-object @a j. - @param[in] value JSON value to serialize - @pre value.type() == value_t::object - */ - static std::size_t calc_bson_object_size(const typename BasicJsonType::object_t& value) + /// @brief an object or array of the BSON document being sized or written + struct bson_frame { - const std::size_t document_size = std::accumulate(value.begin(), value.end(), static_cast(0), - [](size_t result, const typename BasicJsonType::object_t::value_type & el) + explicit bson_frame(const BasicJsonType* value_, const std::size_t size_slot_ = 0) + : value(value_) + , size_slot(size_slot_) { - return result += calc_bson_element_size(el.first, el.second); - }); + if (value->is_object()) + { + member = value->m_data.m_value.object->cbegin(); + } + } - return sizeof(std::int32_t) + document_size + 1ul; + /// the object or array + const BasicJsonType* value; + /// objects: the next member + typename BasicJsonType::object_t::const_iterator member{}; + /// arrays: the index of the next element + std::size_t index = 0; + /// @ref calc_bson_sizes only: where its size goes in the table + std::size_t size_slot; + /// @ref calc_bson_sizes only: the size of its entries seen so far + std::size_t entries_size = 0; + }; + + /*! + @brief the name BSON gives the array element with index @a index + @param[out] name receives the decimal index + */ + static const string_t& bson_index_name(const std::size_t index, string_t& name) + { + // the index is built as a std::string; convert explicitly, as the + // two are only implicitly convertible for some string types + const auto key = std::to_string(index); + name = string_t(key.data(), key.size()); + return name; } /*! - @param[in] value JSON value to serialize - @pre value.type() == value_t::object + @brief Calculates the size of every object and array in the BSON document + @a document, including the document itself. + + BSON prefixes every document and array with its size, so all of them have + to be known before the first byte is written. They are computed in a + single pass, each one from the sizes of its entries, which keeps + serializing linear in the size of the document; computing each size by + walking the entire value below it made it quadratic in the nesting depth. + The pass keeps the objects and arrays it has entered on an explicit stack, + so a deeply nested value cannot exhaust the call stack. + + @param[in] document the JSON object to serialize + @param[out] nested_sizes the sizes of the objects and arrays in + @a document, in the order they are written + @return the size of @a document + @throw out_of_range.409 if a key contains U+0000, before anything is + written */ - void write_bson_object(const typename BasicJsonType::object_t& value) + static std::size_t calc_bson_sizes(const BasicJsonType& document, std::vector& nested_sizes) { - write_number(to_bson_length(calc_bson_object_size(value)), true); + // the object or array whose entries are being sized, and the ones it + // is in; nothing is allocated unless the document nests + bson_frame current(&document); + std::vector parents; + string_t index_name; - for (const auto& el : value) + while (true) { - write_bson_element(el.first, el.second); - } + // size entries until the current object or array is done, or an + // entry is an object or array itself + const BasicJsonType* nested = nullptr; + if (current.value->is_object()) + { + const auto& object = *current.value->m_data.m_value.object; + while (nested == nullptr && current.member != object.cend()) + { + const auto& el = *current.member; + ++current.member; + current.entries_size += calc_bson_entry_header_size(el.first, el.second); + if (el.second.is_structured()) + { + nested = &el.second; + } + else + { + current.entries_size += calc_bson_value_size(el.second); + } + } + } + else + { + const auto& array = *current.value->m_data.m_value.array; + while (nested == nullptr && current.index < array.size()) + { + const BasicJsonType& el = array[current.index]; + current.entries_size += calc_bson_entry_header_size(bson_index_name(current.index, index_name), el); + ++current.index; + if (el.is_structured()) + { + nested = ⪙ + } + else + { + current.entries_size += calc_bson_value_size(el); + } + } + } - oa.write_character(to_char_type(0x00)); + if (nested != nullptr) + { + // its size is added to the current one's once it is done + nested_sizes.push_back(0); + parents.push_back(std::move(current)); + current = bson_frame(nested, nested_sizes.size() - 1); + continue; + } + + // the int32 size, the entries, and the terminating null byte + const std::size_t size = sizeof(std::int32_t) + current.entries_size + 1ul; + if (parents.empty()) + { + return size; + } + nested_sizes[current.size_slot] = size; + current = std::move(parents.back()); + parents.pop_back(); + current.entries_size += size; + } + } + + /*! + @brief Serializes the JSON object @a document as a BSON document + + Writes the objects and arrays in it without the call stack, keeping the + ones it has entered on an explicit stack, so a deeply nested value + cannot exhaust the call stack. + + @param[in] document the JSON object to serialize + @pre document.type() == value_t::object + */ + void write_bson_document(const BasicJsonType& document) + { + std::vector nested_sizes; + const std::size_t document_size = calc_bson_sizes(document, nested_sizes); + write_number(to_bson_length(document_size), true); + + // the object or array whose entries are being written, and the ones + // it is in + bson_frame current(&document); + std::vector parents; + std::size_t next_size = 0; + string_t index_name; + + while (true) + { + // write entries until the current object or array is done, or an + // entry is an object or array itself + const string_t* nested_name = nullptr; + const BasicJsonType* nested = nullptr; + if (current.value->is_object()) + { + const auto& object = *current.value->m_data.m_value.object; + while (nested == nullptr && current.member != object.cend()) + { + const auto& el = *current.member; + ++current.member; + if (el.second.is_structured()) + { + nested_name = &el.first; + nested = &el.second; + } + else + { + write_bson_value(el.first, el.second); + } + } + } + else + { + const auto& array = *current.value->m_data.m_value.array; + while (nested == nullptr && current.index < array.size()) + { + const BasicJsonType& el = array[current.index]; + const string_t& name = bson_index_name(current.index, index_name); + ++current.index; + if (el.is_structured()) + { + nested_name = &name; + nested = ⪙ + } + else + { + write_bson_value(name, el); + } + } + } + + if (nested != nullptr) + { + write_bson_entry_header(*nested_name, nested->is_object() ? 0x03 : 0x04); + write_number(to_bson_length(nested_sizes[next_size++]), true); + parents.push_back(std::move(current)); + current = bson_frame(nested); + continue; + } + + oa.write_character(to_char_type(0x00)); + if (parents.empty()) + { + return; + } + current = std::move(parents.back()); + parents.pop_back(); + } } ////////// diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index c96461458..b94841147 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -19084,7 +19084,7 @@ class binary_writer { case value_t::object: { - write_bson_object(*j.m_data.m_value.object); + write_bson_document(j); break; } @@ -20163,35 +20163,6 @@ class binary_writer } } - /*! - @brief Writes a BSON element with key @a name and object @a value - */ - void write_bson_object_entry(const string_t& name, - const typename BasicJsonType::object_t& value) - { - write_bson_entry_header(name, 0x03); // object - write_bson_object(value); - } - - /*! - @return The size of the BSON-encoded array @a value - */ - static std::size_t calc_bson_array_size(const typename BasicJsonType::array_t& value) - { - std::size_t array_index = 0ul; - - const std::size_t embedded_document_size = std::accumulate(std::begin(value), std::end(value), static_cast(0), [&array_index](std::size_t result, const typename BasicJsonType::array_t::value_type & el) - { - // the index is built as a std::string, while calc_bson_element_size - // takes a string_t; convert explicitly, as the two are only - // implicitly convertible for some string types - const auto key = std::to_string(array_index++); - return result + calc_bson_element_size(string_t(key.data(), key.size()), el); - }); - - return sizeof(std::int32_t) + embedded_document_size + 1ul; - } - /*! @return The size of the BSON-encoded binary array @a value */ @@ -20200,29 +20171,6 @@ class binary_writer return sizeof(std::int32_t) + value.size() + 1ul; } - /*! - @brief Writes a BSON element with key @a name and array @a value - */ - void write_bson_array(const string_t& name, - const typename BasicJsonType::array_t& value) - { - write_bson_entry_header(name, 0x04); // array - write_number(to_bson_length(calc_bson_array_size(value)), true); - - std::size_t array_index = 0ul; - - for (const auto& el : value) - { - // the index is built as a std::string, while write_bson_element takes - // a string_t; convert explicitly, as the two are only implicitly - // convertible for some string types - const auto key = std::to_string(array_index++); - write_bson_element(string_t(key.data(), key.size()), el); - } - - oa.write_character(to_char_type(0x00)); - } - /*! @brief Writes a BSON element with key @a name and binary value @a value */ @@ -20244,43 +20192,37 @@ class binary_writer } /*! - @brief Calculates the size necessary to serialize the JSON value @a j with its @a name - @return The calculated size for the BSON document entry for @a j with the given @a name. + @return The size of the value of the BSON document entry for @a j, which + is neither an object nor an array */ - static std::size_t calc_bson_element_size(const string_t& name, - const BasicJsonType& j) + static std::size_t calc_bson_value_size(const BasicJsonType& j) { - const auto header_size = calc_bson_entry_header_size(name, j); switch (j.type()) { - case value_t::object: - return header_size + calc_bson_object_size(*j.m_data.m_value.object); - - case value_t::array: - return header_size + calc_bson_array_size(*j.m_data.m_value.array); - case value_t::binary: - return header_size + calc_bson_binary_size(*j.m_data.m_value.binary); + return calc_bson_binary_size(*j.m_data.m_value.binary); case value_t::boolean: - return header_size + 1ul; + return 1ul; case value_t::number_float: - return header_size + 8ul; + return 8ul; case value_t::number_integer: - return header_size + calc_bson_integer_size(j.m_data.m_value.number_integer); + return calc_bson_integer_size(j.m_data.m_value.number_integer); case value_t::number_unsigned: - return header_size + calc_bson_unsigned_size(j.m_data.m_value.number_unsigned); + return calc_bson_unsigned_size(j.m_data.m_value.number_unsigned); case value_t::string: - return header_size + calc_bson_string_size(*j.m_data.m_value.string); + return calc_bson_string_size(*j.m_data.m_value.string); case value_t::null: - return header_size + 0ul; + return 0ul; // LCOV_EXCL_START + case value_t::object: + case value_t::array: case value_t::discarded: default: JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) @@ -20290,22 +20232,13 @@ class binary_writer } /*! - @brief Serializes the JSON value @a j to BSON and associates it with the - key @a name. - @param name The name to associate with the JSON entity @a j within the - current BSON document + @brief Writes the BSON document entry with key @a name for @a j, which is + neither an object nor an array */ - void write_bson_element(const string_t& name, - const BasicJsonType& j) + void write_bson_value(const string_t& name, const BasicJsonType& j) { switch (j.type()) { - case value_t::object: - return write_bson_object_entry(name, *j.m_data.m_value.object); - - case value_t::array: - return write_bson_array(name, *j.m_data.m_value.array); - case value_t::binary: return write_bson_binary(name, *j.m_data.m_value.binary); @@ -20328,6 +20261,8 @@ class binary_writer return write_bson_null(name); // LCOV_EXCL_START + case value_t::object: + case value_t::array: case value_t::discarded: default: JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) @@ -20336,37 +20271,219 @@ class binary_writer } } - /*! - @brief Calculates the size of the BSON serialization of the given - JSON-object @a j. - @param[in] value JSON value to serialize - @pre value.type() == value_t::object - */ - static std::size_t calc_bson_object_size(const typename BasicJsonType::object_t& value) + /// @brief an object or array of the BSON document being sized or written + struct bson_frame { - const std::size_t document_size = std::accumulate(value.begin(), value.end(), static_cast(0), - [](size_t result, const typename BasicJsonType::object_t::value_type & el) + explicit bson_frame(const BasicJsonType* value_, const std::size_t size_slot_ = 0) + : value(value_) + , size_slot(size_slot_) { - return result += calc_bson_element_size(el.first, el.second); - }); + if (value->is_object()) + { + member = value->m_data.m_value.object->cbegin(); + } + } - return sizeof(std::int32_t) + document_size + 1ul; + /// the object or array + const BasicJsonType* value; + /// objects: the next member + typename BasicJsonType::object_t::const_iterator member{}; + /// arrays: the index of the next element + std::size_t index = 0; + /// @ref calc_bson_sizes only: where its size goes in the table + std::size_t size_slot; + /// @ref calc_bson_sizes only: the size of its entries seen so far + std::size_t entries_size = 0; + }; + + /*! + @brief the name BSON gives the array element with index @a index + @param[out] name receives the decimal index + */ + static const string_t& bson_index_name(const std::size_t index, string_t& name) + { + // the index is built as a std::string; convert explicitly, as the + // two are only implicitly convertible for some string types + const auto key = std::to_string(index); + name = string_t(key.data(), key.size()); + return name; } /*! - @param[in] value JSON value to serialize - @pre value.type() == value_t::object + @brief Calculates the size of every object and array in the BSON document + @a document, including the document itself. + + BSON prefixes every document and array with its size, so all of them have + to be known before the first byte is written. They are computed in a + single pass, each one from the sizes of its entries, which keeps + serializing linear in the size of the document; computing each size by + walking the entire value below it made it quadratic in the nesting depth. + The pass keeps the objects and arrays it has entered on an explicit stack, + so a deeply nested value cannot exhaust the call stack. + + @param[in] document the JSON object to serialize + @param[out] nested_sizes the sizes of the objects and arrays in + @a document, in the order they are written + @return the size of @a document + @throw out_of_range.409 if a key contains U+0000, before anything is + written */ - void write_bson_object(const typename BasicJsonType::object_t& value) + static std::size_t calc_bson_sizes(const BasicJsonType& document, std::vector& nested_sizes) { - write_number(to_bson_length(calc_bson_object_size(value)), true); + // the object or array whose entries are being sized, and the ones it + // is in; nothing is allocated unless the document nests + bson_frame current(&document); + std::vector parents; + string_t index_name; - for (const auto& el : value) + while (true) { - write_bson_element(el.first, el.second); - } + // size entries until the current object or array is done, or an + // entry is an object or array itself + const BasicJsonType* nested = nullptr; + if (current.value->is_object()) + { + const auto& object = *current.value->m_data.m_value.object; + while (nested == nullptr && current.member != object.cend()) + { + const auto& el = *current.member; + ++current.member; + current.entries_size += calc_bson_entry_header_size(el.first, el.second); + if (el.second.is_structured()) + { + nested = &el.second; + } + else + { + current.entries_size += calc_bson_value_size(el.second); + } + } + } + else + { + const auto& array = *current.value->m_data.m_value.array; + while (nested == nullptr && current.index < array.size()) + { + const BasicJsonType& el = array[current.index]; + current.entries_size += calc_bson_entry_header_size(bson_index_name(current.index, index_name), el); + ++current.index; + if (el.is_structured()) + { + nested = ⪙ + } + else + { + current.entries_size += calc_bson_value_size(el); + } + } + } - oa.write_character(to_char_type(0x00)); + if (nested != nullptr) + { + // its size is added to the current one's once it is done + nested_sizes.push_back(0); + parents.push_back(std::move(current)); + current = bson_frame(nested, nested_sizes.size() - 1); + continue; + } + + // the int32 size, the entries, and the terminating null byte + const std::size_t size = sizeof(std::int32_t) + current.entries_size + 1ul; + if (parents.empty()) + { + return size; + } + nested_sizes[current.size_slot] = size; + current = std::move(parents.back()); + parents.pop_back(); + current.entries_size += size; + } + } + + /*! + @brief Serializes the JSON object @a document as a BSON document + + Writes the objects and arrays in it without the call stack, keeping the + ones it has entered on an explicit stack, so a deeply nested value + cannot exhaust the call stack. + + @param[in] document the JSON object to serialize + @pre document.type() == value_t::object + */ + void write_bson_document(const BasicJsonType& document) + { + std::vector nested_sizes; + const std::size_t document_size = calc_bson_sizes(document, nested_sizes); + write_number(to_bson_length(document_size), true); + + // the object or array whose entries are being written, and the ones + // it is in + bson_frame current(&document); + std::vector parents; + std::size_t next_size = 0; + string_t index_name; + + while (true) + { + // write entries until the current object or array is done, or an + // entry is an object or array itself + const string_t* nested_name = nullptr; + const BasicJsonType* nested = nullptr; + if (current.value->is_object()) + { + const auto& object = *current.value->m_data.m_value.object; + while (nested == nullptr && current.member != object.cend()) + { + const auto& el = *current.member; + ++current.member; + if (el.second.is_structured()) + { + nested_name = &el.first; + nested = &el.second; + } + else + { + write_bson_value(el.first, el.second); + } + } + } + else + { + const auto& array = *current.value->m_data.m_value.array; + while (nested == nullptr && current.index < array.size()) + { + const BasicJsonType& el = array[current.index]; + const string_t& name = bson_index_name(current.index, index_name); + ++current.index; + if (el.is_structured()) + { + nested_name = &name; + nested = ⪙ + } + else + { + write_bson_value(name, el); + } + } + } + + if (nested != nullptr) + { + write_bson_entry_header(*nested_name, nested->is_object() ? 0x03 : 0x04); + write_number(to_bson_length(nested_sizes[next_size++]), true); + parents.push_back(std::move(current)); + current = bson_frame(nested); + continue; + } + + oa.write_character(to_char_type(0x00)); + if (parents.empty()) + { + return; + } + current = std::move(parents.back()); + parents.pop_back(); + } } ////////// diff --git a/tests/src/unit-bson.cpp b/tests/src/unit-bson.cpp index 669a4bfe1..2cc095a37 100644 --- a/tests/src/unit-bson.cpp +++ b/tests/src/unit-bson.cpp @@ -49,7 +49,7 @@ using huge_binary_json = nlohmann::basic_json < // for *object keys* (e.g. "s" or "nested" below). Only the designated test // value is meant to lie about its size - if every huge_string_t (including // keys) reported a huge size, the running totals computed while walking the -// BSON document (see calc_bson_object_size & friends in binary_writer.hpp) +// BSON document (see calc_bson_sizes in binary_writer.hpp) // would need more than 32 bits, and on platforms where std::size_t is only // 32 bits wide that arithmetic would silently wrap around, producing wrong // (or even unguarded) lengths. The fake size is therefore opt-in via @@ -1697,3 +1697,67 @@ TEST_CASE("BSON roundtrips" * doctest::skip()) } } } + +TEST_CASE("BSON: deeply nested values") +{ + SECTION("documents and arrays round-trip at every depth") + { + // nested documents and arrays, with siblings on every level, so + // every length prefix covers entries of both kinds + json value = "leaf"; + for (std::size_t depth = 0; depth <= 300; ++depth) + { + CAPTURE(depth); + const json document = {{"value", value}, {"n", depth}}; + CHECK(json::from_bson(json::to_bson(document)) == document); + +value = depth % 2 == 0 ? json{{"a", std::move(value)}, {"b", {1, "x"}}} : + json::array({std::move(value), depth, json::object()}); + } + } + + SECTION("a key containing U+0000 is rejected before anything is written") + { + json value = json::object({{std::string("bad\0key", 7), 1}}); + for (std::size_t depth = 0; depth < 200; ++depth) + { + value = json{{"a", {{"b", 1}}}, {"z", std::move(value)}}; + } + std::vector output; + CHECK_THROWS_AS(json::to_bson(value, output), json::out_of_range&); + CHECK(output.empty()); + } + + SECTION("values nested too deeply for the call stack (#5392)") + { + // serializing recursed once per nesting level, and computed every + // nested document's length by walking everything below it again. + // The values are only parsed, serialized and walked, never copied or + // compared, since those recurse too. + const std::size_t depth = 100000; + for (const bool objects : + { + false, true + }) + { + CAPTURE(objects); + std::string text = "{\"a\":"; + for (std::size_t i = 0; i < depth; ++i) + { + text += objects ? "{\"a\":" : "["; + } + text += "1"; + text.append(depth, objects ? '}' : ']'); + text += "}"; + + const auto bson = json::to_bson(json::parse(text)); + const auto result = json::from_bson(bson); + const json* p = &result.at("a"); + for (std::size_t i = 0; i < depth; ++i) + { + p = objects ? &p->at("a") : &p->at(0); + } + CHECK(*p == 1); + } + } +}