From b38131ec40acf69eb3be4d57a51e0a1e051a66ad Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 11 Nov 2026 16:29:37 +0100 Subject: [PATCH] Add JSON_RECURSION_DEPTH_LIMIT to set how deep value walks recurse Copying, comparing, serializing, hashing, merging, diffing, converting and destroying a value, and the CBOR, MessagePack, UBJSON, BJData and BON8 writers recurse a fixed number of nesting levels on the call stack and finish deeper levels with an explicit stack (#5387). The number was hard-coded to 128. That is too much for small stacks: a 128-level dump() needs about 288 KiB at -O0 with Clang (2.3 KiB per level; 256 KiB with GCC) and crashed in a thread with a 256 KiB stack, and musl threads have 128 KiB by default. Others may want the opposite. The macro JSON_RECURSION_DEPTH_LIMIT, defined before including the library, sets the number; the default stays 128. The value must be an integer constant from 0 to 254 (not a floating-point number or bool), otherwise a static_assert names the macro. 254 is the largest value because the copy constructor and the comparison operators cannot pass a depth down and count it in a byte per thread (nesting_depth()). The guard that counts now counts only the levels it enters, so the count never exceeds the limit. It used to count the level that finds the limit reached as well, which made the count 255 at limit 254: a copy made by a custom base class while a copy was going on then wrapped the byte to 0 and recursed another 254 levels (twice the stack). 0 is allowed: every walk then uses its explicit stack from the first level on and needs a constant amount of stack (16 KiB or less at -O0 with Clang and GCC, for a value nested 20,000 levels deep). All uses of the limit compare a depth with it; none computes limit - 1 or limit + 1, and no walk has a constant 128 of its own. The macro changes only stack use and speed, never a result. Like JSON_NO_THREAD_LOCAL it is not part of the ABI tag and must be the same in all translation units (ODR); the page of the macro says what happens otherwise. Supporting a limit of 0 needed one fix: update() without merge_objects reached the explicit-stack merge at depth 0, which always merges objects. update_members now takes that path only with merge_objects, which is the only case in which it recurses at all. Measurements (macOS arm64, Apple clang 21 and g++ 16, documented in the macro page with a table per operation and optimization level, and a program to reproduce them): the stack needed at the default of 128 levels is up to 288 KiB (dump at -O0, Clang), 224 KiB (copy), and 48 KiB or less at -O2; at 254 levels up to 576 KiB at -O0 and 112 KiB at -O2. Speed at -O2 relative to the default: ordinary documents (nested 2 to 10 levels) are 1.1 to 2.4 times as slow with a limit of 0 or 1 and unchanged (within about 0.1) with 16 or 254; values nested 1,000 or 100,000 levels deep show no difference for any limit. The default build does the same work as before: the instructions retired for copy and conversion (the walks whose guard changed; net of loading, canada, citm, twitter, jeopardy and deep values), ==, dump and hash differ from the base by at most 0.02 % for copy and 0.71 % in either direction for the others (noise), and the cycles for copy by at most 0.65 %. Tests: the existing tests that depended on 128 (comparison, allocator, custom base class, diagnostics, binary writers, patch, merge_diff, conversions, custom array type) derive their depths from the limit, and two assumed a limit of at least 1 (limit - 1 wrapped around). tests/src/unit-recursion_depth_limit.cpp checks that the macro is honoured (the array type of a value counts the levels that a copy and a comparison descend) and that all walks return the same results for values nested 1 to 300 levels deep as arrays, objects and both, against expected strings and bytes built independently of the library; CMake builds it with limits 0, 1 and 254 besides the default, and the CI target ci_test_recursion_depth_limit_0 runs the whole suite with a limit of 0. A test with a base class that copies the value from its copy constructor covers the wrap-around at 254. The new test does not compile against the previous header with -DJSON_RECURSION_DEPTH_LIMIT=1. Docs: new macro page (range, stack and speed tables, small-stack guidance, ODR), index, nav, macro overview, "Nesting depth" of the untrusted-input page (which claimed that no walk could exhaust the stack however deep a value is nested; the default assumes a stack of a few hundred KiB), the assurance case, and the page of JSON_NO_THREAD_LOCAL. No breaking changes: the default is unchanged, and the macro is new. Signed-off-by: Niels Lohmann --- .github/workflows/ubuntu.yml | 2 +- cmake/ci.cmake | 21 + docs/mkdocs/docs/api/macros/index.md | 2 + .../docs/api/macros/json_no_thread_local.md | 1 + .../api/macros/json_recursion_depth_limit.md | 307 +++++++++ docs/mkdocs/docs/community/assurance_case.md | 7 +- .../examples/json_recursion_depth_limit.cpp | 29 + .../json_recursion_depth_limit.output | 4 + docs/mkdocs/docs/features/macros.md | 10 + .../docs/features/parsing/untrusted_input.md | 30 +- docs/mkdocs/mkdocs.yml | 1 + include/nlohmann/detail/macro_scope.hpp | 6 + .../nlohmann/detail/recursion_depth_limit.hpp | 69 +- include/nlohmann/json.hpp | 39 +- single_include/nlohmann/json.hpp | 114 +++- tests/CMakeLists.txt | 14 + tests/src/unit-allocator.cpp | 26 +- tests/src/unit-comparison.cpp | 20 +- tests/src/unit-custom-array-type.cpp | 8 +- tests/src/unit-custom-base-class.cpp | 4 +- tests/src/unit-diagnostic-positions.cpp | 24 +- tests/src/unit-diagnostics.cpp | 4 +- tests/src/unit-json_patch.cpp | 7 +- tests/src/unit-large_json_binary_writers.cpp | 29 +- tests/src/unit-large_json_conversions.cpp | 4 +- tests/src/unit-merge_diff.cpp | 5 +- tests/src/unit-merge_patch.cpp | 4 +- tests/src/unit-modifiers.cpp | 4 +- tests/src/unit-recursion_depth_limit.cpp | 600 ++++++++++++++++++ tests/src/unit-regression1.cpp | 6 +- 30 files changed, 1306 insertions(+), 95 deletions(-) create mode 100644 docs/mkdocs/docs/api/macros/json_recursion_depth_limit.md create mode 100644 docs/mkdocs/docs/examples/json_recursion_depth_limit.cpp create mode 100644 docs/mkdocs/docs/examples/json_recursion_depth_limit.output create mode 100644 tests/src/unit-recursion_depth_limit.cpp diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index e0cd83af9..0f6956254 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -153,7 +153,7 @@ jobs: container: ubuntu:24.04 strategy: matrix: - target: [ci_cmake_flags, ci_test_diagnostics, ci_test_diagnostic_positions, ci_test_noexceptions, ci_test_noimplicitconversions, ci_test_legacycomparison, ci_test_noglobaludls, ci_test_disableenumserialization, ci_test_disabletuplereferenceconversion, ci_test_skiplibraryversioncheck, ci_test_simdutf, ci_test_strict_nul_handling, ci_test_delete_deprecated_functions, ci_test_no_thread_local] + target: [ci_cmake_flags, ci_test_diagnostics, ci_test_diagnostic_positions, ci_test_noexceptions, ci_test_noimplicitconversions, ci_test_legacycomparison, ci_test_noglobaludls, ci_test_disableenumserialization, ci_test_disabletuplereferenceconversion, ci_test_skiplibraryversioncheck, ci_test_simdutf, ci_test_strict_nul_handling, ci_test_delete_deprecated_functions, ci_test_no_thread_local, ci_test_recursion_depth_limit_0] steps: - name: Install build-essential run: apt-get update ; apt-get install -y build-essential unzip wget git diff --git a/cmake/ci.cmake b/cmake/ci.cmake index b41ab8b62..7cc5adebe 100644 --- a/cmake/ci.cmake +++ b/cmake/ci.cmake @@ -342,6 +342,27 @@ add_custom_target(ci_test_no_thread_local COMMENT "Compile and test without thread-local storage" ) +############################################################################### +# Change the number of levels the walks over a value recurse into. +############################################################################### + +# JSON_RECURSION_DEPTH_LIMIT is 128 by default, which is what the whole test +# suite otherwise runs with. 0 sends every walk to its explicit stack at once, +# so this target runs the whole suite through the walks that the default only +# uses for values nested more deeply than 128 levels: results must not depend on +# the limit. The largest limit, 254, is covered by a build of +# tests/src/unit-recursion_depth_limit.cpp (see tests/CMakeLists.txt). +add_custom_target(ci_test_recursion_depth_limit_0 + COMMAND ${CMAKE_COMMAND} + -DCMAKE_BUILD_TYPE=Debug -GNinja + -DJSON_BuildTests=ON -DJSON_FastTests=ON + -DCMAKE_CXX_FLAGS=-DJSON_RECURSION_DEPTH_LIMIT=0 + -S${PROJECT_SOURCE_DIR} -B${PROJECT_BINARY_DIR}/build_recursion_depth_limit_0 + COMMAND ${CMAKE_COMMAND} --build ${PROJECT_BINARY_DIR}/build_recursion_depth_limit_0 + COMMAND cd ${PROJECT_BINARY_DIR}/build_recursion_depth_limit_0 && ${CMAKE_CTEST_COMMAND} --parallel ${N} --output-on-failure + COMMENT "Compile and test with JSON_RECURSION_DEPTH_LIMIT=0" +) + ############################################################################### # Coverage. ############################################################################### diff --git a/docs/mkdocs/docs/api/macros/index.md b/docs/mkdocs/docs/api/macros/index.md index 2ddf6f245..0392ed448 100644 --- a/docs/mkdocs/docs/api/macros/index.md +++ b/docs/mkdocs/docs/api/macros/index.md @@ -40,6 +40,8 @@ header. See also the [macro overview page](../../features/macros.md). - [**JSON_NO_IO**](json_no_io.md) - switch off functions relying on certain C++ I/O headers - [**JSON_NO_SIMD**](json_no_simd.md) - use only portable code instead of vector instructions (SSE2, SSSE3, NEON) - [**JSON_NO_THREAD_LOCAL**](json_no_thread_local.md) - switch off the use of `thread_local` storage +- [**JSON_RECURSION_DEPTH_LIMIT**](json_recursion_depth_limit.md) - set how many nesting levels the operations on a value + recurse into on the call stack (default 128) - [**JSON_SKIP_UNSUPPORTED_COMPILER_CHECK**](json_skip_unsupported_compiler_check.md) - do not warn about unsupported compilers - [**JSON_USE_GLOBAL_UDLS**](json_use_global_udls.md) - place user-defined string literals (UDLs) into the global namespace - [**JSON_USE_SIMDUTF**](json_use_simdutf.md) - use the simdutf library to accelerate UTF-8 validation diff --git a/docs/mkdocs/docs/api/macros/json_no_thread_local.md b/docs/mkdocs/docs/api/macros/json_no_thread_local.md index 400d8fab7..c842ef047 100644 --- a/docs/mkdocs/docs/api/macros/json_no_thread_local.md +++ b/docs/mkdocs/docs/api/macros/json_no_thread_local.md @@ -45,6 +45,7 @@ Copying and comparing fall back to working without the call stack there, as they ## See also +- [`JSON_RECURSION_DEPTH_LIMIT`](json_recursion_depth_limit.md) - set how many nesting levels the operations recurse into - [FAQ: Thread safety](../../home/faq.md#thread-safety) - thread-safety guarantees of the library - [Supported Macros](../../features/macros.md) - guide to all supported macros diff --git a/docs/mkdocs/docs/api/macros/json_recursion_depth_limit.md b/docs/mkdocs/docs/api/macros/json_recursion_depth_limit.md new file mode 100644 index 000000000..2f8a917ae --- /dev/null +++ b/docs/mkdocs/docs/api/macros/json_recursion_depth_limit.md @@ -0,0 +1,307 @@ +# JSON_RECURSION_DEPTH_LIMIT + +```cpp +#define JSON_RECURSION_DEPTH_LIMIT /* value */ +``` + +This macro sets how many nesting levels the operations that walk a value recurse into on the call stack. Beyond that, +they continue with an explicit stack on the heap. The value is an integer constant from `0` to `254`; the default is +`128`. + +The operations are copying a value, comparing two values, serializing with [`dump`](../basic_json/dump.md) or +[`operator<<`](../operator_ltlt.md), hashing with `std::hash`, [`merge_patch`](../basic_json/merge_patch.md), +[`update`](../basic_json/update.md) with `merge_objects`, [`diff`](../basic_json/diff.md), +[`merge_diff`](../basic_json/merge_diff.md), converting a value to another specialization of `basic_json`, destroying a +value, and the binary writers [`to_cbor`](../basic_json/to_cbor.md), +[`to_msgpack`](../basic_json/to_msgpack.md), [`to_ubjson`](../basic_json/to_ubjson.md), +[`to_bjdata`](../basic_json/to_bjdata.md), and [`to_bon8`](../basic_json/to_bon8.md). They share this one limit; none +has +a limit of its own. Recursing is the fastest way to walk a value, but a value nested deeply enough would exhaust the +call +stack, so the library recurses only as far as the limit and finishes the levels below with an explicit stack. This is +what makes all of them safe for untrusted input +(see [Nesting depth](../../features/parsing/untrusted_input.md#nesting-depth)). Parsing, the binary readers, +[`flatten`](../basic_json/flatten.md), [`unflatten`](../basic_json/unflatten.md), and +[`to_bson`](../basic_json/to_bson.md) always use an explicit stack and do not depend on the macro. The lockstep +comparison of two [`json_view`](../basic_json_view/index.md) documents has a fixed bound of its own, 48 levels, +and does not depend on it either. + +The call stack that these operations need grows with the limit: roughly *a few kilobytes plus a cost per level times the +limit*, where the cost per level is up to about 2.3 KiB in an unoptimized and up to about 0.5 KiB in an optimized build +(see [Stack needed](#stack-needed)). The default of `128` levels is tuned for threads with ordinary stacks of a few +hundred kilobytes or more. Set a lower limit if your threads have small stacks (RTOS tasks, fibers, threads of the musl +C +library, which has a default stack of 128 KiB), or a higher limit to trade stack for speed on deeply nested values. + +The macro changes only how much stack is used and how fast the operations are. It never changes a result: every +operation returns the same value for every setting of the limit, for values nested less deeply than the limit as well as +for values nested more deeply. + +## Default definition + +The default value is `128`. + +```cpp +#define JSON_RECURSION_DEPTH_LIMIT 128 +``` + +When the macro is not defined, the library defines it to its default value. + +## Notes + +!!! note "Allowed values" + + The value must be an integer constant expression from `0` to `254`; any other value (a negative number, a number + above `254`, a floating-point number such as `254.0`, or `true` or `false`) is rejected at compile time with an + error message that names the macro. (A character such as `'a'` is an integer, too, and is accepted as its code.) + + - `0` is allowed, and means that every operation uses its explicit stack from the first level on and no operation + recurses over the nesting of a value at all. That needs a constant amount of call stack (at most 16 KiB in the + measurements below, for a value nested 20,000 levels deep) and is the setting for tiny stacks, at the price of + speed (see [Speed](#speed)). + - `254` is the largest allowed value. Copying a value and comparing two values (with `<=>`, or with the operators on + objects and arrays of a custom type) cannot pass the depth down as an argument, because the signatures of the + copy constructor and the operators are fixed. They count the levels they have descended into in a variable of type + `#!cpp std::uint8_t` that exists once per thread (see [`JSON_NO_THREAD_LOCAL`](json_no_thread_local.md)). Only + the levels that are descended into are counted, so the count never exceeds the limit, not even when user code (a + copy constructor of a custom base class, say) copies a value while a copy is going on. A byte holds `255` at + most, and the limit stays below that, so that the count cannot wrap around. (More levels would hardly be + useful: `254` levels need up to 576 KiB of stack in an unoptimized build already, twice as much as the + default.) + +### Stack needed + +The tables show the smallest stack that an operation needs, in KiB, at several limits, on a value that is nested +more deeply than the limit (limit + 10 levels). Each cell shows the result for Clang (Apple clang 21) and for +GCC 16, separated by a slash; both measured on macOS with arm64, compiling in C++20 mode. The operations run in a +thread whose stack has the size given, and are not given any stack that they do not need: the value is built, and +the result destroyed, outside of that thread. Stacks can only be set in steps of 16 KiB (the page size), so `16` +means "16 KiB or less". A row stands for all operations listed; the table shows the largest value among them. + +Unoptimized (`-O0`): + +| Operation | limit 16 | limit 32 | limit 64 | limit 128 | limit 254 | +|---|---:|---:|---:|---:|---:| +| `dump()` and `dump(indent)`, `operator<<` | 48 / 32 | 80 / 64 | 144 / 128 | 288 / 256 | 576 / 512 | +| copy construction and assignment | 32 / 16 | 48 / 32 | 112 / 80 | 224 / 144 | 432 / 304 | +| `==` | 16 / 16 | 16 / 16 | 16 / 16 | 48 / 32 | 96 / 48 | +| `<=>` (C++20) | 16 / 16 | 16 / 16 | 32 / 32 | 64 / 64 | 144 / 128 | +| `std::hash` | 16 / 16 | 16 / 16 | 32 / 32 | 80 / 64 | 160 / 128 | +| `merge_patch`, `update(j, true)`, `patch` | 32 / 16 | 48 / 32 | 112 / 80 | 224 / 144 | 432 / 304 | +| `diff` | 16 / 16 | 16 / 16 | 32 / 32 | 64 / 48 | 144 / 96 | +| `merge_diff` | 16 / 16 | 48 / 32 | 80 / 48 | 176 / 112 | 352 / 224 | +| `to_cbor`, `to_msgpack` | 16 / 16 | 32 / 16 | 64 / 32 | 112 / 80 | 224 / 160 | +| `to_ubjson`, `to_bjdata` | 16 / 16 | 16 / 16 | 48 / 32 | 96 / 64 | 176 / 112 | +| `to_bon8` | 16 / 16 | 16 / 16 | 16 / 16 | 32 / 16 | 64 / 48 | +| conversion to another `basic_json` type | 32 / 16 | 64 / 32 | 112 / 80 | 224 / 144 | 448 / 304 | +| destruction | 16 / 16 | 16 / 16 | 16 / 16 | 16 / 16 | 32 / 32 | + +Optimized (`-O2`): + +| Operation | limit 16 | limit 32 | limit 64 | limit 128 | limit 254 | +|---|---:|---:|---:|---:|---:| +| `dump()` and `dump(indent)`, `operator<<` | 16 / 16 | 16 / 16 | 16 / 16 | 16 / 16 | 32 / 32 | +| copy construction and assignment | 16 / 16 | 16 / 16 | 16 / 16 | 32 / 32 | 64 / 48 | +| `==` | 16 / 16 | 16 / 16 | 16 / 16 | 16 / 16 | 16 / 16 | +| `<=>` (C++20) | 16 / 16 | 16 / 16 | 16 / 16 | 16 / 16 | 48 / 16 | +| `std::hash` | 16 / 16 | 16 / 16 | 16 / 16 | 32 / 48 | 80 / 80 | +| `merge_patch`, `update(j, true)`, `patch` | 16 / 16 | 16 / 16 | 16 / 16 | 32 / 32 | 64 / 48 | +| `diff` | 16 / 16 | 16 / 16 | 16 / 32 | 48 / 48 | 96 / 112 | +| `merge_diff` | 16 / 16 | 16 / 16 | 16 / 16 | 48 / 32 | 96 / 64 | +| `to_cbor`, `to_msgpack` | 16 / 16 | 16 / 16 | 16 / 16 | 16 / 16 | 16 / 16 | +| `to_ubjson`, `to_bjdata` | 16 / 16 | 16 / 16 | 16 / 16 | 32 / 32 | 64 / 64 | +| `to_bon8` | 16 / 16 | 16 / 16 | 16 / 16 | 16 / 32 | 32 / 64 | +| conversion to another `basic_json` type | 16 / 16 | 16 / 16 | 16 / 16 | 32 / 48 | 64 / 96 | +| destruction | 16 / 16 | 16 / 16 | 16 / 16 | 16 / 16 | 16 / 16 | + +With a limit of `0`, every operation in the tables ran in a 16 KiB stack at `-O0` (Clang and GCC), on a value nested +20,000 levels deep. The operations that do not depend on the macro (`flatten`, `unflatten`, `to_bson`) needed +16 KiB or less at every limit. + +The rule of thumb is + +> stack needed ≈ a few KiB + cost per level × limit + +with a cost per level of up to 2.3 KiB (`dump` at `-O0` with Clang; 2.1 KiB with GCC), 1.7 KiB for copying and merging +(1.3 KiB with GCC), and 0.1 to 0.5 KiB for most other operations at `-O0`; and 0.5 KiB or less at `-O2`. The numbers +come from the difference between the limits `128` and `254` in the tables, so they are rough: other compilers, +other targets (the frames of x86-64 code are about as large), other standard libraries, `JSON_DIAGNOSTICS`, and +sanitizers change them. Measure with your own tool chain (see the example below) rather than relying on the tables. +The stack that your own code and the operating system need comes on top of these numbers. + +Which limit to choose, for the stack that you can spare for the library (the largest limit of those measured +for which the slowest operation needs at most that much): + +| Stack for the library | optimized (`-O2`) | unoptimized (`-O0`) | +|---|---:|---:| +| 32 KiB | 64 | 0 | +| 64 KiB | 128 | 16 | +| 128 KiB (the default stack of threads on musl) | 254 | 32 | +| 256 KiB | 254 | 64 | +| 512 KiB | 254 | 128 | +| 1 MiB or more | 254 | 254 | + +### Speed + +Operations are faster the more of a value they handle with plain recursion. The levels below the limit are handled with +an explicit stack, which has to be kept on the heap and costs more per level. The macro therefore trades speed against +stack. + +The table shows the time of copying a value (including destroying the copy), of `==` on two equal values, of `dump`, and +of `std::hash` for several limits, relative to the default of `128` (`1.00` means the same time; more is slower). It was +measured with Clang 21 at `-O2` in C++17 mode on macOS with arm64, as the best of seven runs of each setting (two +rounds, best of both). The documents are ordinary JSON files that are nested 7 (`canada.json`), 8 +(`citm_catalog.json`), 10 (`twitter.json`), and 2 levels deep (`jeopardy.json`); the other values are one array or object +with one member per level, nested 1,000 and 100,000 levels deep. The absolute time at the limit `128` is given in the +third column. Differences of up to about 0.1 are noise: a different limit changes the layout of the compiled code, and +timings of the same binary vary by about as much. The slow columns (limits 0 and 1) vary more: repeating `std::hash` on +`canada.json` gave 1.27 to 1.30 for limit 0 and 1.4 to 1.9 for limit 1, so read the cells of those two columns as +accurate to about 0.3. + +| Value | Operation | limit 128 (absolute) | limit 0 | limit 1 | limit 16 | limit 254 | +|---|---|---:|---:|---:|---:|---:| +| `canada.json` (2.1 MB) | copy | 3.5 ms | 1.80 | 1.80 | 0.99 | 1.00 | +| `canada.json` (2.1 MB) | `==` | 350.5 µs | 2.36 | 2.38 | 1.00 | 0.99 | +| `canada.json` (2.1 MB) | `dump` | 2.4 ms | 1.11 | 1.11 | 1.00 | 1.00 | +| `canada.json` (2.1 MB) | `std::hash` | 920.2 µs | 1.33 | 1.59 | 1.00 | 1.00 | +| `citm_catalog.json` (1.7 MB) | copy | 1.8 ms | 1.38 | 1.36 | 1.00 | 0.99 | +| `citm_catalog.json` (1.7 MB) | `==` | 275.5 µs | 1.31 | 1.31 | 0.99 | 1.00 | +| `citm_catalog.json` (1.7 MB) | `dump` | 314.6 µs | 1.24 | 1.23 | 1.00 | 1.00 | +| `citm_catalog.json` (1.7 MB) | `std::hash` | 309.1 µs | 1.77 | 1.42 | 1.06 | 1.05 | +| `twitter.json` (0.6 MB) | copy | 865.7 µs | 1.45 | 1.47 | 1.01 | 1.00 | +| `twitter.json` (0.6 MB) | `==` | 135.8 µs | 1.26 | 1.23 | 1.00 | 1.00 | +| `twitter.json` (0.6 MB) | `dump` | 224.6 µs | 1.12 | 1.12 | 1.01 | 1.00 | +| `twitter.json` (0.6 MB) | `std::hash` | 132.9 µs | 1.28 | 1.27 | 1.03 | 1.04 | +| `jeopardy.json` (53 MB) | copy | 111.1 ms | 1.22 | 1.25 | 1.00 | 1.00 | +| `jeopardy.json` (53 MB) | `==` | 23.5 ms | 1.29 | 1.53 | 1.07 | 1.00 | +| `jeopardy.json` (53 MB) | `dump` | 33.2 ms | 1.14 | 1.34 | 1.03 | 1.01 | +| `jeopardy.json` (53 MB) | `std::hash` | 19.9 ms | 1.17 | 1.52 | 1.05 | 1.00 | +| arrays, 1,000 levels | copy | 69.9 µs | 0.95 | 0.98 | 0.92 | 0.99 | +| arrays, 1,000 levels | `==` | 10.4 µs | 1.02 | 1.01 | 0.96 | 1.00 | +| arrays, 1,000 levels | `dump` | 9.6 µs | 0.97 | 0.96 | 0.97 | 1.06 | +| arrays, 1,000 levels | `std::hash` | 16.9 µs | 1.00 | 1.01 | 1.01 | 0.98 | +| objects, 1,000 levels | copy | 77.1 µs | 0.98 | 0.97 | 0.97 | 1.03 | +| objects, 1,000 levels | `==` | 12.0 µs | 0.96 | 0.95 | 0.95 | 1.07 | +| objects, 1,000 levels | `dump` | 12.6 µs | 0.97 | 0.95 | 0.98 | 1.01 | +| objects, 1,000 levels | `std::hash` | 17.3 µs | 0.99 | 1.01 | 0.98 | 1.00 | +| arrays, 100,000 levels | copy | 6.4 ms | 1.00 | 1.00 | 1.00 | 1.00 | +| arrays, 100,000 levels | `==` | 1.1 ms | 1.01 | 1.03 | 1.00 | 1.06 | +| arrays, 100,000 levels | `dump` | 917.9 µs | 1.01 | 1.01 | 1.02 | 1.01 | +| arrays, 100,000 levels | `std::hash` | 1.7 ms | 1.04 | 1.05 | 0.99 | 0.99 | +| objects, 100,000 levels | copy | 7.5 ms | 0.99 | 1.02 | 1.01 | 0.99 | +| objects, 100,000 levels | `==` | 1.3 ms | 1.02 | 1.09 | 1.08 | 0.96 | +| objects, 100,000 levels | `dump` | 1.2 ms | 1.06 | 0.99 | 1.01 | 0.99 | +| objects, 100,000 levels | `std::hash` | 1.8 ms | 1.00 | 1.00 | 1.00 | 1.00 | + +The results are as follows. + +- A limit of `16` or more does not change the speed of ordinary documents (differences are within noise), because they + are not nested more deeply than that anyway. A document nested more deeply than the limit is walked partly with the + explicit stack. +- A limit of `0` or `1` makes the operations on ordinary documents 1.1 to 2.4 times as slow, because even their first + levels are handled with the explicit stack. The slowdown is largest for `==` on a document of many small containers + (`canada.json`: 2.4 times) and for copying (up to 1.8 times). +- For values that are nested far more deeply than any limit (1,000 and 100,000 levels deep), the limit makes no + measurable difference: almost all of the levels are handled with the explicit stack for every setting. + Raising the limit gains nothing for them, and lowering it costs nothing. + +!!! note "Other macros" + + - With [`JSON_NO_THREAD_LOCAL`](json_no_thread_local.md) defined (or for Clang targeting MinGW, where the library + defines it by itself), copying and comparing never descend, so for them the limit does not matter. The other + operations still recurse as many levels as the limit says. The range of the limit is the same. + - The macro is not part of the [ABI tag](../../features/namespace.md), as `JSON_NO_THREAD_LOCAL` is not, and it is + not undefined at the end of `json.hpp`. + +!!! warning "Use the same value in every translation unit" + + All translation units of a program that include the library must define the same value. Otherwise, the program + violates the One Definition Rule: the function `nlohmann::detail::recursion_depth_limit()` and all the functions + that + use it are defined differently in different translation units. The linker keeps one definition of each function that + several translation units define, and every caller then recurses as far as that definition says, whatever value + its own translation unit defined. Each function uses the limit of whichever translation unit's copy was kept, and + which one that is is not specified. + + Mixing values cannot corrupt the byte per thread that counts the levels of copying and comparing: each limit + is at most `254`, and all of them stop counting at their own limit, so the count never exceeds `254`, whichever + translation unit's definition runs. The counter exists once per thread and `basic_json` specialization, and is + shared by all translation units. Nor can mixing values change any result, as no result depends on the limit. What + changes is how much stack is used and how fast the operations are. Nevertheless, avoid it: the program has undefined + behavior as far as the C++ standard is concerned. + +## Examples + +??? example "Example: lower the limit" + + The example nests an array 5,000 levels deep. With the limit set to 16, the library recurses over the first 16 + levels + and over the rest uses an explicit stack. The results are those of the default limit. + + ```cpp + --8<-- "examples/json_recursion_depth_limit.cpp" + ``` + + Output: + + ``` + --8<-- "examples/json_recursion_depth_limit.output" + ``` + +??? example "Example: find out how much stack an operation needs" + + The program runs `dump()` on a value nested `depth` levels deep in a thread with a stack of `kib` KiB. Compile it + with the options and the limit that you use, e.g. `-DJSON_RECURSION_DEPTH_LIMIT=64`, run it with `depth` set to the + limit plus 10 and decreasing values of `kib` (the stack size must be a multiple of the page size, often 4 or 16 + KiB), + and note the smallest stack with which it still finishes. That is how the tables above were measured. + + ```cpp + #include + #include + #include + #include + #include + #include + + static nlohmann::json value; + + static void* run(void*) + { + std::cout << "dump() ok, " << value.dump().size() << " bytes\n"; + return nullptr; + } + + int main(int argc, char** argv) + { + const int depth = std::atoi(argv[1]); + const std::size_t kib = static_cast(std::atoi(argv[2])); + + // built on the main thread, without recursion + value = 1; + for (int i = 0; i < depth; ++i) + { + nlohmann::json outer = nlohmann::json::array(); + outer.push_back(std::move(value)); + value = std::move(outer); + } + + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setstacksize(&attr, kib * 1024); + pthread_t thread; + pthread_create(&thread, &attr, run, nullptr); + pthread_join(thread, nullptr); + } + ``` + +## See also + +- [Nesting depth](../../features/parsing/untrusted_input.md#nesting-depth) - how the library handles deeply nested + values +- [`JSON_NO_THREAD_LOCAL`](json_no_thread_local.md) - switch off the use of `thread_local` storage +- [Supported Macros](../../features/macros.md) - guide to all supported macros + +## Version history + +- Added in version 3.13.0. diff --git a/docs/mkdocs/docs/community/assurance_case.md b/docs/mkdocs/docs/community/assurance_case.md index 02dc6d7d7..54c9a344b 100644 --- a/docs/mkdocs/docs/community/assurance_case.md +++ b/docs/mkdocs/docs/community/assurance_case.md @@ -53,12 +53,15 @@ flowchart LR only as bytes are actually read, arrays reserve at most a fixed number of elements up front, and sizes that no container can hold are rejected. - **Memory is owned by values.** Each `basic_json` value owns its content, and there is no manual memory management in - user code. The destructor does not recurse, so destroying a deeply nested value does not exhaust the stack. + user code. The destructor recurses only over a bounded number of nesting levels and then continues without the call + stack, so destroying a deeply nested value does not exhaust the stack. - **Bounded recursion.** The JSON parser and the binary readers keep their state in explicit stacks instead of recursing per nesting level. Operations that walk a value, such as [`dump`](../api/basic_json/dump.md), copying, comparison, hashing, [`merge_patch`](../api/basic_json/merge_patch.md), [`diff`](../api/basic_json/diff.md), [`flatten`](../api/basic_json/flatten.md), and the binary writers, recurse only up to a fixed depth and continue with - an explicit stack below it. Applications that process untrusted input can limit its nesting depth with a + an explicit stack below it. The depth is 128 levels by default and can be changed with + [`JSON_RECURSION_DEPTH_LIMIT`](../api/macros/json_recursion_depth_limit.md), for example to run on a small stack. + Applications that process untrusted input can limit its nesting depth with a [parser callback](../features/parsing/parser_callbacks.md). - **Invariants are checked.** The class invariant (for instance, that the pointer for the stored type is never null) is checked with runtime assertions throughout the test suite. diff --git a/docs/mkdocs/docs/examples/json_recursion_depth_limit.cpp b/docs/mkdocs/docs/examples/json_recursion_depth_limit.cpp new file mode 100644 index 000000000..7a2bfdfc3 --- /dev/null +++ b/docs/mkdocs/docs/examples/json_recursion_depth_limit.cpp @@ -0,0 +1,29 @@ +#include +#include +#include + +// walk over at most 16 nesting levels on the call stack, and over the rest with an explicit stack +#define JSON_RECURSION_DEPTH_LIMIT 16 +#include + +using json = nlohmann::json; + +int main() +{ + // an array nested 5,000 levels deep, built without recursion + json j = 1; + for (int i = 0; i < 5000; ++i) + { + json outer = json::array(); + outer.push_back(std::move(j)); + j = std::move(outer); + } + + // the results are those of any other limit + const json copy = j; + std::cout << std::boolalpha; + std::cout << "limit: " << JSON_RECURSION_DEPTH_LIMIT << '\n'; + std::cout << "copy equals original: " << (copy == j) << '\n'; + std::cout << "length of dump(): " << j.dump().size() << '\n'; + std::cout << "same hash: " << (std::hash {}(copy) == std::hash {}(j)) << '\n'; +} diff --git a/docs/mkdocs/docs/examples/json_recursion_depth_limit.output b/docs/mkdocs/docs/examples/json_recursion_depth_limit.output new file mode 100644 index 000000000..efb8175db --- /dev/null +++ b/docs/mkdocs/docs/examples/json_recursion_depth_limit.output @@ -0,0 +1,4 @@ +limit: 16 +copy equals original: true +length of dump(): 10001 +same hash: true diff --git a/docs/mkdocs/docs/features/macros.md b/docs/mkdocs/docs/features/macros.md index 33b2273b9..ace29de02 100644 --- a/docs/mkdocs/docs/features/macros.md +++ b/docs/mkdocs/docs/features/macros.md @@ -198,6 +198,16 @@ this is planned to become the default in version 4.0.0. See [full documentation of `JSON_PRECISE_STREAM_POSITION`](../api/macros/json_precise_stream_position.md). +## `JSON_RECURSION_DEPTH_LIMIT` + +Sets the number of nesting levels that the operations walking a value (copying, comparing, serializing, hashing, +merging, diffing, converting, destroying, and the binary writers) recurse into on the call stack before they continue +with an explicit stack on the heap. The default is `128`; allowed are the integers from `0` to `254`. Lower it for +threads with small stacks, raise it to trade stack for speed. Results are the same for every setting; the value must be +the same in all translation units of a program. + +See [full documentation of `JSON_RECURSION_DEPTH_LIMIT`](../api/macros/json_recursion_depth_limit.md). + ## `JSON_SKIP_LIBRARY_VERSION_CHECK` When defined, the library will not create a compiler warning when a different version of the library was already diff --git a/docs/mkdocs/docs/features/parsing/untrusted_input.md b/docs/mkdocs/docs/features/parsing/untrusted_input.md index 1fe7b64aa..d0c56ca44 100644 --- a/docs/mkdocs/docs/features/parsing/untrusted_input.md +++ b/docs/mkdocs/docs/features/parsing/untrusted_input.md @@ -54,10 +54,32 @@ Once a value has been parsed, operations that walk it recursively — serializin [`dump`](../../api/basic_json/dump.md), hashing it, copying it, comparing two values with `#!cpp ==`, `#!cpp <`, or (in C++20) `#!cpp <=>`, merging with [`update`](../../api/basic_json/update.md), applying a [`merge_patch`](../../api/basic_json/merge_patch.md), creating a patch with [`diff`](../../api/basic_json/diff.md) or -[`merge_diff`](../../api/basic_json/merge_diff.md), [`flatten`](../../api/basic_json/flatten.md), and the binary -writers (`to_cbor`, `to_msgpack`, ...) — descend at most 128 levels on the call stack and continue -below that with an explicit stack instead, so none of them can exhaust the stack either, however deeply the value is -nested. Destroying a value (its destructor) never recurses at all, regardless of nesting depth, for the same reason. +[`merge_diff`](../../api/basic_json/merge_diff.md), converting it to another `basic_json` type, destroying it, and the +binary writers (`to_cbor`, `to_msgpack`, ...) — descend at most +[`JSON_RECURSION_DEPTH_LIMIT`](../../api/macros/json_recursion_depth_limit.md) levels on the call stack, 128 by default, +and continue below that with an explicit stack on the heap. So the call stack does not grow with the nesting depth of the +value, however deeply it is nested. ([`flatten`](../../api/basic_json/flatten.md), +[`unflatten`](../../api/basic_json/unflatten.md), and [`to_bson`](../../api/basic_json/to_bson.md) never use the call +stack for the nesting.) + +The call stack still needs room for the levels that are recursed into, and the default limit assumes a stack of ordinary +size. The stack that an operation needs is roughly proportional to the limit; with the default of 128 levels, the most +demanding operation (`dump` in an unoptimized build) needed up to 288 KiB, and an optimized build needed 48 KiB or less, +in measurements on macOS with arm64. The most demanding operation needed this much stack at the limits `0`, `32`, +`128`, and `254`: + +| Stack needed, in KiB | limit 0 | limit 32 | limit 128 (default) | limit 254 | +|---|---:|---:|---:|---:| +| unoptimized build (`-O0`) | 16 or less | 80 | 288 | 576 | +| optimized build (`-O2`) | 16 or less | 16 or less | 48 | 112 | + +A thread with a smaller stack (a task of a real-time operating system, a fiber, or a thread of the musl C library, whose +default stack is 128 KiB) can still overflow its stack on deeply nested input, which is why the limit can be lowered: +define `JSON_RECURSION_DEPTH_LIMIT` to a smaller value, down to `0`, before including the library. Such a limit changes +only the stack that is needed and the speed (values nested more deeply than the limit are walked more slowly), never a +result. See the [documentation of the macro](../../api/macros/json_recursion_depth_limit.md) for the complete tables, +the speed, and the rules for choosing a value. Destroying a value (its destructor) recurses over the same number of +levels and then continues without the call stack, and needs 32 KiB or less at every limit. ## Input size diff --git a/docs/mkdocs/mkdocs.yml b/docs/mkdocs/mkdocs.yml index 4c587ad9d..44e06c4ff 100644 --- a/docs/mkdocs/mkdocs.yml +++ b/docs/mkdocs/mkdocs.yml @@ -454,6 +454,7 @@ nav: - 'JSON_NO_SIMD': api/macros/json_no_simd.md - 'JSON_NO_THREAD_LOCAL': api/macros/json_no_thread_local.md - 'JSON_PRECISE_STREAM_POSITION': api/macros/json_precise_stream_position.md + - 'JSON_RECURSION_DEPTH_LIMIT': api/macros/json_recursion_depth_limit.md - 'JSON_SKIP_LIBRARY_VERSION_CHECK': api/macros/json_skip_library_version_check.md - 'JSON_SKIP_UNSUPPORTED_COMPILER_CHECK': api/macros/json_skip_unsupported_compiler_check.md - 'JSON_STRICT_BINARY_UTF8': api/macros/json_strict_binary_utf8.md diff --git a/include/nlohmann/detail/macro_scope.hpp b/include/nlohmann/detail/macro_scope.hpp index 208c1c3ce..76d191128 100644 --- a/include/nlohmann/detail/macro_scope.hpp +++ b/include/nlohmann/detail/macro_scope.hpp @@ -224,6 +224,12 @@ #define JSON_NO_THREAD_LOCAL 1 #endif +// JSON_RECURSION_DEPTH_LIMIT (the number of nesting levels the walks over a +// value recurse into before they continue with an explicit stack; 128 by +// default, 0 to 254 allowed) is defined and checked where it is used, in +// detail/recursion_depth_limit.hpp. Like JSON_NO_THREAD_LOCAL it is neither part +// of the ABI tag nor undefined at the end of json.hpp. + // allow disabling exceptions #if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) #define JSON_THROW(exception) throw exception diff --git a/include/nlohmann/detail/recursion_depth_limit.hpp b/include/nlohmann/detail/recursion_depth_limit.hpp index 6553a4d05..9eccd2fdb 100644 --- a/include/nlohmann/detail/recursion_depth_limit.hpp +++ b/include/nlohmann/detail/recursion_depth_limit.hpp @@ -9,9 +9,22 @@ #pragma once #include // size_t +#include // is_integral, is_same #include +// JSON_RECURSION_DEPTH_LIMIT: the number of nesting levels the operations that +// walk a value recurse into on the call stack before they continue with an +// explicit stack (see detail::recursion_depth_limit() below). +// +// May be defined by the user before the first include of the library; it has +// to be the same in every translation unit of a program (it is not part of the +// ABI tag, see https://json.nlohmann.me/api/macros/json_recursion_depth_limit/). +// Allowed: an integer constant expression from 0 to 254. The default is 128. +#ifndef JSON_RECURSION_DEPTH_LIMIT + #define JSON_RECURSION_DEPTH_LIMIT 128 +#endif + NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail { @@ -20,21 +33,59 @@ namespace detail @brief the number of nesting levels an operation recurses into Operations that walk a value (copying, comparing, serializing, hashing, merging, -...) recurse once -per nesting level, which is fastest, but a value nested deeply enough would -exhaust the call stack. So they recurse only this many levels deep and finish -whatever lies below with an explicit stack. All of them share this limit. +diffing, converting, destroying, and writing the binary formats CBOR, +MessagePack, UBJSON, BJData, and BON8) recurse once per nesting level, which is +fastest, but a value nested deeply enough would exhaust the call stack. So they +recurse only this many levels deep and finish whatever lies below with an +explicit stack that lives on the heap. All of them share this limit and none has +a constant of its own. It is the value of the macro JSON_RECURSION_DEPTH_LIMIT, +128 by default. (Parsing, the binary readers, flatten(), unflatten(), and +to_bson() always use an explicit stack and do not read it, and the lockstep +comparison of json_view documents has a fixed bound of its own, 48 levels.) -Most of them pass the depth down as an argument. The copy constructor and the -comparison operators cannot, as their signatures are fixed, so they count it -in basic_json::nesting_depth() instead, a byte per thread; the limit must -therefore stay below 255. +Most of the operations pass the depth down as an argument. The copy constructor +and the comparison operators cannot, as their signatures are fixed, so they +count it in basic_json::nesting_depth() instead, a byte per thread. Only the +levels that are descended into are counted, so the count never exceeds the limit. +The limit must fit in the byte with some headroom, so that the count cannot wrap +around: it must be below 255, and 254 is the largest value allowed. (More levels +would hardly be useful: 254 levels need up to 576 KiB of stack in an unoptimized +build already, twice the stack of the default.) +The smallest value is 0: every operation then uses its explicit stack from the +first level on and no operation recurses over the nesting of a value at all. +That is slower, but needs a constant amount of call stack. It works because +every use of the limit compares a depth with it (`depth >= limit`, or `<` for +the opposite), which holds for 0 as for any other value; nothing computes +`limit - 1` or `limit + 1`, so nothing wraps around. Each operation's explicit +stack handles any nesting level, including the first, so none of them relies on +having recursed first. + +The call stack that an operation needs is roughly a few KiB plus a cost per level +times the limit. At the default of 128 levels, an unoptimized build needs up to +about 290 KiB (`dump()` with Clang at -O0: about 2.3 KiB per level) and an +optimized build up to about 48 KiB; with 254 levels it is up to 576 KiB and +112 KiB, and with 0 it is 16 KiB or less. Lower the limit for threads with small +stacks (RTOS tasks, fibers, musl threads), raise it to trade stack for speed on +deeply nested values. All of this, with the measurements, is in the +documentation of the macro. + +@sa https://json.nlohmann.me/api/macros/json_recursion_depth_limit/ @sa https://github.com/nlohmann/json/issues/5387 */ constexpr std::size_t recursion_depth_limit() noexcept { - return 128; + // an integer: not a floating-point number, whose range check would pass for 1.5 or 254.0 + // (bool is an integer type, too, but 1 and 0 spelled true and false are no way to say a limit) + static_assert(std::is_integral::value + && !std::is_same::value, + "JSON_RECURSION_DEPTH_LIMIT must be an integer constant"); + static_assert(JSON_RECURSION_DEPTH_LIMIT >= 0 && JSON_RECURSION_DEPTH_LIMIT <= 254, + "JSON_RECURSION_DEPTH_LIMIT must be between 0 and 254: the nesting depth is counted in a byte per thread " + "and may exceed the limit by one (see https://json.nlohmann.me/api/macros/json_recursion_depth_limit/)"); + // (a value out of range has been reported by now; the remainder keeps it + // below 255 for any value, so that basic_json does not report it again) + return static_cast(JSON_RECURSION_DEPTH_LIMIT) % 255; } } // namespace detail diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index bd06811d9..0312f7c27 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -1256,8 +1256,12 @@ public: } #ifndef JSON_NO_THREAD_LOCAL - // nesting_depth() is a byte and may exceed the limit by one level - static_assert(detail::recursion_depth_limit() < 255, "the nesting depth count must fit in a byte"); + // nesting_depth() is a byte that never exceeds the limit (JSON_RECURSION_DEPTH_LIMIT), + // which is at most 254: the largest value stays one below the byte's maximum, so + // that the count cannot wrap around to 0 and let a descent go on without a bound. + // (recursion_depth_limit() checks the macro itself, with a message that + // names it; this is the check that cannot be edited away unnoticed.) + static_assert(detail::recursion_depth_limit() < 255, "the nesting depth count must fit in a byte: JSON_RECURSION_DEPTH_LIMIT must not exceed 254"); /*! @brief how many levels the operation going on in this thread has descended into @@ -1268,8 +1272,15 @@ public: user code nests them anyway, sharing the count only ends a descent sooner than it had to, which costs a little speed and is never wrong. - A byte is enough: the count never exceeds the limit by more than the single - level that notices the limit has been reached. + A byte is enough: only the levels that are descended into are counted + (@ref nesting_depth_guard), so the count never exceeds the limit, and the + limit is at most 254. + + The count is per thread and not per value or per library configuration: all + translation units of a program share it, even if they were compiled with + different values of JSON_RECURSION_DEPTH_LIMIT (which is not supported, see + the documentation of the macro); each of them then compares the shared count + with its own limit. */ static std::uint8_t& nesting_depth() noexcept { @@ -1327,14 +1338,25 @@ public: #endif { #ifndef JSON_NO_THREAD_LOCAL - ++nesting_depth(); + // Only a level that is entered is counted, so that the count never + // exceeds the limit. A level that is not entered is finished without + // descending, and whatever runs while it is (a copy made by a custom + // base class, say) sees the same count as the level itself did and + // is not entered either. + if (m_okay) + { + ++nesting_depth(); + } #endif } ~nesting_depth_guard() { #ifndef JSON_NO_THREAD_LOCAL - --nesting_depth(); + if (m_okay) + { + --nesting_depth(); + } #endif } @@ -5246,7 +5268,10 @@ public: */ void update_members(const iterator& first, const iterator& last, const bool merge_objects, const std::size_t depth) { - if (JSON_HEDLEY_UNLIKELY(depth >= detail::recursion_depth_limit())) + // without merge_objects the loop below never descends, so there is no + // call stack to save (and the explicit stack always merges); the + // distinction matters for a limit of 0, which reaches this at depth 0 + if (JSON_HEDLEY_UNLIKELY(merge_objects && depth >= detail::recursion_depth_limit())) { update_members_iteratively(first, last); return; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 0db1793ac..0d67996c2 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -2597,6 +2597,12 @@ JSON_HEDLEY_DIAGNOSTIC_POP #define JSON_NO_THREAD_LOCAL 1 #endif +// JSON_RECURSION_DEPTH_LIMIT (the number of nesting levels the walks over a +// value recurse into before they continue with an explicit stack; 128 by +// default, 0 to 254 allowed) is defined and checked where it is used, in +// detail/recursion_depth_limit.hpp. Like JSON_NO_THREAD_LOCAL it is neither part +// of the ABI tag nor undefined at the end of json.hpp. + // allow disabling exceptions #if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) #define JSON_THROW(exception) throw exception @@ -8600,10 +8606,23 @@ NLOHMANN_JSON_NAMESPACE_END #include // size_t +#include // is_integral, is_same // #include +// JSON_RECURSION_DEPTH_LIMIT: the number of nesting levels the operations that +// walk a value recurse into on the call stack before they continue with an +// explicit stack (see detail::recursion_depth_limit() below). +// +// May be defined by the user before the first include of the library; it has +// to be the same in every translation unit of a program (it is not part of the +// ABI tag, see https://json.nlohmann.me/api/macros/json_recursion_depth_limit/). +// Allowed: an integer constant expression from 0 to 254. The default is 128. +#ifndef JSON_RECURSION_DEPTH_LIMIT + #define JSON_RECURSION_DEPTH_LIMIT 128 +#endif + NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail { @@ -8612,21 +8631,59 @@ namespace detail @brief the number of nesting levels an operation recurses into Operations that walk a value (copying, comparing, serializing, hashing, merging, -...) recurse once -per nesting level, which is fastest, but a value nested deeply enough would -exhaust the call stack. So they recurse only this many levels deep and finish -whatever lies below with an explicit stack. All of them share this limit. +diffing, converting, destroying, and writing the binary formats CBOR, +MessagePack, UBJSON, BJData, and BON8) recurse once per nesting level, which is +fastest, but a value nested deeply enough would exhaust the call stack. So they +recurse only this many levels deep and finish whatever lies below with an +explicit stack that lives on the heap. All of them share this limit and none has +a constant of its own. It is the value of the macro JSON_RECURSION_DEPTH_LIMIT, +128 by default. (Parsing, the binary readers, flatten(), unflatten(), and +to_bson() always use an explicit stack and do not read it, and the lockstep +comparison of json_view documents has a fixed bound of its own, 48 levels.) -Most of them pass the depth down as an argument. The copy constructor and the -comparison operators cannot, as their signatures are fixed, so they count it -in basic_json::nesting_depth() instead, a byte per thread; the limit must -therefore stay below 255. +Most of the operations pass the depth down as an argument. The copy constructor +and the comparison operators cannot, as their signatures are fixed, so they +count it in basic_json::nesting_depth() instead, a byte per thread. Only the +levels that are descended into are counted, so the count never exceeds the limit. +The limit must fit in the byte with some headroom, so that the count cannot wrap +around: it must be below 255, and 254 is the largest value allowed. (More levels +would hardly be useful: 254 levels need up to 576 KiB of stack in an unoptimized +build already, twice the stack of the default.) +The smallest value is 0: every operation then uses its explicit stack from the +first level on and no operation recurses over the nesting of a value at all. +That is slower, but needs a constant amount of call stack. It works because +every use of the limit compares a depth with it (`depth >= limit`, or `<` for +the opposite), which holds for 0 as for any other value; nothing computes +`limit - 1` or `limit + 1`, so nothing wraps around. Each operation's explicit +stack handles any nesting level, including the first, so none of them relies on +having recursed first. + +The call stack that an operation needs is roughly a few KiB plus a cost per level +times the limit. At the default of 128 levels, an unoptimized build needs up to +about 290 KiB (`dump()` with Clang at -O0: about 2.3 KiB per level) and an +optimized build up to about 48 KiB; with 254 levels it is up to 576 KiB and +112 KiB, and with 0 it is 16 KiB or less. Lower the limit for threads with small +stacks (RTOS tasks, fibers, musl threads), raise it to trade stack for speed on +deeply nested values. All of this, with the measurements, is in the +documentation of the macro. + +@sa https://json.nlohmann.me/api/macros/json_recursion_depth_limit/ @sa https://github.com/nlohmann/json/issues/5387 */ constexpr std::size_t recursion_depth_limit() noexcept { - return 128; + // an integer: not a floating-point number, whose range check would pass for 1.5 or 254.0 + // (bool is an integer type, too, but 1 and 0 spelled true and false are no way to say a limit) + static_assert(std::is_integral::value + && !std::is_same::value, + "JSON_RECURSION_DEPTH_LIMIT must be an integer constant"); + static_assert(JSON_RECURSION_DEPTH_LIMIT >= 0 && JSON_RECURSION_DEPTH_LIMIT <= 254, + "JSON_RECURSION_DEPTH_LIMIT must be between 0 and 254: the nesting depth is counted in a byte per thread " + "and may exceed the limit by one (see https://json.nlohmann.me/api/macros/json_recursion_depth_limit/)"); + // (a value out of range has been reported by now; the remainder keeps it + // below 255 for any value, so that basic_json does not report it again) + return static_cast(JSON_RECURSION_DEPTH_LIMIT) % 255; } } // namespace detail @@ -40406,8 +40463,12 @@ public: } #ifndef JSON_NO_THREAD_LOCAL - // nesting_depth() is a byte and may exceed the limit by one level - static_assert(detail::recursion_depth_limit() < 255, "the nesting depth count must fit in a byte"); + // nesting_depth() is a byte that never exceeds the limit (JSON_RECURSION_DEPTH_LIMIT), + // which is at most 254: the largest value stays one below the byte's maximum, so + // that the count cannot wrap around to 0 and let a descent go on without a bound. + // (recursion_depth_limit() checks the macro itself, with a message that + // names it; this is the check that cannot be edited away unnoticed.) + static_assert(detail::recursion_depth_limit() < 255, "the nesting depth count must fit in a byte: JSON_RECURSION_DEPTH_LIMIT must not exceed 254"); /*! @brief how many levels the operation going on in this thread has descended into @@ -40418,8 +40479,15 @@ public: user code nests them anyway, sharing the count only ends a descent sooner than it had to, which costs a little speed and is never wrong. - A byte is enough: the count never exceeds the limit by more than the single - level that notices the limit has been reached. + A byte is enough: only the levels that are descended into are counted + (@ref nesting_depth_guard), so the count never exceeds the limit, and the + limit is at most 254. + + The count is per thread and not per value or per library configuration: all + translation units of a program share it, even if they were compiled with + different values of JSON_RECURSION_DEPTH_LIMIT (which is not supported, see + the documentation of the macro); each of them then compares the shared count + with its own limit. */ static std::uint8_t& nesting_depth() noexcept { @@ -40477,14 +40545,25 @@ public: #endif { #ifndef JSON_NO_THREAD_LOCAL - ++nesting_depth(); + // Only a level that is entered is counted, so that the count never + // exceeds the limit. A level that is not entered is finished without + // descending, and whatever runs while it is (a copy made by a custom + // base class, say) sees the same count as the level itself did and + // is not entered either. + if (m_okay) + { + ++nesting_depth(); + } #endif } ~nesting_depth_guard() { #ifndef JSON_NO_THREAD_LOCAL - --nesting_depth(); + if (m_okay) + { + --nesting_depth(); + } #endif } @@ -44396,7 +44475,10 @@ public: */ void update_members(const iterator& first, const iterator& last, const bool merge_objects, const std::size_t depth) { - if (JSON_HEDLEY_UNLIKELY(depth >= detail::recursion_depth_limit())) + // without merge_objects the loop below never descends, so there is no + // call stack to save (and the explicit stack always merges); the + // distinction matters for a limit of 0, which reaches this at depth 0 + if (JSON_HEDLEY_UNLIKELY(merge_objects && depth >= detail::recursion_depth_limit())) { update_members_iteratively(first, last); return; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f4e574fc1..e7d90cecd 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -355,6 +355,20 @@ json_test_add_test_for(src/unit-comparison.cpp MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force} ) +# test the walks over deeply nested values with the extreme settings of +# JSON_RECURSION_DEPTH_LIMIT: 0 (every walk uses its explicit stack from the +# first level on), 1, and 254 (the largest limit that fits the byte that counts +# the nesting of a copy) +foreach(recursion_depth_limit 0 1 254) + json_test_set_test_options(test-recursion_depth_limit_${recursion_depth_limit} + COMPILE_DEFINITIONS JSON_RECURSION_DEPTH_LIMIT=${recursion_depth_limit} + ) + json_test_add_test_for(src/unit-recursion_depth_limit.cpp + NAME test-recursion_depth_limit_${recursion_depth_limit} + MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force} + ) +endforeach() + # test the parser again with JSON_DIAGNOSTIC_POSITIONS enabled json_test_set_test_options(test-class_parser_diagnostic_positions COMPILE_DEFINITIONS JSON_DIAGNOSTIC_POSITIONS=1 diff --git a/tests/src/unit-allocator.cpp b/tests/src/unit-allocator.cpp index 9862e8c5e..73693347f 100644 --- a/tests/src/unit-allocator.cpp +++ b/tests/src/unit-allocator.cpp @@ -236,7 +236,8 @@ TEST_CASE("controlled bad_alloc") next_construct_fails = false; - // deeper than the 128 levels the copy constructor descends into + // deeper than the levels the copy constructor descends into + // (recursion_depth_limit(): 128 by default, 254 at most) const std::size_t depth = 300; my_json j = 1; @@ -300,8 +301,9 @@ struct nth_alloc_fails_allocator : std::allocator }; }; -// builds a value nested more than 128 levels deep - the bound the copy -// constructor descends into before it continues without the call stack - and +// builds a value nested more than recursion_depth_limit() (128 by default) +// levels deep - the bound the copy constructor descends into before it +// continues without the call stack - and // checks that a copy survives any single allocation of it failing: every // attempt either throws std::bad_alloc, without crashing or leaving the // source altered, or completes the copy @@ -312,9 +314,9 @@ void check_deep_copy_survives_failing_allocation(bool nest_objects) fail_at_alloc_call = -1; - // [[[ ... [1] ... ]]], or the same nesting with objects, 130 levels deep + // [[[ ... [1] ... ]]], or the same nesting with objects, two levels deeper than the limit BasicJsonType src = 1; - for (std::size_t i = 0; i < 130; ++i) + for (std::size_t i = 0; i < nlohmann::detail::recursion_depth_limit() + 2; ++i) { if (nest_objects) { @@ -440,8 +442,9 @@ TEST_CASE("deep copy uses the provided allocator") { using counting_json = nlohmann::json::with_allocator_t; - // deeper than the 128 levels the copy constructor descends into, so the - // innermost objects are copied by the iterative deep copy + // deeper than the levels the copy constructor descends into (128 by + // default, 254 at most), so the innermost objects are copied by the + // iterative deep copy counting_json j = 1; for (std::size_t i = 0; i < 300; ++i) { @@ -499,11 +502,12 @@ TEST_CASE("converting a deeply nested value from another specialization fails cl #if !(defined(_MSC_VER) && _MSC_VER < 1910 && defined(_ITERATOR_DEBUG_LEVEL) && _ITERATOR_DEBUG_LEVEL > 0) using countdown_json = nlohmann::json::with_allocator_t; - // deeper than the 128 levels the converting constructor descends into, so - // that failures land on both sides of the bound - or, built with - // JSON_NO_THREAD_LOCAL, all in the iterative conversion + // deeper than the levels the converting constructor descends into + // (recursion_depth_limit(), 128 by default), so that failures land on both + // sides of the bound - or, built with JSON_NO_THREAD_LOCAL, all in the + // iterative conversion json j = {1, "two", {{"three", 3}}}; - for (std::size_t i = 0; i < 150; ++i) + for (std::size_t i = 0; i < nlohmann::detail::recursion_depth_limit() + 22; ++i) { j = json{{"a", json::array({j, "sibling"})}}; } diff --git a/tests/src/unit-comparison.cpp b/tests/src/unit-comparison.cpp index 4ac43320b..bec6b136e 100644 --- a/tests/src/unit-comparison.cpp +++ b/tests/src/unit-comparison.cpp @@ -62,6 +62,15 @@ template<> struct StringMaker namespace { +// nesting depths on either side of detail::recursion_depth_limit(): none, the +// last one within the limit, the first one past it, and well past it (0, 127, +// 128, and 200 by default) +std::vector depths_around_recursion_limit() +{ + const std::size_t limit = nlohmann::detail::recursion_depth_limit(); + return {0, limit > 0 ? limit - 1 : 0, limit, limit + 72}; +} + // helper function to check std::less // see https://en.cppreference.com/w/cpp/utility/functional/less template > @@ -968,7 +977,7 @@ TEST_CASE("copying an object preserves its comparator's state") object["a"] = 3; REQUIRE(object.dump() == R"({"B":2,"a":3,"b":1})"); - for (const std::size_t depth : std::vector {0, 127, 128, 200}) + for (const std::size_t depth : depths_around_recursion_limit()) { CAPTURE(depth) @@ -1011,7 +1020,7 @@ TEST_CASE("equality of an object whose comparator treats different keys as equiv // sanity check: the object type's own comparison already disagrees CHECK_FALSE(a.get_ref() == b.get_ref()); - for (const std::size_t depth : std::vector {0, 127, 128, 200}) + for (const std::size_t depth : depths_around_recursion_limit()) { CAPTURE(depth) @@ -1135,9 +1144,10 @@ TEST_CASE("operator<=> of binary values with a different subtype does not depend return result; }; - // 127 levels stay within nesting_depth_limit() (128); 128 and 200 do not, - // and must still agree with the levels that do - for (const std::size_t depth : std::vector {0, 127, 128, 200}) + // one level less than the recursion depth limit (127 by default) stays within it; + // the limit itself and more (128 and 200) do not, and must still agree with the + // levels that do + for (const std::size_t depth : depths_around_recursion_limit()) { CAPTURE(depth) const json x = deep(a, depth); diff --git a/tests/src/unit-custom-array-type.cpp b/tests/src/unit-custom-array-type.cpp index 65285eb54..e5e3307b9 100644 --- a/tests/src/unit-custom-array-type.cpp +++ b/tests/src/unit-custom-array-type.cpp @@ -11,6 +11,7 @@ #include using nlohmann::json; +#include #include #include #include @@ -217,16 +218,17 @@ TEST_CASE("array type with std::deque nested beyond the recursion limit") // recursion_depth_limit() with an explicit stack of frames; with the // iterators of std::deque, these frames are copied and moved through // their own special members + const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 72; // 200 by default deque_json j = 1; - for (int i = 0; i < 200; ++i) + for (std::size_t i = 0; i < depth; ++i) { deque_json outer = deque_json::array(); outer.push_back(std::move(j)); j = std::move(outer); } - // 200 arrays of one element each (0x81) around the integer 1 - std::vector expected(200, 0x81); + // `depth` arrays of one element each (0x81) around the integer 1 + std::vector expected(depth, 0x81); expected.push_back(0x01); CHECK(deque_json::to_cbor(j) == expected); } diff --git a/tests/src/unit-custom-base-class.cpp b/tests/src/unit-custom-base-class.cpp index 71698d041..c3033afe9 100644 --- a/tests/src/unit-custom-base-class.cpp +++ b/tests/src/unit-custom-base-class.cpp @@ -475,10 +475,10 @@ TEST_CASE("Regression test for issue #5674 - copy construction must not require SECTION("nested deeper than the copy constructor's descent bound") { - // beyond nesting_depth_limit() (128) levels, the copy constructor + // beyond recursion_depth_limit() (128 by default) levels, the copy constructor // copies without the call stack (copy_iteratively / copy_array_level), // which used to assign the base class of every element it created - const std::size_t depth = 300; + const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 172; const json_with_const_base j = make_nested_array(depth); const json_with_const_base copy = j; // NOLINT(performance-unnecessary-copy-initialization) diff --git a/tests/src/unit-diagnostic-positions.cpp b/tests/src/unit-diagnostic-positions.cpp index 85284d9d4..8722a8787 100644 --- a/tests/src/unit-diagnostic-positions.cpp +++ b/tests/src/unit-diagnostic-positions.cpp @@ -128,16 +128,19 @@ TEST_CASE("Better diagnostics with positions") check_copy(depth, true); }; + // around detail::recursion_depth_limit() (127, 128, and 129 by default) + const std::size_t limit = nlohmann::detail::recursion_depth_limit(); + check_arrays(1); - check_arrays(127); - check_arrays(128); - check_arrays(129); + check_arrays(limit > 0 ? limit - 1 : 0); + check_arrays(limit); + check_arrays(limit + 1); check_arrays(300); check_objects(1); - check_objects(127); - check_objects(128); - check_objects(129); + check_objects(limit > 0 ? limit - 1 : 0); + check_objects(limit); + check_objects(limit + 1); check_objects(300); } @@ -186,10 +189,13 @@ TEST_CASE("Better diagnostics with positions") } }; + // around detail::recursion_depth_limit() (127, 128, and 129 by default) + const std::size_t limit = nlohmann::detail::recursion_depth_limit(); + check_conversion(1); - check_conversion(127); - check_conversion(128); - check_conversion(129); + check_conversion(limit > 0 ? limit - 1 : 0); + check_conversion(limit); + check_conversion(limit + 1); check_conversion(300); } diff --git a/tests/src/unit-diagnostics.cpp b/tests/src/unit-diagnostics.cpp index 2987d348c..801b041cd 100644 --- a/tests/src/unit-diagnostics.cpp +++ b/tests/src/unit-diagnostics.cpp @@ -515,9 +515,11 @@ TEST_CASE("Regression tests for extended diagnostics") // update(j, true) around its descent bound, where the nested vectors // grow while the objects are merged without recursing + const std::size_t limit = nlohmann::detail::recursion_depth_limit(); + // (the limit can be 0, where "one level less" is no level) for (const std::size_t depth : { - nlohmann::detail::recursion_depth_limit() - 1, nlohmann::detail::recursion_depth_limit(), nlohmann::detail::recursion_depth_limit() + 2 + limit > 0 ? limit - 1 : 0, limit, limit + 2 }) { ordered_json j = {{"z", {{"x", 1}}}}; diff --git a/tests/src/unit-json_patch.cpp b/tests/src/unit-json_patch.cpp index 37c6e29d9..1edf5051a 100644 --- a/tests/src/unit-json_patch.cpp +++ b/tests/src/unit-json_patch.cpp @@ -1975,14 +1975,15 @@ TEST_CASE("JSON patch: diff of deeply nested values") SECTION("the diff reproduces the target at every depth") { // depths on either side of the nesting depth up to which diff() - // recurses (detail::recursion_depth_limit(), 128); not every depth up + // recurses (detail::recursion_depth_limit(), 128 by default); not every depth up // to 300, as the test would then time out under Valgrind std::vector depths; for (std::size_t depth = 0; depth <= 16; ++depth) { depths.push_back(depth); } - for (std::size_t depth = 120; depth <= 136; ++depth) + const std::size_t limit = nlohmann::detail::recursion_depth_limit(); + for (std::size_t depth = limit > 8 ? limit - 8 : 0; depth <= limit + 8; ++depth) { depths.push_back(depth); } @@ -2524,7 +2525,7 @@ TEST_CASE("JSON patch - diff of leaf values below the bound of the recursive des // the values at the innermost level are diffed without recursion: equal // leaves yield nothing, leaves of different types and different leaves // are replaced - const std::size_t depth = 200; + const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 72; // 200 by default json source = {1, "text", 3, {{"x", 1}, {"y", 2}}, true, nullptr}; json target = {1, 2, 4, {{"x", 1}, {"y", 3}}, true, nullptr}; diff --git a/tests/src/unit-large_json_binary_writers.cpp b/tests/src/unit-large_json_binary_writers.cpp index 339c62135..8c8755e81 100644 --- a/tests/src/unit-large_json_binary_writers.cpp +++ b/tests/src/unit-large_json_binary_writers.cpp @@ -50,13 +50,18 @@ json nested_object(const std::size_t depth, json leaf) TEST_CASE("issue #5392 - binary writers on deeply nested values") { - // 200 is past the point where the writers stop recursing, and still - // shallow enough that from_* and operator== (which still recurse) are fine. - const json deep_array = nested_array(200, json(0)); - const json deep_object = nested_object(200, json("x")); - const json empty_array = nested_array(200, json::array()); - const json empty_object = nested_object(200, json::object()); - const json mixed = nested_object(80, nested_array(80, json(true))); + // The limit (128 by default) of the levels that the writers recurse into. + const std::size_t limit = nlohmann::detail::recursion_depth_limit(); + + // 72 levels more than the limit (200 by default) is past the point where + // the writers stop recursing, and still shallow enough that from_* (which + // does not recurse) and operator== (which does, up to the limit) are quick. + const json deep_array = nested_array(limit + 72, json(0)); + const json deep_object = nested_object(limit + 72, json("x")); + const json empty_array = nested_array(limit + 72, json::array()); + const json empty_object = nested_object(limit + 72, json::object()); + // 2 * (limit / 2 + 16) levels: 160 by default, and more than the limit for every limit + const json mixed = nested_object((limit / 2) + 16, nested_array((limit / 2) + 16, json(true))); SECTION("roundtrip past the recursion bound") { @@ -95,7 +100,7 @@ TEST_CASE("issue #5392 - binary writers on deeply nested values") SECTION("the two ways of writing a value meet at the bound") { - for (std::size_t depth = 120; depth <= 140; ++depth) + for (std::size_t depth = limit > 8 ? limit - 8 : 0; depth <= limit + 12; ++depth) { CAPTURE(depth) @@ -118,8 +123,8 @@ TEST_CASE("issue #5392 - binary writers on deeply nested values") const json ndarray = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2, 3}}, {"_ArrayData_", {1, 2, 3, 4, 5, 6}}}); const json invalid = json({{"_ArrayType_", "nope"}, {"_ArraySize_", {1}}, {"_ArrayData_", {1}}}); - const json deep_ndarray = nested_array(140, ndarray); - const json deep_invalid = nested_array(140, invalid); + const json deep_ndarray = nested_array(limit + 12, ndarray); + const json deep_invalid = nested_array(limit + 12, invalid); CHECK(json::from_bjdata(json::to_bjdata(deep_ndarray)) == deep_ndarray); CHECK(json::from_bjdata(json::to_bjdata(deep_invalid)) == deep_invalid); @@ -131,10 +136,10 @@ TEST_CASE("issue #5392 - binary writers on deeply nested values") // nested one-element arrays around the recursion bound: the exact // bytes a writer produces do not depend on whether it stayed on the // call stack or moved to the heap one partway through + // (the limit can be 0, where "one level less" is no level) for (const std::size_t depth : { - nlohmann::detail::recursion_depth_limit() - 1, nlohmann::detail::recursion_depth_limit(), - nlohmann::detail::recursion_depth_limit() + 1, nlohmann::detail::recursion_depth_limit() + 2 + limit > 0 ? limit - 1 : 0, limit, limit + 1, limit + 2 }) { CAPTURE(depth) diff --git a/tests/src/unit-large_json_conversions.cpp b/tests/src/unit-large_json_conversions.cpp index 1fd30aa89..ba584df62 100644 --- a/tests/src/unit-large_json_conversions.cpp +++ b/tests/src/unit-large_json_conversions.cpp @@ -115,7 +115,7 @@ TEST_CASE("issue #5650 - stack overflow converting between specializations") // members that the two object types enumerate in different orders. const auto bury = [](nlohmann::ordered_json value) { - for (std::size_t i = 0; i < 200; ++i) + for (std::size_t i = 0; i < nlohmann::detail::recursion_depth_limit() + 72; ++i) { value = nlohmann::ordered_json::array({std::move(value)}); } @@ -124,7 +124,7 @@ TEST_CASE("issue #5650 - stack overflow converting between specializations") const auto dig = [](const json & value) { const json* current = &value; - for (std::size_t i = 0; i < 200; ++i) + for (std::size_t i = 0; i < nlohmann::detail::recursion_depth_limit() + 72; ++i) { current = ¤t->at(0); } diff --git a/tests/src/unit-merge_diff.cpp b/tests/src/unit-merge_diff.cpp index 77fa58768..6b3e67278 100644 --- a/tests/src/unit-merge_diff.cpp +++ b/tests/src/unit-merge_diff.cpp @@ -930,14 +930,15 @@ TEST_CASE("JSON Merge Patch diff of deeply nested values") SECTION("the patch is the same past the descent bound") { // depths on either side of the nesting depth up to which merge_diff() - // recurses (detail::recursion_depth_limit(), 128); not every depth up + // recurses (detail::recursion_depth_limit(), 128 by default); not every depth up // to 300, to keep the test fast under Valgrind std::vector depths; for (std::size_t depth = 0; depth <= 16; ++depth) { depths.push_back(depth); } - for (std::size_t depth = 120; depth <= 136; ++depth) + const std::size_t limit = nlohmann::detail::recursion_depth_limit(); + for (std::size_t depth = limit > 8 ? limit - 8 : 0; depth <= limit + 8; ++depth) { depths.push_back(depth); } diff --git a/tests/src/unit-merge_patch.cpp b/tests/src/unit-merge_patch.cpp index 3f5d79b3c..bb48cf070 100644 --- a/tests/src/unit-merge_patch.cpp +++ b/tests/src/unit-merge_patch.cpp @@ -302,7 +302,7 @@ TEST_CASE("JSON Merge Patch on deeply nested values") SECTION("patching past the descent bound gives the same result") { // every depth on either side of where the iterative version takes - // over (detail::recursion_depth_limit(), 128) + // over (detail::recursion_depth_limit(), 128 by default) for (std::size_t depth = 0; depth <= 300; ++depth) { CAPTURE(depth) @@ -396,7 +396,7 @@ TEST_CASE("merge_patch() with an argument that aliases *this (#5641)") SECTION("a patch nested past the iterative descent bound aliases *this") { // every depth on either side of where the iterative version takes - // over (detail::recursion_depth_limit(), 128); patching *this with + // over (detail::recursion_depth_limit(), 128 by default); patching *this with // itself is idempotent, aliased or not for (const std::size_t depth : { diff --git a/tests/src/unit-modifiers.cpp b/tests/src/unit-modifiers.cpp index 3377eace0..0963b9881 100644 --- a/tests/src/unit-modifiers.cpp +++ b/tests/src/unit-modifiers.cpp @@ -1124,7 +1124,7 @@ TEST_CASE("update() on deeply nested values") SECTION("merging past the descent bound gives the same result") { // every depth on either side of where the iterative version takes - // over (detail::recursion_depth_limit(), 128) + // over (detail::recursion_depth_limit(), 128 by default) for (std::size_t depth = 0; depth <= 300; ++depth) { CAPTURE(depth) @@ -1215,7 +1215,7 @@ TEST_CASE("update() with an argument that aliases *this (#5641)") SECTION("nested past the iterative descent bound aliases *this") { // every depth on either side of where the iterative version takes - // over (detail::recursion_depth_limit(), 128); merging *this into + // over (detail::recursion_depth_limit(), 128 by default); merging *this into // itself is idempotent, aliased or not for (const std::size_t depth : { diff --git a/tests/src/unit-recursion_depth_limit.cpp b/tests/src/unit-recursion_depth_limit.cpp new file mode 100644 index 000000000..924683383 --- /dev/null +++ b/tests/src/unit-recursion_depth_limit.cpp @@ -0,0 +1,600 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// JSON_RECURSION_DEPTH_LIMIT changes only how much call stack the walks over a +// value use, never what they return. This file checks both: the macro is +// honoured (the number of levels the containers handle themselves is the limit) +// and every walk returns what it returns with the default limit. The expected +// results are built here, without the library, so that the same file passes for +// every limit; tests/CMakeLists.txt builds it with limits 0, 1, and 254 besides +// the default. Values are nested up to 300 levels (and past every limit, so that +// the explicit stack of every walk is exercised, even for the limit of 254) and +// are built and taken apart without recursion. Keep the depths moderate: walks +// like diff and dump with indentation are quadratic in the depth. + +namespace +{ +constexpr std::size_t limit() +{ + return nlohmann::detail::recursion_depth_limit(); +} + +// a value nested as @a kinds says, outermost level first: 'a' is an array of +// one element, 'o' an object with the single key "a"; @a leaf is innermost +json build(const std::string& kinds, json leaf) +{ + json j = std::move(leaf); + for (std::size_t i = kinds.size(); i-- > 0;) + { + if (kinds[i] == 'a') + { + json a = json::array(); + a.push_back(std::move(j)); + j = std::move(a); + } + else + { + json o = json::object(); + o["a"] = std::move(j); + j = std::move(o); + } + } + return j; +} + +std::string repeat(const std::string& s, std::size_t n) +{ + std::string result; + for (std::size_t i = 0; i < n; ++i) + { + result += s; + } + return result; +} + +// the nesting of the given shape and depth: arrays, objects, or both alternating +std::string make_kinds(char shape, std::size_t depth) +{ + std::string kinds; + for (std::size_t i = 0; i < depth; ++i) + { + if (shape == 'm') + { + kinds += i % 2 == 0 ? 'a' : 'o'; + } + else + { + kinds += shape; + } + } + return kinds; +} + +std::string expected_compact(const std::string& kinds, const std::string& leaf) +{ + std::string s; + for (const char kind : kinds) + { + s += kind == 'a' ? "[" : "{\"a\":"; + } + s += leaf; + for (std::size_t i = kinds.size(); i-- > 0;) + { + s += kinds[i] == 'a' ? "]" : "}"; + } + return s; +} + +// dump(1) +std::string expected_indented(const std::string& kinds, const std::string& leaf) +{ + std::string s; + for (std::size_t i = 0; i < kinds.size(); ++i) + { + s += kinds[i] == 'a' ? "[\n" : "{\n"; + s += std::string(i + 1, ' '); + if (kinds[i] == 'o') + { + s += "\"a\": "; + } + } + s += leaf; + for (std::size_t i = kinds.size(); i-- > 0;) + { + s += "\n" + std::string(i, ' ') + (kinds[i] == 'a' ? "]" : "}"); + } + return s; +} + +// the JSON pointer to the innermost value +std::string pointer(const std::string& kinds) +{ + std::string s; + for (const char kind : kinds) + { + s += kind == 'a' ? "/0" : "/a"; + } + return s; +} + +// the binary formats: the bytes that open and close a level, and the leaf 1 +std::vector expected_cbor(const std::string& kinds) +{ + std::vector bytes; + for (const char kind : kinds) + { + if (kind == 'a') + { + bytes.push_back(0x81); + } + else + { + bytes.push_back(0xA1); + bytes.push_back(0x61); + bytes.push_back('a'); + } + } + bytes.push_back(0x01); + return bytes; +} + +std::vector expected_msgpack(const std::string& kinds) +{ + std::vector bytes; + for (const char kind : kinds) + { + if (kind == 'a') + { + bytes.push_back(0x91); + } + else + { + bytes.push_back(0x81); + bytes.push_back(0xA1); + bytes.push_back('a'); + } + } + bytes.push_back(0x01); + return bytes; +} + +std::vector expected_ubjson(const std::string& kinds) +{ + std::vector bytes; + for (const char kind : kinds) + { + if (kind == 'a') + { + bytes.push_back('['); + } + else + { + bytes.push_back('{'); + bytes.push_back('i'); + bytes.push_back(1); + bytes.push_back('a'); + } + } + bytes.push_back('i'); + bytes.push_back(1); + for (std::size_t i = kinds.size(); i-- > 0;) + { + bytes.push_back(kinds[i] == 'a' ? ']' : '}'); + } + return bytes; +} + +// the depths worth trying for the limit: either side of it, twice it (if that +// is not too deep to be quick), and a value nested well past all limits +std::vector depths() +{ + std::vector result; + result.push_back(1); + result.push_back(2); + result.push_back(limit() > 0 ? limit() - 1 : 3); + result.push_back(limit()); + result.push_back(limit() + 1); + if ((2 * limit()) + 5 <= 300) + { + result.push_back((2 * limit()) + 5); + } + result.push_back(300); + std::sort(result.begin(), result.end()); + result.erase(std::unique(result.begin(), result.end()), result.end()); + result.erase(std::remove(result.begin(), result.end(), static_cast(0)), result.end()); + return result; +} + +// the call for every shape and depth +void for_each_value(const std::function& check) +{ + const std::string shapes = "aom"; + for (const char shape : shapes) + { + for (const std::size_t depth : depths()) + { + CAPTURE(shape) + CAPTURE(depth) + check(make_kinds(shape, depth)); + } + } +} + +// An array type that counts how many of its copies are being made at once. A +// copy of a value copies one level at a time through the array type until the +// limit has been reached, so the peak is the number of levels that were copied +// with the call stack. +std::size_t copies_going_on = 0; +std::size_t copies_peak = 0; +std::size_t equal_going_on = 0; +std::size_t equal_peak = 0; + +template> +class counting_vector : public std::vector +{ + public: + counting_vector() = default; + + template + counting_vector(InputIt first, InputIt last) : std::vector(first, last) {} + + counting_vector(const counting_vector& other) : std::vector() + { + ++copies_going_on; + copies_peak = (std::max)(copies_peak, copies_going_on); + this->std::vector::operator=(static_cast&>(other)); + --copies_going_on; + } + + counting_vector(counting_vector&&) = default; + counting_vector& operator=(const counting_vector&) = default; + counting_vector& operator=(counting_vector&&) = default; + ~counting_vector() = default; + + friend bool operator==(const counting_vector& lhs, const counting_vector& rhs) + { + ++equal_going_on; + equal_peak = (std::max)(equal_peak, equal_going_on); + const bool result = static_cast&>(lhs) == static_cast&>(rhs); + --equal_going_on; + return result; + } +}; + +using counting_json = json::with_array_t; + +// A base class whose copy constructor copies a whole value of the same +// specialization when it copies a marked element, as user code may do from a +// hook. That copy runs while the copy that it is part of is going on. +struct copy_hook // NOLINT(cppcoreguidelines-special-member-functions,hicpp-special-member-functions) +{ + int mark = 0; + + copy_hook() = default; + copy_hook(const copy_hook& other); + copy_hook(copy_hook&&) = default; + copy_hook& operator=(const copy_hook&) = default; + copy_hook& operator=(copy_hook&&) = default; +}; + +using hook_json = nlohmann::basic_json, copy_hook>; + +const hook_json* hook_copy_source = nullptr; +int hook_copies_going_on = 0; + +copy_hook::copy_hook(const copy_hook& other) : mark(other.mark) +{ + // the value that is copied is marked, too: its copy copies it again, once + if (other.mark == 1 && hook_copy_source != nullptr && hook_copies_going_on < 2) + { + ++hook_copies_going_on; + const hook_json nested = *hook_copy_source; // NOLINT(performance-unnecessary-copy-initialization) + static_cast(nested); + --hook_copies_going_on; + } +} + +// an array nested @a depth levels around the integer 1, which is marked if @a marked +hook_json hook_nested(std::size_t depth, bool marked) +{ + hook_json j = 1; + if (marked) + { + static_cast(j).mark = 1; + } + for (std::size_t i = 0; i < depth; ++i) + { + hook_json a = hook_json::array(); + a.push_back(std::move(j)); + j = std::move(a); + } + return j; +} +} // namespace + +// the macro is the limit, and is in the range the library supports +static_assert(nlohmann::detail::recursion_depth_limit() == static_cast(JSON_RECURSION_DEPTH_LIMIT), + "the library does not use JSON_RECURSION_DEPTH_LIMIT"); +static_assert(nlohmann::detail::recursion_depth_limit() <= 254, "the nesting depth must fit in a byte"); + +TEST_CASE("JSON_RECURSION_DEPTH_LIMIT is honoured") +{ + SECTION("copying and comparing stop descending at the limit") + { + // copy and compare a value nested deeper than the limit, with the array type that counts + counting_json inner = 1; + for (std::size_t i = 0; i < (2 * limit()) + 5; ++i) + { + counting_json a = counting_json::array(); + a.push_back(std::move(inner)); + inner = std::move(a); + } + + copies_going_on = copies_peak = 0; + const counting_json copy = inner; + CHECK(copies_going_on == 0); + + equal_going_on = equal_peak = 0; + CHECK((inner == copy)); + CHECK(equal_going_on == 0); + +#ifdef JSON_NO_THREAD_LOCAL + // without the counters of the levels, no level is descended into + CHECK(copies_peak == 0); + CHECK(equal_peak == 0); +#else + // copying descends into as many levels as the limit says, and nothing further + CHECK(copies_peak == limit()); + CHECK(equal_peak == limit()); +#endif + + // a value that is not nested deeper than the limit is not cut short + const std::size_t shallow = limit() > 0 ? limit() : 1; + counting_json small = 1; + for (std::size_t i = 0; i < shallow; ++i) + { + counting_json a = counting_json::array(); + a.push_back(std::move(small)); + small = std::move(a); + } + copies_going_on = copies_peak = 0; + const counting_json small_copy = small; + CHECK(copies_going_on == 0); +#ifndef JSON_NO_THREAD_LOCAL + CHECK(copies_peak == limit()); +#endif + CHECK(small_copy.dump() == expected_compact(make_kinds('a', shallow), "1")); + } +} + +TEST_CASE("a copy made while a copy is going on is bounded, too") +{ + // The innermost element of the value is marked, and copying it (which happens + // after limit() levels have been entered, when the copy continues without the + // call stack) copies the value again, and that copy once more. Those copies + // must not be given limit() levels of their own: the count of the levels entered + // never exceeds the limit, not even by one, which for the largest limit would + // wrap the byte that holds it around to 0. + const hook_json marked = hook_nested((2 * limit()) + 5, true); + hook_copy_source = &marked; + + copies_going_on = copies_peak = 0; + const hook_json copy = marked; // NOLINT(performance-unnecessary-copy-initialization) + hook_copy_source = nullptr; + CHECK(copies_going_on == 0); + CHECK(static_cast(copy).mark == 0); // (the top level is not marked) + +#ifdef JSON_NO_THREAD_LOCAL + CHECK(copies_peak == 0); +#else + CHECK(copies_peak == limit()); +#endif +} + +TEST_CASE("walks over values nested around the recursion depth limit") +{ + SECTION("copying and comparing") + { + for_each_value([](const std::string & kinds) + { + const json a = build(kinds, 1); + const json b = a; // NOLINT(performance-unnecessary-copy-initialization) + CHECK((a == b)); + CHECK_FALSE((a != b)); + CHECK_FALSE((a < b)); + CHECK((a <= b)); + CHECK((a >= b)); + CHECK_FALSE((a > b)); + + json assigned; + assigned = a; + CHECK((assigned == a)); + + // differing only in the innermost value + const json c = build(kinds, 2); + CHECK((a != c)); + CHECK((a < c)); + CHECK((c > a)); + CHECK_FALSE((c < a)); + CHECK_FALSE((a == c)); + + // a same-shape array or object that is one level shallower + const json shallower = build(kinds.substr(1), 1); + CHECK((shallower != a)); + const json outer = build(kinds.substr(0, 1), 1); + CHECK((outer != a) == (kinds.size() > 1)); + + const json moved = std::move(assigned); + CHECK((moved == a)); + }); + } + + SECTION("serializing") + { + for_each_value([](const std::string & kinds) + { + const json a = build(kinds, 1); + const std::string compact = expected_compact(kinds, "1"); + const std::string indented = expected_indented(kinds, "1"); + + CHECK(a.dump() == compact); + CHECK(a.dump(1) == indented); + + std::ostringstream compact_stream; + compact_stream << a; + CHECK(compact_stream.str() == compact); + std::ostringstream indented_stream; + indented_stream << std::setw(1) << a; + CHECK(indented_stream.str() == indented); + + CHECK((json::parse(compact) == a)); + CHECK((json::parse(indented) == a)); + }); + } + + SECTION("hashing") + { + for_each_value([](const std::string & kinds) + { + const json a = build(kinds, 1); + const std::hash hash; + CHECK(hash(a) == hash(json(a))); + CHECK(hash(a) == hash(json::parse(expected_compact(kinds, "1")))); + CHECK(hash(a) != hash(build(kinds, 2))); + CHECK(hash(a) != hash(build(kinds.substr(1), 1))); + }); + } + + SECTION("diff and patch") + { + for_each_value([](const std::string & kinds) + { + const json a = build(kinds, 1); + const json b = build(kinds, 2); + + const json difference = json::diff(a, b); + CHECK(difference.dump() == "[{\"op\":\"replace\",\"path\":\"" + pointer(kinds) + "\",\"value\":2}]"); + CHECK((a.patch(difference) == b)); + CHECK((json::diff(a, a) == json::array())); + }); + } + + SECTION("flatten and unflatten") + { + for_each_value([](const std::string & kinds) + { + const json a = build(kinds, 1); + const json flat = a.flatten(); + CHECK(flat.dump() == "{\"" + pointer(kinds) + "\":1}"); + CHECK((flat.unflatten() == a)); + }); + } + + SECTION("merge_patch and update") + { + for (const std::size_t depth : depths()) + { + CAPTURE(depth) + const std::string kinds = make_kinds('o', depth); + const std::string all_but_leaf = repeat("{\"a\":", depth - 1); + + // the patch removes the innermost member, or replaces it + json target = build(kinds, 1); + target.merge_patch(build(kinds, json(nullptr))); + CHECK(target.dump() == all_but_leaf + "{}" + repeat("}", depth - 1)); + + target = build(kinds, 1); + target.merge_patch(build(kinds, 2)); + CHECK(target.dump() == expected_compact(kinds, "2")); + + // merging the objects of two values nested alike, and replacing the first level + const json base = build(kinds.substr(1), json::parse("{\"x\":1}")); + const json other = build(kinds.substr(1), json::parse("{\"y\":2}")); + json merged = json{{"a", base}}; + merged.update(json{{"a", other}}, true); + CHECK(merged.dump() == expected_compact(kinds, "{\"x\":1,\"y\":2}")); + + json replaced = json{{"a", base}}; + replaced.update(json{{"a", other}}); + CHECK(replaced.dump() == expected_compact(kinds, "{\"y\":2}")); + + // update from a range + json from_range = json{{"a", base}}; + const json source = json{{"a", other}}; + from_range.update(source.begin(), source.end(), true); + CHECK((from_range == merged)); + } + } + + SECTION("binary formats") + { + for_each_value([](const std::string & kinds) + { + const json a = build(kinds, 1); + + const std::vector cbor = json::to_cbor(a); + CHECK(cbor == expected_cbor(kinds)); + CHECK((json::from_cbor(cbor) == a)); + + const std::vector msgpack = json::to_msgpack(a); + CHECK(msgpack == expected_msgpack(kinds)); + CHECK((json::from_msgpack(msgpack) == a)); + + const std::vector ubjson = json::to_ubjson(a); + CHECK(ubjson == expected_ubjson(kinds)); + CHECK((json::from_ubjson(ubjson) == a)); + + CHECK((json::from_bjdata(json::to_bjdata(a)) == a)); + CHECK((json::from_bon8(json::to_bon8(a)) == a)); + + // BSON documents are objects + const json document = json{{"a", a}}; + CHECK((json::from_bson(json::to_bson(document)) == document)); + }); + } + + SECTION("conversion to another specialization") + { + for_each_value([](const std::string & kinds) + { + const json a = build(kinds, 1); + const nlohmann::ordered_json ordered = a; + CHECK(ordered.dump() == expected_compact(kinds, "1")); + const json back = ordered; + CHECK((back == a)); + }); + } + + SECTION("destroying") + { + for_each_value([](const std::string & kinds) + { + json a = build(kinds, 1); + a = nullptr; + CHECK(a.is_null()); + }); + } +} diff --git a/tests/src/unit-regression1.cpp b/tests/src/unit-regression1.cpp index 367f52dfa..fcb533c5d 100644 --- a/tests/src/unit-regression1.cpp +++ b/tests/src/unit-regression1.cpp @@ -1554,8 +1554,10 @@ TEST_CASE("regression tests 1") SECTION("a missing member of an object") { my_json j = make(); -#ifdef JSON_NO_THREAD_LOCAL - // (set by the user, or by the library for clang with MinGW) +#if defined(JSON_NO_THREAD_LOCAL) || JSON_RECURSION_DEPTH_LIMIT < 2 + // (set by the user, or by the library for clang with MinGW; or fewer than two + // levels are copied by the object type itself, as for a recursion depth limit + // of 0 or 1: the objects below the outermost one are the second level) // the objects below the outermost one are not copied by their // object_t here, but member by member, which keeps their // timestamps correct: "c" is not mistaken for "b" and is inserted