mirror of
https://github.com/nlohmann/json.git
synced 2026-11-11 18:08:22 +00:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b38131ec40 |
No files matched your search
@@ -153,7 +153,7 @@ jobs:
|
||||
container: ubuntu:24.04
|
||||
strategy:
|
||||
matrix:
|
||||
target: [ci_cmake_flags, ci_test_diagnostics, ci_test_diagnostic_positions, ci_test_noexceptions, ci_test_noimplicitconversions, ci_test_legacycomparison, ci_test_noglobaludls, ci_test_disableenumserialization, ci_test_disabletuplereferenceconversion, ci_test_skiplibraryversioncheck, ci_test_simdutf, ci_test_strict_nul_handling, ci_test_delete_deprecated_functions, ci_test_no_thread_local]
|
||||
target: [ci_cmake_flags, ci_test_diagnostics, ci_test_diagnostic_positions, ci_test_noexceptions, ci_test_noimplicitconversions, ci_test_legacycomparison, ci_test_noglobaludls, ci_test_disableenumserialization, ci_test_disabletuplereferenceconversion, ci_test_skiplibraryversioncheck, ci_test_simdutf, ci_test_strict_nul_handling, ci_test_delete_deprecated_functions, ci_test_no_thread_local, ci_test_recursion_depth_limit_0]
|
||||
steps:
|
||||
- name: Install build-essential
|
||||
run: apt-get update ; apt-get install -y build-essential unzip wget git
|
||||
|
||||
@@ -342,6 +342,27 @@ add_custom_target(ci_test_no_thread_local
|
||||
COMMENT "Compile and test without thread-local storage"
|
||||
)
|
||||
|
||||
###############################################################################
|
||||
# Change the number of levels the walks over a value recurse into.
|
||||
###############################################################################
|
||||
|
||||
# JSON_RECURSION_DEPTH_LIMIT is 128 by default, which is what the whole test
|
||||
# suite otherwise runs with. 0 sends every walk to its explicit stack at once,
|
||||
# so this target runs the whole suite through the walks that the default only
|
||||
# uses for values nested more deeply than 128 levels: results must not depend on
|
||||
# the limit. The largest limit, 254, is covered by a build of
|
||||
# tests/src/unit-recursion_depth_limit.cpp (see tests/CMakeLists.txt).
|
||||
add_custom_target(ci_test_recursion_depth_limit_0
|
||||
COMMAND ${CMAKE_COMMAND}
|
||||
-DCMAKE_BUILD_TYPE=Debug -GNinja
|
||||
-DJSON_BuildTests=ON -DJSON_FastTests=ON
|
||||
-DCMAKE_CXX_FLAGS=-DJSON_RECURSION_DEPTH_LIMIT=0
|
||||
-S${PROJECT_SOURCE_DIR} -B${PROJECT_BINARY_DIR}/build_recursion_depth_limit_0
|
||||
COMMAND ${CMAKE_COMMAND} --build ${PROJECT_BINARY_DIR}/build_recursion_depth_limit_0
|
||||
COMMAND cd ${PROJECT_BINARY_DIR}/build_recursion_depth_limit_0 && ${CMAKE_CTEST_COMMAND} --parallel ${N} --output-on-failure
|
||||
COMMENT "Compile and test with JSON_RECURSION_DEPTH_LIMIT=0"
|
||||
)
|
||||
|
||||
###############################################################################
|
||||
# Coverage.
|
||||
###############################################################################
|
||||
|
||||
@@ -89,11 +89,8 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va
|
||||
- Throws [out_of_range.406](../../home/exceptions.md#jsonexceptionout_of_range406) if a high-precision number, or a
|
||||
float64 when `number_float_t` is `float`, does not fit into `number_float_t`
|
||||
- Throws [out_of_range.408](../../home/exceptions.md#jsonexceptionout_of_range408) if the count of an optimized
|
||||
container exceeds the capacity of the container (or of `std::size_t`; an array of type `N` is empty, so only the
|
||||
latter applies to it), if an optimized array of `Z`, `T`, or `F` has more than 1,048,576 elements, or if the
|
||||
optimized arrays of `Z`, `T`, and `F` in the input together have more than 1,048,576 elements plus
|
||||
[`JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE`](../macros/json_ubjson_valueless_elements_per_byte.md) (4 by default) for
|
||||
every byte read so far
|
||||
container exceeds the capacity of the container (or of `std::size_t`), or if an optimized array of `Z`, `T`, or `F` has
|
||||
more than 1,048,576 elements
|
||||
|
||||
## Complexity
|
||||
|
||||
@@ -122,8 +119,6 @@ Linear in the size of the input.
|
||||
- [Parsing untrusted input](../../features/parsing/untrusted_input.md) - limits and safe handling of hostile data
|
||||
- [Binary values](../../features/binary_values.md) - how binary values and subtypes are handled
|
||||
- [`to_ubjson`](to_ubjson.md) - create a UBJSON serialization of a JSON value
|
||||
- [`JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE`](../macros/json_ubjson_valueless_elements_per_byte.md) -
|
||||
set how many elements of arrays of `null`, `true`, or `false` the input may announce per byte
|
||||
- [`from_cbor`](from_cbor.md) - create a JSON value from an input in CBOR format
|
||||
- [`from_msgpack`](from_msgpack.md) - create a JSON value from an input in MessagePack format
|
||||
- [`from_bson`](from_bson.md) - create a JSON value from an input in BSON format
|
||||
@@ -137,12 +132,6 @@ Linear in the size of the input.
|
||||
- Extended container support (1) to include types with lvalue-only ADL `begin`/`end` (matching `std::begin`/`std::end` semantics) in version 3.13.0.
|
||||
- Extended overload (2) to accept heterogeneous iterator+sentinel pairs (C++20 ranges support) in version 3.13.0.
|
||||
- Added `error_handler` parameter in version 3.13.0.
|
||||
- Report `start_array(0)` to a SAX handler for an optimized array of type `N` (`[$N#n`), whatever its count, in version
|
||||
3.13.0.
|
||||
- Limited the total number of elements of optimized arrays of `Z`, `T`, and `F` in one input in version 3.13.0; the
|
||||
number of elements per byte can be set with
|
||||
[`JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE`](../macros/json_ubjson_valueless_elements_per_byte.md), for example to read
|
||||
files written by version 3.12.0 and earlier that exceed the default.
|
||||
|
||||
!!! warning "Deprecation"
|
||||
|
||||
|
||||
@@ -105,8 +105,6 @@ Linear in the size of the JSON value `j`.
|
||||
- [Binary formats](../../features/binary_formats/index.md) - overview and comparison of the supported binary formats
|
||||
- [Binary values](../../features/binary_values.md) - how binary values and subtypes are handled
|
||||
- [`from_ubjson`](from_ubjson.md) - create a JSON value from an input in UBJSON format
|
||||
- [`JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE`](../macros/json_ubjson_valueless_elements_per_byte.md) -
|
||||
set how many elements of arrays of `null`, `true`, or `false` a reader accepts per byte of input
|
||||
- [`to_cbor`](to_cbor.md) - create a CBOR serialization of a JSON value
|
||||
- [`to_msgpack`](to_msgpack.md) - create a MessagePack serialization of a JSON value
|
||||
- [`to_bson`](to_bson.md) - create a BSON serialization of a JSON value
|
||||
@@ -119,10 +117,5 @@ Linear in the size of the JSON value `j`.
|
||||
- Added `error_handler` parameter in version 3.13.0. Its default, `keep`, writes the bytes of a string or object key
|
||||
that is not valid UTF-8 unchanged, as before; `strict` (the default if
|
||||
[`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled) throws `type_error.316`.
|
||||
- Writes arrays of `Z`, `T`, or `F` without the `$` type annotation (`[#n Z Z ...`) once their elements in one call
|
||||
add up to more than 1,048,576, in version 3.13.0, so that `from_ubjson` can read the output back. Such output is
|
||||
larger: 270,000 rows of 100 `null` values take 26.8 MB instead of 1.35 MB. The limit does not depend on
|
||||
[`JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE`](../macros/json_ubjson_valueless_elements_per_byte.md): a reader accepts at
|
||||
least 1,048,576 elements whatever it is set to, so the writer follows that base only.
|
||||
- Throws `type_error.321` for a discarded value since version 3.13.0; previously, a discarded value nested in an
|
||||
array or object was silently skipped, producing invalid UBJSON.
|
||||
@@ -26,8 +26,6 @@ header. See also the [macro overview page](../../features/macros.md).
|
||||
UBJSON, BJData, and BSON writers
|
||||
- [**JSON_STRICT_NUL_HANDLING**](json_strict_nul_handling.md) - opt in to rejecting a NUL byte in the input instead of
|
||||
treating it as end of input
|
||||
- [**JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE**](json_ubjson_valueless_elements_per_byte.md) - set how many elements of
|
||||
UBJSON arrays of `null`, `true`, or `false` an input may announce per byte it has
|
||||
|
||||
## Language support
|
||||
|
||||
@@ -42,6 +40,8 @@ header. See also the [macro overview page](../../features/macros.md).
|
||||
- [**JSON_NO_IO**](json_no_io.md) - switch off functions relying on certain C++ I/O headers
|
||||
- [**JSON_NO_SIMD**](json_no_simd.md) - use only portable code instead of vector instructions (SSE2, SSSE3, NEON)
|
||||
- [**JSON_NO_THREAD_LOCAL**](json_no_thread_local.md) - switch off the use of `thread_local` storage
|
||||
- [**JSON_RECURSION_DEPTH_LIMIT**](json_recursion_depth_limit.md) - set how many nesting levels the operations on a value
|
||||
recurse into on the call stack (default 128)
|
||||
- [**JSON_SKIP_UNSUPPORTED_COMPILER_CHECK**](json_skip_unsupported_compiler_check.md) - do not warn about unsupported compilers
|
||||
- [**JSON_USE_GLOBAL_UDLS**](json_use_global_udls.md) - place user-defined string literals (UDLs) into the global namespace
|
||||
- [**JSON_USE_SIMDUTF**](json_use_simdutf.md) - use the simdutf library to accelerate UTF-8 validation
|
||||
|
||||
@@ -45,6 +45,7 @@ Copying and comparing fall back to working without the call stack there, as they
|
||||
|
||||
## See also
|
||||
|
||||
- [`JSON_RECURSION_DEPTH_LIMIT`](json_recursion_depth_limit.md) - set how many nesting levels the operations recurse into
|
||||
- [FAQ: Thread safety](../../home/faq.md#thread-safety) - thread-safety guarantees of the library
|
||||
- [Supported Macros](../../features/macros.md) - guide to all supported macros
|
||||
|
||||
|
||||
@@ -0,0 +1,307 @@
|
||||
# JSON_RECURSION_DEPTH_LIMIT
|
||||
|
||||
```cpp
|
||||
#define JSON_RECURSION_DEPTH_LIMIT /* value */
|
||||
```
|
||||
|
||||
This macro sets how many nesting levels the operations that walk a value recurse into on the call stack. Beyond that,
|
||||
they continue with an explicit stack on the heap. The value is an integer constant from `0` to `254`; the default is
|
||||
`128`.
|
||||
|
||||
The operations are copying a value, comparing two values, serializing with [`dump`](../basic_json/dump.md) or
|
||||
[`operator<<`](../operator_ltlt.md), hashing with `std::hash`, [`merge_patch`](../basic_json/merge_patch.md),
|
||||
[`update`](../basic_json/update.md) with `merge_objects`, [`diff`](../basic_json/diff.md),
|
||||
[`merge_diff`](../basic_json/merge_diff.md), converting a value to another specialization of `basic_json`, destroying a
|
||||
value, and the binary writers [`to_cbor`](../basic_json/to_cbor.md),
|
||||
[`to_msgpack`](../basic_json/to_msgpack.md), [`to_ubjson`](../basic_json/to_ubjson.md),
|
||||
[`to_bjdata`](../basic_json/to_bjdata.md), and [`to_bon8`](../basic_json/to_bon8.md). They share this one limit; none
|
||||
has
|
||||
a limit of its own. Recursing is the fastest way to walk a value, but a value nested deeply enough would exhaust the
|
||||
call
|
||||
stack, so the library recurses only as far as the limit and finishes the levels below with an explicit stack. This is
|
||||
what makes all of them safe for untrusted input
|
||||
(see [Nesting depth](../../features/parsing/untrusted_input.md#nesting-depth)). Parsing, the binary readers,
|
||||
[`flatten`](../basic_json/flatten.md), [`unflatten`](../basic_json/unflatten.md), and
|
||||
[`to_bson`](../basic_json/to_bson.md) always use an explicit stack and do not depend on the macro. The lockstep
|
||||
comparison of two [`json_view`](../basic_json_view/index.md) documents has a fixed bound of its own, 48 levels,
|
||||
and does not depend on it either.
|
||||
|
||||
The call stack that these operations need grows with the limit: roughly *a few kilobytes plus a cost per level times the
|
||||
limit*, where the cost per level is up to about 2.3 KiB in an unoptimized and up to about 0.5 KiB in an optimized build
|
||||
(see [Stack needed](#stack-needed)). The default of `128` levels is tuned for threads with ordinary stacks of a few
|
||||
hundred kilobytes or more. Set a lower limit if your threads have small stacks (RTOS tasks, fibers, threads of the musl
|
||||
C
|
||||
library, which has a default stack of 128 KiB), or a higher limit to trade stack for speed on deeply nested values.
|
||||
|
||||
The macro changes only how much stack is used and how fast the operations are. It never changes a result: every
|
||||
operation returns the same value for every setting of the limit, for values nested less deeply than the limit as well as
|
||||
for values nested more deeply.
|
||||
|
||||
## Default definition
|
||||
|
||||
The default value is `128`.
|
||||
|
||||
```cpp
|
||||
#define JSON_RECURSION_DEPTH_LIMIT 128
|
||||
```
|
||||
|
||||
When the macro is not defined, the library defines it to its default value.
|
||||
|
||||
## Notes
|
||||
|
||||
!!! note "Allowed values"
|
||||
|
||||
The value must be an integer constant expression from `0` to `254`; any other value (a negative number, a number
|
||||
above `254`, a floating-point number such as `254.0`, or `true` or `false`) is rejected at compile time with an
|
||||
error message that names the macro. (A character such as `'a'` is an integer, too, and is accepted as its code.)
|
||||
|
||||
- `0` is allowed, and means that every operation uses its explicit stack from the first level on and no operation
|
||||
recurses over the nesting of a value at all. That needs a constant amount of call stack (at most 16 KiB in the
|
||||
measurements below, for a value nested 20,000 levels deep) and is the setting for tiny stacks, at the price of
|
||||
speed (see [Speed](#speed)).
|
||||
- `254` is the largest allowed value. Copying a value and comparing two values (with `<=>`, or with the operators on
|
||||
objects and arrays of a custom type) cannot pass the depth down as an argument, because the signatures of the
|
||||
copy constructor and the operators are fixed. They count the levels they have descended into in a variable of type
|
||||
`#!cpp std::uint8_t` that exists once per thread (see [`JSON_NO_THREAD_LOCAL`](json_no_thread_local.md)). Only
|
||||
the levels that are descended into are counted, so the count never exceeds the limit, not even when user code (a
|
||||
copy constructor of a custom base class, say) copies a value while a copy is going on. A byte holds `255` at
|
||||
most, and the limit stays below that, so that the count cannot wrap around. (More levels would hardly be
|
||||
useful: `254` levels need up to 576 KiB of stack in an unoptimized build already, twice as much as the
|
||||
default.)
|
||||
|
||||
### Stack needed
|
||||
|
||||
The tables show the smallest stack that an operation needs, in KiB, at several limits, on a value that is nested
|
||||
more deeply than the limit (limit + 10 levels). Each cell shows the result for Clang (Apple clang 21) and for
|
||||
GCC 16, separated by a slash; both measured on macOS with arm64, compiling in C++20 mode. The operations run in a
|
||||
thread whose stack has the size given, and are not given any stack that they do not need: the value is built, and
|
||||
the result destroyed, outside of that thread. Stacks can only be set in steps of 16 KiB (the page size), so `16`
|
||||
means "16 KiB or less". A row stands for all operations listed; the table shows the largest value among them.
|
||||
|
||||
Unoptimized (`-O0`):
|
||||
|
||||
| Operation | limit 16 | limit 32 | limit 64 | limit 128 | limit 254 |
|
||||
|---|---:|---:|---:|---:|---:|
|
||||
| `dump()` and `dump(indent)`, `operator<<` | 48 / 32 | 80 / 64 | 144 / 128 | 288 / 256 | 576 / 512 |
|
||||
| copy construction and assignment | 32 / 16 | 48 / 32 | 112 / 80 | 224 / 144 | 432 / 304 |
|
||||
| `==` | 16 / 16 | 16 / 16 | 16 / 16 | 48 / 32 | 96 / 48 |
|
||||
| `<=>` (C++20) | 16 / 16 | 16 / 16 | 32 / 32 | 64 / 64 | 144 / 128 |
|
||||
| `std::hash` | 16 / 16 | 16 / 16 | 32 / 32 | 80 / 64 | 160 / 128 |
|
||||
| `merge_patch`, `update(j, true)`, `patch` | 32 / 16 | 48 / 32 | 112 / 80 | 224 / 144 | 432 / 304 |
|
||||
| `diff` | 16 / 16 | 16 / 16 | 32 / 32 | 64 / 48 | 144 / 96 |
|
||||
| `merge_diff` | 16 / 16 | 48 / 32 | 80 / 48 | 176 / 112 | 352 / 224 |
|
||||
| `to_cbor`, `to_msgpack` | 16 / 16 | 32 / 16 | 64 / 32 | 112 / 80 | 224 / 160 |
|
||||
| `to_ubjson`, `to_bjdata` | 16 / 16 | 16 / 16 | 48 / 32 | 96 / 64 | 176 / 112 |
|
||||
| `to_bon8` | 16 / 16 | 16 / 16 | 16 / 16 | 32 / 16 | 64 / 48 |
|
||||
| conversion to another `basic_json` type | 32 / 16 | 64 / 32 | 112 / 80 | 224 / 144 | 448 / 304 |
|
||||
| destruction | 16 / 16 | 16 / 16 | 16 / 16 | 16 / 16 | 32 / 32 |
|
||||
|
||||
Optimized (`-O2`):
|
||||
|
||||
| Operation | limit 16 | limit 32 | limit 64 | limit 128 | limit 254 |
|
||||
|---|---:|---:|---:|---:|---:|
|
||||
| `dump()` and `dump(indent)`, `operator<<` | 16 / 16 | 16 / 16 | 16 / 16 | 16 / 16 | 32 / 32 |
|
||||
| copy construction and assignment | 16 / 16 | 16 / 16 | 16 / 16 | 32 / 32 | 64 / 48 |
|
||||
| `==` | 16 / 16 | 16 / 16 | 16 / 16 | 16 / 16 | 16 / 16 |
|
||||
| `<=>` (C++20) | 16 / 16 | 16 / 16 | 16 / 16 | 16 / 16 | 48 / 16 |
|
||||
| `std::hash` | 16 / 16 | 16 / 16 | 16 / 16 | 32 / 48 | 80 / 80 |
|
||||
| `merge_patch`, `update(j, true)`, `patch` | 16 / 16 | 16 / 16 | 16 / 16 | 32 / 32 | 64 / 48 |
|
||||
| `diff` | 16 / 16 | 16 / 16 | 16 / 32 | 48 / 48 | 96 / 112 |
|
||||
| `merge_diff` | 16 / 16 | 16 / 16 | 16 / 16 | 48 / 32 | 96 / 64 |
|
||||
| `to_cbor`, `to_msgpack` | 16 / 16 | 16 / 16 | 16 / 16 | 16 / 16 | 16 / 16 |
|
||||
| `to_ubjson`, `to_bjdata` | 16 / 16 | 16 / 16 | 16 / 16 | 32 / 32 | 64 / 64 |
|
||||
| `to_bon8` | 16 / 16 | 16 / 16 | 16 / 16 | 16 / 32 | 32 / 64 |
|
||||
| conversion to another `basic_json` type | 16 / 16 | 16 / 16 | 16 / 16 | 32 / 48 | 64 / 96 |
|
||||
| destruction | 16 / 16 | 16 / 16 | 16 / 16 | 16 / 16 | 16 / 16 |
|
||||
|
||||
With a limit of `0`, every operation in the tables ran in a 16 KiB stack at `-O0` (Clang and GCC), on a value nested
|
||||
20,000 levels deep. The operations that do not depend on the macro (`flatten`, `unflatten`, `to_bson`) needed
|
||||
16 KiB or less at every limit.
|
||||
|
||||
The rule of thumb is
|
||||
|
||||
> stack needed ≈ a few KiB + cost per level × limit
|
||||
|
||||
with a cost per level of up to 2.3 KiB (`dump` at `-O0` with Clang; 2.1 KiB with GCC), 1.7 KiB for copying and merging
|
||||
(1.3 KiB with GCC), and 0.1 to 0.5 KiB for most other operations at `-O0`; and 0.5 KiB or less at `-O2`. The numbers
|
||||
come from the difference between the limits `128` and `254` in the tables, so they are rough: other compilers,
|
||||
other targets (the frames of x86-64 code are about as large), other standard libraries, `JSON_DIAGNOSTICS`, and
|
||||
sanitizers change them. Measure with your own tool chain (see the example below) rather than relying on the tables.
|
||||
The stack that your own code and the operating system need comes on top of these numbers.
|
||||
|
||||
Which limit to choose, for the stack that you can spare for the library (the largest limit of those measured
|
||||
for which the slowest operation needs at most that much):
|
||||
|
||||
| Stack for the library | optimized (`-O2`) | unoptimized (`-O0`) |
|
||||
|---|---:|---:|
|
||||
| 32 KiB | 64 | 0 |
|
||||
| 64 KiB | 128 | 16 |
|
||||
| 128 KiB (the default stack of threads on musl) | 254 | 32 |
|
||||
| 256 KiB | 254 | 64 |
|
||||
| 512 KiB | 254 | 128 |
|
||||
| 1 MiB or more | 254 | 254 |
|
||||
|
||||
### Speed
|
||||
|
||||
Operations are faster the more of a value they handle with plain recursion. The levels below the limit are handled with
|
||||
an explicit stack, which has to be kept on the heap and costs more per level. The macro therefore trades speed against
|
||||
stack.
|
||||
|
||||
The table shows the time of copying a value (including destroying the copy), of `==` on two equal values, of `dump`, and
|
||||
of `std::hash` for several limits, relative to the default of `128` (`1.00` means the same time; more is slower). It was
|
||||
measured with Clang 21 at `-O2` in C++17 mode on macOS with arm64, as the best of seven runs of each setting (two
|
||||
rounds, best of both). The documents are ordinary JSON files that are nested 7 (`canada.json`), 8
|
||||
(`citm_catalog.json`), 10 (`twitter.json`), and 2 levels deep (`jeopardy.json`); the other values are one array or object
|
||||
with one member per level, nested 1,000 and 100,000 levels deep. The absolute time at the limit `128` is given in the
|
||||
third column. Differences of up to about 0.1 are noise: a different limit changes the layout of the compiled code, and
|
||||
timings of the same binary vary by about as much. The slow columns (limits 0 and 1) vary more: repeating `std::hash` on
|
||||
`canada.json` gave 1.27 to 1.30 for limit 0 and 1.4 to 1.9 for limit 1, so read the cells of those two columns as
|
||||
accurate to about 0.3.
|
||||
|
||||
| Value | Operation | limit 128 (absolute) | limit 0 | limit 1 | limit 16 | limit 254 |
|
||||
|---|---|---:|---:|---:|---:|---:|
|
||||
| `canada.json` (2.1 MB) | copy | 3.5 ms | 1.80 | 1.80 | 0.99 | 1.00 |
|
||||
| `canada.json` (2.1 MB) | `==` | 350.5 µs | 2.36 | 2.38 | 1.00 | 0.99 |
|
||||
| `canada.json` (2.1 MB) | `dump` | 2.4 ms | 1.11 | 1.11 | 1.00 | 1.00 |
|
||||
| `canada.json` (2.1 MB) | `std::hash` | 920.2 µs | 1.33 | 1.59 | 1.00 | 1.00 |
|
||||
| `citm_catalog.json` (1.7 MB) | copy | 1.8 ms | 1.38 | 1.36 | 1.00 | 0.99 |
|
||||
| `citm_catalog.json` (1.7 MB) | `==` | 275.5 µs | 1.31 | 1.31 | 0.99 | 1.00 |
|
||||
| `citm_catalog.json` (1.7 MB) | `dump` | 314.6 µs | 1.24 | 1.23 | 1.00 | 1.00 |
|
||||
| `citm_catalog.json` (1.7 MB) | `std::hash` | 309.1 µs | 1.77 | 1.42 | 1.06 | 1.05 |
|
||||
| `twitter.json` (0.6 MB) | copy | 865.7 µs | 1.45 | 1.47 | 1.01 | 1.00 |
|
||||
| `twitter.json` (0.6 MB) | `==` | 135.8 µs | 1.26 | 1.23 | 1.00 | 1.00 |
|
||||
| `twitter.json` (0.6 MB) | `dump` | 224.6 µs | 1.12 | 1.12 | 1.01 | 1.00 |
|
||||
| `twitter.json` (0.6 MB) | `std::hash` | 132.9 µs | 1.28 | 1.27 | 1.03 | 1.04 |
|
||||
| `jeopardy.json` (53 MB) | copy | 111.1 ms | 1.22 | 1.25 | 1.00 | 1.00 |
|
||||
| `jeopardy.json` (53 MB) | `==` | 23.5 ms | 1.29 | 1.53 | 1.07 | 1.00 |
|
||||
| `jeopardy.json` (53 MB) | `dump` | 33.2 ms | 1.14 | 1.34 | 1.03 | 1.01 |
|
||||
| `jeopardy.json` (53 MB) | `std::hash` | 19.9 ms | 1.17 | 1.52 | 1.05 | 1.00 |
|
||||
| arrays, 1,000 levels | copy | 69.9 µs | 0.95 | 0.98 | 0.92 | 0.99 |
|
||||
| arrays, 1,000 levels | `==` | 10.4 µs | 1.02 | 1.01 | 0.96 | 1.00 |
|
||||
| arrays, 1,000 levels | `dump` | 9.6 µs | 0.97 | 0.96 | 0.97 | 1.06 |
|
||||
| arrays, 1,000 levels | `std::hash` | 16.9 µs | 1.00 | 1.01 | 1.01 | 0.98 |
|
||||
| objects, 1,000 levels | copy | 77.1 µs | 0.98 | 0.97 | 0.97 | 1.03 |
|
||||
| objects, 1,000 levels | `==` | 12.0 µs | 0.96 | 0.95 | 0.95 | 1.07 |
|
||||
| objects, 1,000 levels | `dump` | 12.6 µs | 0.97 | 0.95 | 0.98 | 1.01 |
|
||||
| objects, 1,000 levels | `std::hash` | 17.3 µs | 0.99 | 1.01 | 0.98 | 1.00 |
|
||||
| arrays, 100,000 levels | copy | 6.4 ms | 1.00 | 1.00 | 1.00 | 1.00 |
|
||||
| arrays, 100,000 levels | `==` | 1.1 ms | 1.01 | 1.03 | 1.00 | 1.06 |
|
||||
| arrays, 100,000 levels | `dump` | 917.9 µs | 1.01 | 1.01 | 1.02 | 1.01 |
|
||||
| arrays, 100,000 levels | `std::hash` | 1.7 ms | 1.04 | 1.05 | 0.99 | 0.99 |
|
||||
| objects, 100,000 levels | copy | 7.5 ms | 0.99 | 1.02 | 1.01 | 0.99 |
|
||||
| objects, 100,000 levels | `==` | 1.3 ms | 1.02 | 1.09 | 1.08 | 0.96 |
|
||||
| objects, 100,000 levels | `dump` | 1.2 ms | 1.06 | 0.99 | 1.01 | 0.99 |
|
||||
| objects, 100,000 levels | `std::hash` | 1.8 ms | 1.00 | 1.00 | 1.00 | 1.00 |
|
||||
|
||||
The results are as follows.
|
||||
|
||||
- A limit of `16` or more does not change the speed of ordinary documents (differences are within noise), because they
|
||||
are not nested more deeply than that anyway. A document nested more deeply than the limit is walked partly with the
|
||||
explicit stack.
|
||||
- A limit of `0` or `1` makes the operations on ordinary documents 1.1 to 2.4 times as slow, because even their first
|
||||
levels are handled with the explicit stack. The slowdown is largest for `==` on a document of many small containers
|
||||
(`canada.json`: 2.4 times) and for copying (up to 1.8 times).
|
||||
- For values that are nested far more deeply than any limit (1,000 and 100,000 levels deep), the limit makes no
|
||||
measurable difference: almost all of the levels are handled with the explicit stack for every setting.
|
||||
Raising the limit gains nothing for them, and lowering it costs nothing.
|
||||
|
||||
!!! note "Other macros"
|
||||
|
||||
- With [`JSON_NO_THREAD_LOCAL`](json_no_thread_local.md) defined (or for Clang targeting MinGW, where the library
|
||||
defines it by itself), copying and comparing never descend, so for them the limit does not matter. The other
|
||||
operations still recurse as many levels as the limit says. The range of the limit is the same.
|
||||
- The macro is not part of the [ABI tag](../../features/namespace.md), as `JSON_NO_THREAD_LOCAL` is not, and it is
|
||||
not undefined at the end of `json.hpp`.
|
||||
|
||||
!!! warning "Use the same value in every translation unit"
|
||||
|
||||
All translation units of a program that include the library must define the same value. Otherwise, the program
|
||||
violates the One Definition Rule: the function `nlohmann::detail::recursion_depth_limit()` and all the functions
|
||||
that
|
||||
use it are defined differently in different translation units. The linker keeps one definition of each function that
|
||||
several translation units define, and every caller then recurses as far as that definition says, whatever value
|
||||
its own translation unit defined. Each function uses the limit of whichever translation unit's copy was kept, and
|
||||
which one that is is not specified.
|
||||
|
||||
Mixing values cannot corrupt the byte per thread that counts the levels of copying and comparing: each limit
|
||||
is at most `254`, and all of them stop counting at their own limit, so the count never exceeds `254`, whichever
|
||||
translation unit's definition runs. The counter exists once per thread and `basic_json` specialization, and is
|
||||
shared by all translation units. Nor can mixing values change any result, as no result depends on the limit. What
|
||||
changes is how much stack is used and how fast the operations are. Nevertheless, avoid it: the program has undefined
|
||||
behavior as far as the C++ standard is concerned.
|
||||
|
||||
## Examples
|
||||
|
||||
??? example "Example: lower the limit"
|
||||
|
||||
The example nests an array 5,000 levels deep. With the limit set to 16, the library recurses over the first 16
|
||||
levels
|
||||
and over the rest uses an explicit stack. The results are those of the default limit.
|
||||
|
||||
```cpp
|
||||
--8<-- "examples/json_recursion_depth_limit.cpp"
|
||||
```
|
||||
|
||||
Output:
|
||||
|
||||
```
|
||||
--8<-- "examples/json_recursion_depth_limit.output"
|
||||
```
|
||||
|
||||
??? example "Example: find out how much stack an operation needs"
|
||||
|
||||
The program runs `dump()` on a value nested `depth` levels deep in a thread with a stack of `kib` KiB. Compile it
|
||||
with the options and the limit that you use, e.g. `-DJSON_RECURSION_DEPTH_LIMIT=64`, run it with `depth` set to the
|
||||
limit plus 10 and decreasing values of `kib` (the stack size must be a multiple of the page size, often 4 or 16
|
||||
KiB),
|
||||
and note the smallest stack with which it still finishes. That is how the tables above were measured.
|
||||
|
||||
```cpp
|
||||
#include <cstdlib>
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <pthread.h>
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
static nlohmann::json value;
|
||||
|
||||
static void* run(void*)
|
||||
{
|
||||
std::cout << "dump() ok, " << value.dump().size() << " bytes\n";
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
const int depth = std::atoi(argv[1]);
|
||||
const std::size_t kib = static_cast<std::size_t>(std::atoi(argv[2]));
|
||||
|
||||
// built on the main thread, without recursion
|
||||
value = 1;
|
||||
for (int i = 0; i < depth; ++i)
|
||||
{
|
||||
nlohmann::json outer = nlohmann::json::array();
|
||||
outer.push_back(std::move(value));
|
||||
value = std::move(outer);
|
||||
}
|
||||
|
||||
pthread_attr_t attr;
|
||||
pthread_attr_init(&attr);
|
||||
pthread_attr_setstacksize(&attr, kib * 1024);
|
||||
pthread_t thread;
|
||||
pthread_create(&thread, &attr, run, nullptr);
|
||||
pthread_join(thread, nullptr);
|
||||
}
|
||||
```
|
||||
|
||||
## See also
|
||||
|
||||
- [Nesting depth](../../features/parsing/untrusted_input.md#nesting-depth) - how the library handles deeply nested
|
||||
values
|
||||
- [`JSON_NO_THREAD_LOCAL`](json_no_thread_local.md) - switch off the use of `thread_local` storage
|
||||
- [Supported Macros](../../features/macros.md) - guide to all supported macros
|
||||
|
||||
## Version history
|
||||
|
||||
- Added in version 3.13.0.
|
||||
@@ -1,173 +0,0 @@
|
||||
# JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE
|
||||
|
||||
```cpp
|
||||
#define JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE /* value */
|
||||
```
|
||||
|
||||
[`from_ubjson`](../basic_json/from_ubjson.md) and [`sax_parse`](../basic_json/sax_parse.md) refuse an input in which the
|
||||
optimized arrays of type `Z` (null), `T` (true), and `F` (false) together have more than 1,048,576 elements plus
|
||||
`JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE` for every byte read so far, with
|
||||
[`out_of_range.408`](../../home/exceptions.md#jsonexceptionout_of_range408). The macro sets that number. It is the
|
||||
elements per byte that the input is allowed to announce without paying for them: an element of such an array is a
|
||||
single marker that is implied by the array's `$Z#n` header, so a few bytes can announce a million values (see
|
||||
[Parsing Untrusted Input](../../features/parsing/untrusted_input.md)).
|
||||
|
||||
The macro only matters for input that was not written by this library 3.13.0 or later:
|
||||
[`to_ubjson`](../basic_json/to_ubjson.md) never writes more than the base of 1,048,576 elements in typed arrays of
|
||||
these types, and a reader accepts at least that many whatever the macro is. Its main use is reading files written by
|
||||
version 3.12.0 and earlier, which did not limit them (see
|
||||
[below](#reading-files-written-by-version-3120-and-earlier)).
|
||||
|
||||
## Default definition
|
||||
|
||||
The default value is `4`.
|
||||
|
||||
```cpp
|
||||
#define JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE 4
|
||||
```
|
||||
|
||||
## Notes
|
||||
|
||||
!!! info "Allowed values"
|
||||
|
||||
The value must be a **positive integer** (a value of `0` would not allow any element beyond the base, and a negative
|
||||
or fractional rate has no meaning). Any other value is a compile-time error with a message that names the macro:
|
||||
|
||||
```
|
||||
error: static assertion failed: JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE must be a positive integer
|
||||
error: static assertion failed: JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE must be an integer (not bool or a character type)
|
||||
error: static assertion failed: JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE must not be wider than std::uintmax_t
|
||||
```
|
||||
|
||||
The first is reported for `0` and negative values, the second for values such as `4.5`, `1e3`, `true`, or `'a'`, and
|
||||
the third for an integer type that is wider than `#!cpp std::uintmax_t` (for example `unsigned __int128`). A value
|
||||
that is larger than the largest `#!cpp std::size_t` is treated as that largest value, so that a project can use one
|
||||
spelling, for example `0xFFFFFFFFFFFFFFFFu`, for 32-bit and 64-bit targets.
|
||||
|
||||
!!! info "Large values: the limit is switched off"
|
||||
|
||||
The limit is the base plus the rate times the bytes read. It saturates at the largest `#!cpp std::size_t` instead of
|
||||
overflowing, for any rate and any input size (also with a 32-bit `#!cpp std::size_t`), so a large value never makes
|
||||
the limit smaller. A rate of **at least 131,072** (`1048576 / 8`) effectively switches the limit for the whole
|
||||
input off: the shortest header of an array of more than 32,767 elements takes 8 bytes (`$Z#l` and four bytes of the
|
||||
count, inside an optimized array of arrays; 9 bytes, `[$Z#l`, at the top level), and a shorter header (`$Z#I` and
|
||||
two bytes) can announce at most 32,767. No array can then announce more than the rate times the bytes of its own
|
||||
header. With a rate of 131,071 that is not so: rows of 1,048,576 elements in 8 bytes each gain 8 elements per row,
|
||||
and a table of 278,527 such rows (2.2 MB) is the shortest input that is refused.
|
||||
|
||||
!!! info "What the macro does not change"
|
||||
|
||||
- The limit of **one array** stays at 1,048,576 elements
|
||||
([`out_of_range.408`](../../home/exceptions.md#jsonexceptionout_of_range408), "excessive array size"), whatever
|
||||
the macro is. A value that switches the whole-input limit off does not allow a single array of 1,048,577
|
||||
elements.
|
||||
- The **base** of 1,048,576 elements is a fixed constant, the same as the limit of one array. An array that was
|
||||
accepted on its own before the whole-input limit existed is therefore still accepted at the start of an input.
|
||||
- The **writer** is not affected. [`to_ubjson`](../basic_json/to_ubjson.md) follows the base only: it writes a typed
|
||||
array of `Z`, `T`, or `F` only while the arrays written so far and this one stay within 1,048,576 elements, and
|
||||
writes the elements one by one otherwise. That is within the limit of every reader, because the limit is never
|
||||
below the base.
|
||||
- **Objects** and arrays of other types are not counted: every member of an object has a key, and every element of
|
||||
another type takes at least a byte of input.
|
||||
- [`from_bjdata`](../basic_json/from_bjdata.md) is not affected: BJData has no optimized arrays of these types.
|
||||
|
||||
!!! info "Memory"
|
||||
|
||||
A value takes 16 bytes in a DOM, so every element that the limit lets pass costs memory. Beyond the first 1,048,576
|
||||
elements (about 33 MB: 16 bytes per element, and a vector that grows by reallocation needs the old and the new
|
||||
storage at once), a malicious input of `n` bytes can take **about `20 * K * n` bytes** of memory in a DOM, where `K`
|
||||
is the value of the macro. The measurements below read inputs that are at the limit, with containers of `9 * K`
|
||||
elements (`[$Z#l` and a count, 9 bytes each), with `from_ubjson` on macOS, arm64, compiled with `-O2`. The numbers
|
||||
on other systems differ by the allocator; the 16 bytes per element are the lower bound.
|
||||
|
||||
| `K` | bytes of memory per byte of input | memory for 1 MB of input |
|
||||
|------:|----------------------------------:|-------------------------:|
|
||||
| 1 | 25 | 25 MB |
|
||||
| 4 | 81 | 81 MB |
|
||||
| 16 | 312 | 312 MB |
|
||||
| 64 | 1,226 | 1.2 GB |
|
||||
| 256 | 5,493 | 5.5 GB |
|
||||
|
||||
A SAX handler receives the same number of events, and no memory beyond what it keeps. Choose the smallest value that
|
||||
accepts your files, not a large one, if the input comes from an untrusted source: a value of 131,072 or more
|
||||
switches the limit of the whole input off, so that every 8 bytes of input can announce another 1,048,576 values,
|
||||
the limit of one array.
|
||||
|
||||
!!! warning "Define it before the include, in every translation unit"
|
||||
|
||||
This macro must be defined **before** including `<nlohmann/json.hpp>`. Defining it after the include has no effect.
|
||||
|
||||
The value is compiled into the member functions of the binary reader, which are inline, but it is **not**
|
||||
encoded in the [namespace](../../features/namespace.md) of the library (there is no ABI tag for it). It must
|
||||
have the same value in every translation unit of a program (for example by defining it on the command line or in
|
||||
a common header); translation units with different values violate the One Definition Rule, which is undefined
|
||||
behavior (no diagnostic is required; in practice the linker keeps one definition of each function, so some
|
||||
inputs are read with one value and some with the other).
|
||||
|
||||
### Reading files written by version 3.12.0 and earlier
|
||||
|
||||
Version 3.12.0 wrote an array of `Z`, `T`, or `F` of any length in the optimized form, and a table of equal rows as an
|
||||
optimized array of optimized arrays: a header of `[$[#` and the number of rows (`i` or `U` and 1 byte, `I` and 2 bytes, or
|
||||
`l` and 4 bytes: `h` = 6, 7, or 9 bytes in all for up to 255, 32,767, or more rows), then for every row `$Z#` and its count:
|
||||
5 bytes for up to 255 elements (`U`), 6 bytes for up to 32,767 (`I`), and 8 bytes above (`l`). Every row announces `N`
|
||||
elements for the `b` bytes of its header, so the limit grows by `K * b` and the number of elements by `N` per row.
|
||||
|
||||
- A table is never refused if its rows have at most `K * b` elements each: at most 5 `K` for rows of up to 255
|
||||
elements (20 at the default of `K = 4`), at most 6 `K` for rows of 256 to 32,767 (rows of 1,000 need `K >= 167`),
|
||||
and at most 8 `K` above.
|
||||
- Otherwise the first `(1,048,576 + h * K) / (N - K * b)` rows are accepted, rounded down (`h` as above; 7 for the
|
||||
numbers here). For the default, a table of rows of 1,000 `null` values is read up to row 1,074 (the input of the first
|
||||
example below has 1,100); for `K = 1`, up to row 1,054; for `K = 16`, up to row 1,160.
|
||||
- In general, the smallest value that accepts an input is the largest of
|
||||
`(the elements of the arrays so far - 1,048,576) / (the bytes read when the header of the last of them has been
|
||||
read)` over all arrays, rounded up. For 1,100 rows of 1,000 elements that is
|
||||
`(1,100,000 - 1,048,576) / 6,607 = 7.8`, so `K = 8`.
|
||||
|
||||
To read such a file, define the macro to the smallest value that accepts it before the include (or with
|
||||
`-DJSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE=8`), and see the table above for what that value allows an untrusted input to
|
||||
cost. A file can be converted once: read it with the larger value and write it again with `to_ubjson`, which writes no
|
||||
typed array that a reader with the default value would refuse, so that the result does not need the macro.
|
||||
|
||||
## Examples
|
||||
|
||||
??? example "Example: default limit"
|
||||
|
||||
The table of 1,100 rows of 1,000 `null` values (6,607 bytes) that version 3.12.0 wrote announces 1,100,000 values,
|
||||
which is more than 1,048,576 plus 4 for every byte read.
|
||||
|
||||
```cpp
|
||||
--8<-- "examples/ubjson_valueless_elements_per_byte.cpp"
|
||||
```
|
||||
|
||||
Output:
|
||||
|
||||
```
|
||||
--8<-- "examples/ubjson_valueless_elements_per_byte.output"
|
||||
```
|
||||
|
||||
??? example "Example: raised limit"
|
||||
|
||||
With 16 elements per byte the same table is read. The limit of one array still applies.
|
||||
|
||||
```cpp
|
||||
--8<-- "examples/ubjson_valueless_elements_per_byte_raised.cpp"
|
||||
```
|
||||
|
||||
Output:
|
||||
|
||||
```
|
||||
--8<-- "examples/ubjson_valueless_elements_per_byte_raised.output"
|
||||
```
|
||||
|
||||
## See also
|
||||
|
||||
- [`from_ubjson`](../basic_json/from_ubjson.md) - create a JSON value from a UBJSON serialization
|
||||
- [`to_ubjson`](../basic_json/to_ubjson.md) - create a UBJSON serialization of a JSON value
|
||||
- [`sax_parse`](../basic_json/sax_parse.md) - generate SAX events
|
||||
- [UBJSON](../../features/binary_formats/ubjson.md) - the format and its limits in the library
|
||||
- [Parsing Untrusted Input](../../features/parsing/untrusted_input.md) - limits and checks for input you do not control
|
||||
- [`out_of_range.408`](../../home/exceptions.md#jsonexceptionout_of_range408) - the exception that reports the limit
|
||||
|
||||
## Version history
|
||||
|
||||
- Added in version 3.13.0.
|
||||
@@ -50,15 +50,18 @@ flowchart LR
|
||||
enabled explicitly. Invalid UTF-8 is rejected.
|
||||
- **Errors are reported, not ignored.** Malformed input results in a [`parse_error`](../home/exceptions.md#parse-errors)
|
||||
with the byte position of the error. Binary readers do not trust announced sizes: strings and binary values grow
|
||||
only as bytes are actually read, arrays reserve at most a fixed number of elements up front (and all of them
|
||||
together no more than the input read so far can back), and sizes that no container can hold are rejected.
|
||||
only as bytes are actually read, arrays reserve at most a fixed number of elements up front, and sizes that no
|
||||
container can hold are rejected.
|
||||
- **Memory is owned by values.** Each `basic_json` value owns its content, and there is no manual memory management in
|
||||
user code. The destructor does not recurse, so destroying a deeply nested value does not exhaust the stack.
|
||||
user code. The destructor recurses only over a bounded number of nesting levels and then continues without the call
|
||||
stack, so destroying a deeply nested value does not exhaust the stack.
|
||||
- **Bounded recursion.** The JSON parser and the binary readers keep their state in explicit stacks instead of
|
||||
recursing per nesting level. Operations that walk a value, such as [`dump`](../api/basic_json/dump.md), copying,
|
||||
comparison, hashing, [`merge_patch`](../api/basic_json/merge_patch.md), [`diff`](../api/basic_json/diff.md),
|
||||
[`flatten`](../api/basic_json/flatten.md), and the binary writers, recurse only up to a fixed depth and continue with
|
||||
an explicit stack below it. Applications that process untrusted input can limit its nesting depth with a
|
||||
an explicit stack below it. The depth is 128 levels by default and can be changed with
|
||||
[`JSON_RECURSION_DEPTH_LIMIT`](../api/macros/json_recursion_depth_limit.md), for example to run on a small stack.
|
||||
Applications that process untrusted input can limit its nesting depth with a
|
||||
[parser callback](../features/parsing/parser_callbacks.md).
|
||||
- **Invariants are checked.** The class invariant (for instance, that the pointer for the stored type is never null) is
|
||||
checked with runtime assertions throughout the test suite.
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
#include <functional>
|
||||
#include <iostream>
|
||||
#include <utility>
|
||||
|
||||
// walk over at most 16 nesting levels on the call stack, and over the rest with an explicit stack
|
||||
#define JSON_RECURSION_DEPTH_LIMIT 16
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
using json = nlohmann::json;
|
||||
|
||||
int main()
|
||||
{
|
||||
// an array nested 5,000 levels deep, built without recursion
|
||||
json j = 1;
|
||||
for (int i = 0; i < 5000; ++i)
|
||||
{
|
||||
json outer = json::array();
|
||||
outer.push_back(std::move(j));
|
||||
j = std::move(outer);
|
||||
}
|
||||
|
||||
// the results are those of any other limit
|
||||
const json copy = j;
|
||||
std::cout << std::boolalpha;
|
||||
std::cout << "limit: " << JSON_RECURSION_DEPTH_LIMIT << '\n';
|
||||
std::cout << "copy equals original: " << (copy == j) << '\n';
|
||||
std::cout << "length of dump(): " << j.dump().size() << '\n';
|
||||
std::cout << "same hash: " << (std::hash<json> {}(copy) == std::hash<json> {}(j)) << '\n';
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
limit: 16
|
||||
copy equals original: true
|
||||
length of dump(): 10001
|
||||
same hash: true
|
||||
@@ -1,34 +0,0 @@
|
||||
#include <cstdint>
|
||||
#include <iostream>
|
||||
#include <vector>
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
using json = nlohmann::json;
|
||||
|
||||
// the bytes version 3.12.0 wrote for a table of 1100 rows of 1000 null values
|
||||
// with to_ubjson(table, true, true): `[$[#I` and the number of rows, then
|
||||
// `$Z#I` and the number of elements for every row
|
||||
std::vector<std::uint8_t> table_written_by_3_12_0()
|
||||
{
|
||||
std::vector<std::uint8_t> bytes = {'[', '$', '[', '#', 'I', 1100 >> 8, 1100 & 0xFF};
|
||||
for (int row = 0; row < 1100; ++row)
|
||||
{
|
||||
bytes.insert(bytes.end(), {'$', 'Z', '#', 'I', 1000 >> 8, 1000 & 0xFF});
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
int main()
|
||||
{
|
||||
try
|
||||
{
|
||||
// 6607 bytes announce 1,100,000 values, more than 1,048,576 plus 4 for
|
||||
// every byte read (the limit is exceeded in row 1075)
|
||||
json table = json::from_ubjson(table_written_by_3_12_0());
|
||||
std::cout << table.size() << " rows\n";
|
||||
}
|
||||
catch (const json::out_of_range& e)
|
||||
{
|
||||
std::cout << e.what() << '\n';
|
||||
}
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
[json.exception.out_of_range.408] syntax error while parsing UBJSON size: excessive number of elements without payload
|
||||
@@ -1,42 +0,0 @@
|
||||
#include <cstdint>
|
||||
#include <iostream>
|
||||
#include <vector>
|
||||
|
||||
// accept 16 values per byte of input instead of 4; define the macro before the
|
||||
// first inclusion of the header, and to the same value in every translation unit
|
||||
#define JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE 16
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
using json = nlohmann::json;
|
||||
|
||||
// the bytes version 3.12.0 wrote for a table of 1100 rows of 1000 null values
|
||||
// with to_ubjson(table, true, true): `[$[#I` and the number of rows, then
|
||||
// `$Z#I` and the number of elements for every row
|
||||
std::vector<std::uint8_t> table_written_by_3_12_0()
|
||||
{
|
||||
std::vector<std::uint8_t> bytes = {'[', '$', '[', '#', 'I', 1100 >> 8, 1100 & 0xFF};
|
||||
for (int row = 0; row < 1100; ++row)
|
||||
{
|
||||
bytes.insert(bytes.end(), {'$', 'Z', '#', 'I', 1000 >> 8, 1000 & 0xFF});
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
int main()
|
||||
{
|
||||
// 6607 bytes announce 1,100,000 values, which is below 1,048,576 plus 16
|
||||
// for every byte read
|
||||
json table = json::from_ubjson(table_written_by_3_12_0());
|
||||
std::cout << table.size() << " rows of " << table[0].size() << " values\n";
|
||||
|
||||
// a single container is still limited to 1,048,576 elements
|
||||
const std::vector<std::uint8_t> too_long = {'[', '$', 'Z', '#', 'l', 0x00, 0x10, 0x00, 0x01};
|
||||
try
|
||||
{
|
||||
json j = json::from_ubjson(too_long);
|
||||
}
|
||||
catch (const json::out_of_range& e)
|
||||
{
|
||||
std::cout << e.what() << '\n';
|
||||
}
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
1100 rows of 1000 values
|
||||
[json.exception.out_of_range.408] syntax error while parsing UBJSON size: excessive array size
|
||||
@@ -119,27 +119,12 @@ The library uses the following mapping from JSON values types to UBJSON types ac
|
||||
receiving side is immediately informed on the number of elements of the container.
|
||||
|
||||
An array whose type marker is `Z` (null), `T` (true) or `F` (false) stores no payload at all, because the marker
|
||||
already is the value. Its declared count is therefore the only thing that decides how much memory the receiving
|
||||
side allocates, and a handful of bytes can describe billions of elements. `from_ubjson` rejects such an array with
|
||||
already is the value. Its declared count is therefore the only thing that decides how much memory the receiving side
|
||||
allocates, and a handful of bytes can describe billions of elements. `from_ubjson` rejects such an array with
|
||||
[`out_of_range.408`](../../home/exceptions.md#jsonexceptionout_of_range408) when the count exceeds 1,048,576
|
||||
(`1 << 20`), or when the counts of all such arrays in the input add up to more than 1,048,576 plus _K_ for every byte
|
||||
read so far, where _K_ is the value of the macro
|
||||
[`JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE`](../../api/macros/json_ubjson_valueless_elements_per_byte.md) (4 by
|
||||
default). `to_ubjson` therefore writes an array of these types in the counted but untyped form when it is longer
|
||||
than that, or when it would bring the total of the typed arrays written so far above 1,048,576: the `$` type
|
||||
marker is dropped, the `#` count is kept, and every element is written with its own marker
|
||||
(`[#l 1048577 Z Z ...`), so everything it produces can be read back. Only input from another encoder, or written by version 3.12.0
|
||||
and earlier of this library, can exceed the limit: for example, 1,100 rows of 1,000 `null` values or 210,000 rows
|
||||
of 25, each with a `$Z` annotation. Version 3.12.0 writes a row in 5 bytes (6 above 255 elements, 8 above 32,767),
|
||||
which raises the limit by 5 _K_ (6 _K_, 8 _K_), so rows of up to 5 _K_ values (20 at the default) are never
|
||||
rejected. To read such a file, define the macro to a larger value (`8` is enough for the 1,100 rows of 1,000
|
||||
`null` values; memory for the arrays then rises to about 20 _K_ bytes per byte of input in a DOM); the macro's
|
||||
page explains how to choose a value and what it costs. The limit of one array of 1,048,576 elements does not
|
||||
depend on the macro. Objects and untyped counted arrays are not subject to this limit.
|
||||
|
||||
An optimized array of type `N` (no-op) is empty whatever its count is, because a no-op is not a value. Its count is
|
||||
not passed on: `sax_parse` reports `start_array(0)` (versions before 3.13.0 reported the declared count), and the DOM
|
||||
parser reserves nothing for it.
|
||||
(`1 << 20`), and `to_ubjson` writes longer arrays of these types in the counted but untyped form: the `$` type marker is
|
||||
dropped, the `#` count is kept, and every element is written with its own marker (`[#l 1048577 Z Z ...`), so any value
|
||||
it produces can be read back. Objects and untyped counted arrays are not subject to this limit.
|
||||
|
||||
!!! info "Binary values"
|
||||
|
||||
@@ -195,13 +180,10 @@ The library maps UBJSON types to JSON value types as follows:
|
||||
|
||||
Every well-formed UBJSON value can be converted to a JSON value, with these exceptions: a high-precision number
|
||||
outside the range of `number_float_t` ([`out_of_range.406`](../../home/exceptions.md#jsonexceptionout_of_range406)),
|
||||
optimized arrays of `Z`, `T`, or `F` with more elements than the limit allows (1,048,576 plus
|
||||
[`JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE`](../../api/macros/json_ubjson_valueless_elements_per_byte.md), 4 by default,
|
||||
per byte read),
|
||||
and a count that exceeds the capacity of the container
|
||||
([`out_of_range.408`](../../home/exceptions.md#jsonexceptionout_of_range408)). High-precision numbers that do not
|
||||
fit an integer type are stored as floating-point numbers and may lose precision. Ill-formed input is rejected with a
|
||||
parse error.
|
||||
an optimized array of `Z`, `T`, or `F` with more than 1,048,576 elements, and a count that exceeds the capacity of the
|
||||
container ([`out_of_range.408`](../../home/exceptions.md#jsonexceptionout_of_range408)). High-precision numbers that
|
||||
do not fit an integer type are stored as floating-point numbers and may lose precision. Ill-formed input is rejected
|
||||
with a parse error.
|
||||
|
||||
!!! info "Duplicate object keys"
|
||||
|
||||
|
||||
@@ -198,6 +198,16 @@ this is planned to become the default in version 4.0.0.
|
||||
|
||||
See [full documentation of `JSON_PRECISE_STREAM_POSITION`](../api/macros/json_precise_stream_position.md).
|
||||
|
||||
## `JSON_RECURSION_DEPTH_LIMIT`
|
||||
|
||||
Sets the number of nesting levels that the operations walking a value (copying, comparing, serializing, hashing,
|
||||
merging, diffing, converting, destroying, and the binary writers) recurse into on the call stack before they continue
|
||||
with an explicit stack on the heap. The default is `128`; allowed are the integers from `0` to `254`. Lower it for
|
||||
threads with small stacks, raise it to trade stack for speed. Results are the same for every setting; the value must be
|
||||
the same in all translation units of a program.
|
||||
|
||||
See [full documentation of `JSON_RECURSION_DEPTH_LIMIT`](../api/macros/json_recursion_depth_limit.md).
|
||||
|
||||
## `JSON_SKIP_LIBRARY_VERSION_CHECK`
|
||||
|
||||
When defined, the library will not create a compiler warning when a different version of the library was already
|
||||
@@ -251,16 +261,6 @@ This macro overrides `#!cpp try` calls inside the library.
|
||||
|
||||
See [full documentation of `JSON_TRY_USER`](../api/macros/json_throw_user.md).
|
||||
|
||||
## `JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE`
|
||||
|
||||
Sets how many elements of optimized UBJSON arrays of `Z` (null), `T` (true), and `F` (false) a reader accepts for
|
||||
every byte of input it has read, on top of a base of 1,048,576 elements. These arrays have no payload, so a few bytes
|
||||
can announce a million values. The value must be a positive integer; the default is `4`, and a value of 131,072 or
|
||||
more switches the limit of the whole input off. The limit of 1,048,576 elements for a single array always applies. The
|
||||
macro is not part of the [ABI tag](namespace.md): use the same value in every translation unit.
|
||||
|
||||
See [full documentation of `JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE`](../api/macros/json_ubjson_valueless_elements_per_byte.md).
|
||||
|
||||
## `JSON_USE_IMPLICIT_CONVERSIONS`
|
||||
|
||||
When defined to `0`, implicit conversions are switched off. By default, implicit conversions are switched on.
|
||||
|
||||
@@ -54,10 +54,32 @@ Once a value has been parsed, operations that walk it recursively — serializin
|
||||
[`dump`](../../api/basic_json/dump.md), hashing it, copying it, comparing two values with `#!cpp ==`, `#!cpp <`, or (in
|
||||
C++20) `#!cpp <=>`, merging with [`update`](../../api/basic_json/update.md), applying a
|
||||
[`merge_patch`](../../api/basic_json/merge_patch.md), creating a patch with [`diff`](../../api/basic_json/diff.md) or
|
||||
[`merge_diff`](../../api/basic_json/merge_diff.md), [`flatten`](../../api/basic_json/flatten.md), and the binary
|
||||
writers (`to_cbor`, `to_msgpack`, ...) — descend at most 128 levels on the call stack and continue
|
||||
below that with an explicit stack instead, so none of them can exhaust the stack either, however deeply the value is
|
||||
nested. Destroying a value (its destructor) never recurses at all, regardless of nesting depth, for the same reason.
|
||||
[`merge_diff`](../../api/basic_json/merge_diff.md), converting it to another `basic_json` type, destroying it, and the
|
||||
binary writers (`to_cbor`, `to_msgpack`, ...) — descend at most
|
||||
[`JSON_RECURSION_DEPTH_LIMIT`](../../api/macros/json_recursion_depth_limit.md) levels on the call stack, 128 by default,
|
||||
and continue below that with an explicit stack on the heap. So the call stack does not grow with the nesting depth of the
|
||||
value, however deeply it is nested. ([`flatten`](../../api/basic_json/flatten.md),
|
||||
[`unflatten`](../../api/basic_json/unflatten.md), and [`to_bson`](../../api/basic_json/to_bson.md) never use the call
|
||||
stack for the nesting.)
|
||||
|
||||
The call stack still needs room for the levels that are recursed into, and the default limit assumes a stack of ordinary
|
||||
size. The stack that an operation needs is roughly proportional to the limit; with the default of 128 levels, the most
|
||||
demanding operation (`dump` in an unoptimized build) needed up to 288 KiB, and an optimized build needed 48 KiB or less,
|
||||
in measurements on macOS with arm64. The most demanding operation needed this much stack at the limits `0`, `32`,
|
||||
`128`, and `254`:
|
||||
|
||||
| Stack needed, in KiB | limit 0 | limit 32 | limit 128 (default) | limit 254 |
|
||||
|---|---:|---:|---:|---:|
|
||||
| unoptimized build (`-O0`) | 16 or less | 80 | 288 | 576 |
|
||||
| optimized build (`-O2`) | 16 or less | 16 or less | 48 | 112 |
|
||||
|
||||
A thread with a smaller stack (a task of a real-time operating system, a fiber, or a thread of the musl C library, whose
|
||||
default stack is 128 KiB) can still overflow its stack on deeply nested input, which is why the limit can be lowered:
|
||||
define `JSON_RECURSION_DEPTH_LIMIT` to a smaller value, down to `0`, before including the library. Such a limit changes
|
||||
only the stack that is needed and the speed (values nested more deeply than the limit are walked more slowly), never a
|
||||
result. See the [documentation of the macro](../../api/macros/json_recursion_depth_limit.md) for the complete tables,
|
||||
the speed, and the rules for choosing a value. Destroying a value (its destructor) recurses over the same number of
|
||||
levels and then continues without the call stack, and needs 32 KiB or less at every limit.
|
||||
|
||||
## Input size
|
||||
|
||||
@@ -71,38 +93,11 @@ For the binary formats, an announced size is never trusted outright:
|
||||
actually consumed, rather than allocating the announced length up front — a truncated input runs out of bytes
|
||||
(reported as a parse error) instead of triggering an oversized allocation.
|
||||
- When an array announces its number of elements and the array container supports `reserve()` (as `#!cpp std::vector`,
|
||||
the default, does), the DOM parser reserves storage for them before the elements are read, but for at most 16384
|
||||
elements per array, regardless of how large the announced count is; further elements still grow the container
|
||||
normally as they are read. The arrays that are open at the same time hold their storage at the same time, and an
|
||||
announcement is not backed by input when it is read, so the elements reserved for all arrays of one input together
|
||||
may not exceed 65,536 plus 2 for every byte of input read so far; an array that announces more gets what is left
|
||||
of that and grows as its elements arrive. An element takes at least one byte of input, so the elements of the
|
||||
arrays that were read use up what was reserved for them, and only arrays that are nested deeply, with an array as
|
||||
their first element, can announce more than is left. For a truncated or hostile input, the storage that
|
||||
announcements can hold is therefore at most (65,536 + 2 per byte of input) values, that is about 1 MiB plus 32 bytes
|
||||
per byte of input with 16-byte values (a value is larger with `JSON_DIAGNOSTICS` or `JSON_DIAGNOSTIC_POSITIONS`, 24 to
|
||||
40 bytes, and so is the bound). For example, 100,000 nested CBOR array headers `99 40 00` (300,000 bytes) that
|
||||
announce 16384 elements each peaked at 23 MiB, against 1.65 GiB if every array reserved what it announced and 12 MiB
|
||||
with version 3.12.0, which reserved nothing.
|
||||
A [SAX](../../api/json_sax/index.md) handler is not affected: it is told the announced number of elements.
|
||||
the default, does), the library reserves storage for at most 16384 of them upfront, regardless of how large the
|
||||
announced count is; further elements still grow the container normally as they are read.
|
||||
- An announced array or object size that exceeds what the target container could ever hold (its `max_size()`) is
|
||||
rejected immediately as [`out_of_range.408`](../../home/exceptions.md#jsonexceptionout_of_range408), without
|
||||
attempting to allocate anything.
|
||||
- A [UBJSON](../binary_formats/ubjson.md) array of the no-op type (`[$N#<n>`) is empty whatever count it declares: a
|
||||
SAX handler is told `start_array(0)` and the DOM parser reserves nothing.
|
||||
- A UBJSON array of type `Z`, `T` or `F` has no payload, so its announced count is not bounded by the input. One array
|
||||
may announce at most 1,048,576 elements, and all such arrays of one input together may contain at most 1,048,576
|
||||
elements plus _K_ for every byte read so far (checked when each array's header has been read), where _K_ is the
|
||||
value of the macro
|
||||
[`JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE`](../../api/macros/json_ubjson_valueless_elements_per_byte.md), 4 by
|
||||
default; beyond that the input is rejected as
|
||||
[`out_of_range.408`](../../home/exceptions.md#jsonexceptionout_of_range408). For a DOM that bounds the memory these
|
||||
arrays can need to about 20 _K_ bytes per byte of input (81 at the default, measured; 312 for _K_ = 16) plus about
|
||||
33 MB for the first 1,048,576 elements; a SAX handler receives at most that many events. Every other optimized array
|
||||
element, and every UBJSON object member (it has a key), takes at least one byte of input. Leave the macro at the
|
||||
default for input you do not control; raise it only to read trusted files that version 3.12.0 or earlier wrote with
|
||||
long arrays of these types, and see its page for how a value is chosen and what it costs. The limit of one array
|
||||
does not depend on the macro.
|
||||
|
||||
## Strings
|
||||
|
||||
|
||||
@@ -1050,21 +1050,11 @@ The size of an array or object in a [binary format](../features/binary_formats/i
|
||||
the size following `#` for [UBJSON](../features/binary_formats/ubjson.md)/[BJData](../features/binary_formats/bjdata.md),
|
||||
or the encoded length for [CBOR](../features/binary_formats/cbor.md).
|
||||
|
||||
The exception is also thrown for [UBJSON](../features/binary_formats/ubjson.md) arrays of a type that is encoded by
|
||||
its marker alone (`Z`, `T` or `F`). Such an array has no payload, so its count alone decides how much memory is
|
||||
allocated, and a handful of bytes would otherwise describe billions of values. It is thrown if the declared count of one
|
||||
array exceeds 1,048,576 (`1 << 20`), and if the counts of all such arrays read so far in one input add up to more than
|
||||
1,048,576 plus 4 for every byte of the input read when the header of the last array has been read. The 4 is the
|
||||
default of the macro
|
||||
[`JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE`](../api/macros/json_ubjson_valueless_elements_per_byte.md), which raises it
|
||||
(for the limit of the whole input; the limit of one array stays).
|
||||
[`to_ubjson`](../api/basic_json/to_ubjson.md) writes an array of these types without the `$` type annotation (the `#`
|
||||
count and one marker per element are kept) when it is longer, or when it would bring the total of the typed arrays
|
||||
written so far above 1,048,576, so everything it produces can be read back. Only input written by another encoder, or
|
||||
by version 3.12.0 and earlier of this library, can exceed the limit: for example, 1,100 rows of 1,000 `null` values or
|
||||
210,000 rows of 25, each with a `$Z` annotation. Version 3.12.0 writes such a row in 5 bytes (6 above 255 elements, 8
|
||||
above 32,767), so rows of 20 or fewer such values are never rejected at the default; define the macro to a larger value
|
||||
to read them.
|
||||
The exception is also thrown for a [UBJSON](../features/binary_formats/ubjson.md) array of a type that is encoded by its
|
||||
marker alone (`Z`, `T` or `F`) whose declared count exceeds 1,048,576 (`1 << 20`). Such an array has no payload, so its
|
||||
count alone decides how much memory is allocated, and a handful of bytes would otherwise describe billions of values.
|
||||
[`to_ubjson`](../api/basic_json/to_ubjson.md) writes longer arrays of these types without the `$` type annotation (the `#`
|
||||
count and one marker per element are kept), so any value it produces can still be read back.
|
||||
|
||||
!!! failure "Example messages"
|
||||
|
||||
@@ -1080,9 +1070,6 @@ to read them.
|
||||
```
|
||||
[json.exception.out_of_range.408] syntax error while parsing UBJSON size: excessive array size
|
||||
```
|
||||
```
|
||||
[json.exception.out_of_range.408] syntax error while parsing UBJSON size: excessive number of elements without payload
|
||||
```
|
||||
|
||||
### json.exception.out_of_range.409
|
||||
|
||||
|
||||
@@ -454,11 +454,11 @@ nav:
|
||||
- 'JSON_NO_SIMD': api/macros/json_no_simd.md
|
||||
- 'JSON_NO_THREAD_LOCAL': api/macros/json_no_thread_local.md
|
||||
- 'JSON_PRECISE_STREAM_POSITION': api/macros/json_precise_stream_position.md
|
||||
- 'JSON_RECURSION_DEPTH_LIMIT': api/macros/json_recursion_depth_limit.md
|
||||
- 'JSON_SKIP_LIBRARY_VERSION_CHECK': api/macros/json_skip_library_version_check.md
|
||||
- 'JSON_SKIP_UNSUPPORTED_COMPILER_CHECK': api/macros/json_skip_unsupported_compiler_check.md
|
||||
- 'JSON_STRICT_BINARY_UTF8': api/macros/json_strict_binary_utf8.md
|
||||
- 'JSON_STRICT_NUL_HANDLING': api/macros/json_strict_nul_handling.md
|
||||
- 'JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE': api/macros/json_ubjson_valueless_elements_per_byte.md
|
||||
- 'JSON_USE_GLOBAL_UDLS': api/macros/json_use_global_udls.md
|
||||
- 'JSON_USE_IMPLICIT_CONVERSIONS': api/macros/json_use_implicit_conversions.md
|
||||
- 'JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON': api/macros/json_use_legacy_discarded_value_comparison.md
|
||||
|
||||
@@ -16,7 +16,6 @@
|
||||
#include <iterator> // back_inserter
|
||||
#include <limits> // numeric_limits
|
||||
#include <string> // char_traits, string
|
||||
#include <type_traits> // false_type, integral_constant, is_convertible, is_integral, is_same, remove_cv, remove_reference, true_type
|
||||
#include <utility> // make_pair, move
|
||||
#include <vector> // vector
|
||||
#ifdef __cpp_lib_byteswap
|
||||
@@ -74,124 +73,13 @@ the input.
|
||||
|
||||
This is a sanity bound rather than a security boundary, and it is far above
|
||||
any container met in practice. @ref binary_writer falls back to the
|
||||
unoptimized encoding for longer containers, and for containers that together
|
||||
with the ones before them would exceed @ref valueless_elements_base, so that a
|
||||
value serialized by this library can always be read back.
|
||||
unoptimized encoding for longer containers, so that a value serialized by
|
||||
this library can always be read back.
|
||||
|
||||
@sa https://github.com/nlohmann/json/issues/2793
|
||||
*/
|
||||
JSON_INLINE_VARIABLE constexpr std::size_t max_valueless_container_size = 1 << 20;
|
||||
|
||||
/*!
|
||||
@brief number of elements of valueless UBJSON containers accepted on top of
|
||||
@ref valueless_elements_per_byte
|
||||
|
||||
The limit @ref max_valueless_container_size applies to each container on its
|
||||
own, so on its own it lets 9 bytes announce 1048576 values, and every further
|
||||
8 bytes (a row of an optimized array of arrays) another 1048576. The elements of all containers of a valueless type
|
||||
in one input are therefore added up, and the sum must stay below this base
|
||||
plus @ref valueless_elements_per_byte for every byte read so far. The base is
|
||||
the per-container limit, so a container that was accepted on its own before
|
||||
the sum was introduced is still accepted at the start of an input.
|
||||
*/
|
||||
JSON_INLINE_VARIABLE constexpr std::size_t valueless_elements_base = max_valueless_container_size;
|
||||
|
||||
/// the value of JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE as a std::size_t:
|
||||
/// saturated at the largest one, not truncated, for a value that is wider
|
||||
template<typename T>
|
||||
constexpr std::size_t valueless_elements_rate(const T value) noexcept
|
||||
{
|
||||
return static_cast<std::uintmax_t>(value) > (std::numeric_limits<std::size_t>::max)()
|
||||
? (std::numeric_limits<std::size_t>::max)()
|
||||
: static_cast<std::size_t>(value);
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief number of elements of valueless UBJSON containers accepted per byte of
|
||||
input, in addition to @ref valueless_elements_base
|
||||
|
||||
This is the value of the macro JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE (4 by
|
||||
default), see docs/mkdocs/docs/api/macros/json_ubjson_valueless_elements_per_byte.md.
|
||||
|
||||
An element of a value that has a payload costs at least one byte of input, but
|
||||
an element of a valueless type costs none, so these are the only elements whose
|
||||
number the input does not bound. They are accepted at this rate per byte read,
|
||||
which bounds the memory they take in a DOM: a value takes 16 bytes there, so
|
||||
the arrays at the limit take about 20 times this rate bytes of memory per byte
|
||||
of input (16 for the value, the rest is allocator rounding and the arrays
|
||||
themselves; measured on macOS, DOM, with containers of 9 times the rate
|
||||
elements: 25 bytes per byte at a rate of 1, 81 at 4, 312 at 16, 1226 at 64 and
|
||||
5493 at 256), plus about 33 MB for the first elements of the base (16 bytes for
|
||||
each of the 1048576 elements, and a vector that grows by reallocation, as only
|
||||
16384 elements are reserved upfront, needs the old and the new storage at once).
|
||||
|
||||
The bound is meant for input written by other encoders and by versions before
|
||||
3.13.0, which wrote long arrays of this type optimized without a limit: input
|
||||
written by @ref binary_writer never needs more than the base, because the
|
||||
writer falls back to the unoptimized form once the arrays written so far and
|
||||
the next one add up to more than that, whatever the rate is.
|
||||
|
||||
Range: a positive integer. A value that is not one is a compile-time error (a
|
||||
reader with a rate of 0 would accept nothing beyond the base, and a negative or
|
||||
fractional rate has no meaning). A value that does not fit into std::size_t is
|
||||
used as the largest std::size_t: the rate is so high then that the bound is
|
||||
out of reach anyway, and a project can use one spelling (for example
|
||||
`0xFFFFFFFFFFFFFFFFu`) for 32 and 64 bit targets. The type of the value must
|
||||
be an integer type that is not wider than std::uintmax_t and not bool or a
|
||||
character type. A rate of at least 2^20 / 8 = 131072 effectively switches the
|
||||
whole-input bound off: an array of more than 32767 elements needs a count of
|
||||
type `l`, so its shortest header is `$Z#l` and four bytes (8 bytes) inside an
|
||||
optimized array of arrays (9 bytes, `[$Z#l`, at the top level), and a shorter
|
||||
header (`$Z#I` and two bytes) announces at most 32767. Every container thus has
|
||||
at most (rate * header bytes) elements, which never exceeds the limit. With a
|
||||
rate of 131071 an input that is refused exists (rows of 1048576 elements gain
|
||||
8 elements each); the per-container limit still applies.
|
||||
|
||||
The value must be the same in every translation unit of a program (it is
|
||||
compiled into member functions of binary_reader, which are inline), and it is
|
||||
not part of the ABI tag of the namespace.
|
||||
*/
|
||||
JSON_INLINE_VARIABLE constexpr std::size_t valueless_elements_per_byte =
|
||||
// a value that does not fit into std::size_t is the largest one (see above);
|
||||
// the comparison is made in the widest unsigned type, so that nothing is
|
||||
// truncated before it is made. A function template, so that the casts are
|
||||
// not "useless" (-Wuseless-cast) where the macro already has the type
|
||||
valueless_elements_rate((JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE));
|
||||
|
||||
/// the type of the macro's value is a character or boolean type
|
||||
template<typename T>
|
||||
struct valueless_elements_rate_is_char_or_bool : std::integral_constant < bool,
|
||||
std::is_same<T, bool>::value || std::is_same<T, char>::value || std::is_same<T, signed char>::value
|
||||
|| std::is_same<T, unsigned char>::value || std::is_same<T, wchar_t>::value
|
||||
|| std::is_same<T, char16_t>::value || std::is_same<T, char32_t>::value >
|
||||
{};
|
||||
|
||||
// these messages are what a user sees for a wrong value of the macro. The
|
||||
// integer test comes first because a fractional value would also fail the
|
||||
// positivity test; a value wider than std::uintmax_t would be truncated by
|
||||
// the conversion above, so it is rejected
|
||||
static_assert(std::is_integral<typename std::remove_reference<decltype((JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE))>::type>::value
|
||||
&& !valueless_elements_rate_is_char_or_bool<typename std::remove_cv<typename std::remove_reference<decltype((JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE))>::type>::type>::value,
|
||||
"JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE must be an integer (not bool or a character type)");
|
||||
static_assert(sizeof(decltype((JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE))) <= sizeof(std::uintmax_t),
|
||||
"JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE must not be wider than std::uintmax_t");
|
||||
static_assert((JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE) > 0,
|
||||
"JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE must be a positive integer");
|
||||
|
||||
/*!
|
||||
@brief the number of bytes read up to which
|
||||
@ref valueless_elements_base plus @ref valueless_elements_per_byte times
|
||||
that number can be computed in std::size_t
|
||||
|
||||
For more bytes than this the limit of the valueless elements is the largest
|
||||
std::size_t, which is saturation: the sum is not computed, because it would wrap
|
||||
around, which happens for a rate that is large (the largest std::size_t as the
|
||||
rate wraps already at the first byte) or an input that is large (more than
|
||||
about a billion bytes for the default rate with a 32-bit std::size_t). It is a
|
||||
constant of the build, so no division is made while reading.
|
||||
*/
|
||||
JSON_INLINE_VARIABLE constexpr std::size_t valueless_elements_saturation_chars =
|
||||
((std::numeric_limits<std::size_t>::max)() - valueless_elements_base) / (valueless_elements_per_byte == 0 ? 1 : valueless_elements_per_byte);
|
||||
///////////////////
|
||||
// binary reader //
|
||||
///////////////////
|
||||
@@ -259,8 +147,6 @@ class binary_reader
|
||||
sax = sax_;
|
||||
container_stack.clear();
|
||||
bon8_pushback_size = 0;
|
||||
// the position is only known to the DOM parser while this reader lives
|
||||
const position_binding binding(bound_parser(sax, std::is_convertible<SAX*, json_sax_dom_error_state*> {}), &chars_read);
|
||||
bool result = false;
|
||||
|
||||
switch (input_format)
|
||||
@@ -318,61 +204,6 @@ class binary_reader
|
||||
}
|
||||
|
||||
private:
|
||||
/// the DOM parser of @a s whose reserved storage is bounded by the input
|
||||
/// read so far; nullptr for any other SAX parser, which does not reserve,
|
||||
/// and for one that derives from a DOM parser privately or protectedly
|
||||
/// (or ambiguously), which the reader cannot reach
|
||||
static json_sax_dom_error_state* bound_parser(SAX* s, std::true_type /*is a DOM parser*/) noexcept
|
||||
{
|
||||
return s;
|
||||
}
|
||||
|
||||
static json_sax_dom_error_state* bound_parser(SAX* /*s*/, std::false_type /*is a DOM parser*/) noexcept
|
||||
{
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief tells a DOM parser where to find the number of characters read
|
||||
while sax_parse() runs, and puts back what it knew before when
|
||||
sax_parse() returns or throws
|
||||
|
||||
The reader is a local of the caller and is gone then, so the parser must
|
||||
not keep its position. A parser that was bound by a reader that is still
|
||||
running (a parse inside a parse) gets that binding back; that reader is
|
||||
still alive, as it encloses this one.
|
||||
*/
|
||||
class position_binding
|
||||
{
|
||||
public:
|
||||
position_binding(json_sax_dom_error_state* parser_, const std::size_t* position) noexcept
|
||||
: parser(parser_), saved()
|
||||
{
|
||||
if (parser != nullptr)
|
||||
{
|
||||
saved = parser->get_reserve_state();
|
||||
parser->set_input_position(position);
|
||||
}
|
||||
}
|
||||
|
||||
position_binding(const position_binding&) = delete;
|
||||
position_binding(position_binding&&) = delete;
|
||||
position_binding& operator=(const position_binding&) = delete;
|
||||
position_binding& operator=(position_binding&&) = delete;
|
||||
|
||||
~position_binding()
|
||||
{
|
||||
if (parser != nullptr)
|
||||
{
|
||||
parser->set_reserve_state(saved);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
json_sax_dom_error_state* parser;
|
||||
json_sax_dom_error_state::reserve_state saved;
|
||||
};
|
||||
|
||||
////////////////////////
|
||||
// nested containers //
|
||||
////////////////////////
|
||||
@@ -3869,38 +3700,6 @@ class binary_reader
|
||||
return emit_unsigned(number);
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief add the elements of a container of a valueless type to the total of
|
||||
the input and check the total against the input read so far
|
||||
|
||||
A container of type 'Z', 'T' or 'F' has no payload, so its elements are the
|
||||
only ones for which the input does not bound the count. The check is made
|
||||
once per container, not per element.
|
||||
|
||||
@param[in] count number of elements the container declares; at most
|
||||
@ref max_valueless_container_size
|
||||
@return false if the total is more than @ref valueless_elements_base plus
|
||||
@ref valueless_elements_per_byte for every byte read so far; the
|
||||
total is not changed then
|
||||
*/
|
||||
bool charge_valueless_elements(const std::size_t count) noexcept
|
||||
{
|
||||
// limit = base + rate * chars_read, saturating at the largest size_t
|
||||
// instead of wrapping around, which would make a huge rate or input
|
||||
// reject everything. valueless_elements never exceeds the limit of an
|
||||
// earlier call, and the limit does not decrease, so the difference
|
||||
// below cannot underflow.
|
||||
const std::size_t limit = chars_read <= valueless_elements_saturation_chars
|
||||
? valueless_elements_base + (valueless_elements_per_byte * chars_read)
|
||||
: (std::numeric_limits<std::size_t>::max)();
|
||||
if (JSON_HEDLEY_UNLIKELY(count > limit - valueless_elements))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
valueless_elements += count;
|
||||
return true;
|
||||
}
|
||||
|
||||
/*!
|
||||
@return whether array creation completed
|
||||
*/
|
||||
@@ -3986,27 +3785,24 @@ class binary_reader
|
||||
// declared count alone decides how much is allocated; the check is
|
||||
// made before the start event so that no container is opened that
|
||||
// is then abandoned. See @ref max_valueless_container_size.
|
||||
if (JSON_HEDLEY_UNLIKELY(size_and_type.second == 'Z' || size_and_type.second == 'T' || size_and_type.second == 'F'))
|
||||
if (JSON_HEDLEY_UNLIKELY((size_and_type.second == 'Z' || size_and_type.second == 'T' || size_and_type.second == 'F')
|
||||
&& size_and_type.first > max_valueless_container_size))
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(size_and_type.first > max_valueless_container_size))
|
||||
{
|
||||
return out_of_range_error(exception_id::container_too_large, "excessive array size", "size");
|
||||
}
|
||||
if (JSON_HEDLEY_UNLIKELY(!charge_valueless_elements(size_and_type.first)))
|
||||
{
|
||||
return out_of_range_error(exception_id::container_too_large, "excessive number of elements without payload", "size");
|
||||
}
|
||||
return out_of_range_error(exception_id::container_too_large, "excessive array size", "size");
|
||||
}
|
||||
|
||||
// a no-op is not a value, so an array of them is empty whatever
|
||||
// count it declares: the SAX parser is told so, instead of being
|
||||
// made to reserve storage for a count that no input backs
|
||||
const std::size_t announced = size_and_type.second == 'N' ? 0 : size_and_type.first;
|
||||
if (JSON_HEDLEY_UNLIKELY(!enter_array(announced, size_and_type.second)))
|
||||
if (JSON_HEDLEY_UNLIKELY(!enter_array(size_and_type.first, size_and_type.second)))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (size_and_type.second == 'N')
|
||||
{
|
||||
// a no-op is not a value, so a container of them holds none;
|
||||
// the declared size has already been passed to the SAX parser
|
||||
container_stack.back().remaining = 0;
|
||||
}
|
||||
|
||||
return size_and_type.second == 0
|
||||
|| parse_ubjson_typed_run(container_stack.back().remaining, size_and_type.second, std::integral_constant<bool, bulk_scan> {});
|
||||
}
|
||||
@@ -5318,10 +5114,6 @@ class binary_reader
|
||||
/// the number of characters read
|
||||
std::size_t chars_read = 0;
|
||||
|
||||
/// UBJSON: the sum of the element counts of the containers of a valueless
|
||||
/// type read so far; see @ref charge_valueless_elements
|
||||
std::size_t valueless_elements = 0;
|
||||
|
||||
/// whether we can assume little endianness
|
||||
const bool is_little_endian = little_endianness();
|
||||
|
||||
|
||||
@@ -159,40 +159,6 @@ constexpr std::size_t unknown_size()
|
||||
return (std::numeric_limits<std::size_t>::max)();
|
||||
}
|
||||
|
||||
/// the largest number of elements the DOM parsers reserve for one array upfront
|
||||
constexpr std::size_t reserve_cap()
|
||||
{
|
||||
return 16384;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief number of array elements a DOM parser may reserve upfront on top of
|
||||
@ref reserved_elements_per_byte
|
||||
|
||||
An array that announces its size is given storage for that many elements when
|
||||
it opens (at most @ref reserve_cap of them), before any input backs the
|
||||
announcement, and the arrays that are open at the same time keep their storage
|
||||
until the input is complete: 100000 nested headers of three bytes each that
|
||||
announce 16384 elements would otherwise need 1.7 GB. The elements reserved for
|
||||
all arrays of an input are therefore added up, and the sum must stay below
|
||||
this base plus @ref reserved_elements_per_byte for every byte read so far; an
|
||||
array that would go beyond reserves what is left, and grows as its elements
|
||||
arrive. The base is four arrays of the maximum size, so the first nested
|
||||
arrays of an input are not affected.
|
||||
*/
|
||||
JSON_INLINE_VARIABLE constexpr std::size_t reserved_elements_base = 4 * reserve_cap();
|
||||
|
||||
/*!
|
||||
@brief number of array elements a DOM parser may reserve upfront per byte of
|
||||
input, in addition to @ref reserved_elements_base
|
||||
|
||||
An element of an array takes at least one byte of input, so an array that
|
||||
completes has used what was reserved for it. At a rate of 2 elements per
|
||||
byte, an array that announces more than the input provides keeps at most
|
||||
32 bytes of memory per byte of input, plus the base, until the parse fails.
|
||||
*/
|
||||
JSON_INLINE_VARIABLE constexpr std::size_t reserved_elements_per_byte = 2;
|
||||
|
||||
/*!
|
||||
@brief reserve capacity for @a len elements in array @a arr
|
||||
|
||||
@@ -208,7 +174,8 @@ template<typename ArrayType>
|
||||
auto reserve_array(ArrayType& arr, std::size_t len, priority_tag<1> /*unused*/)
|
||||
-> decltype(arr.reserve(len), void())
|
||||
{
|
||||
arr.reserve((std::min)(len, reserve_cap()));
|
||||
constexpr std::size_t reserve_cap = 16384;
|
||||
arr.reserve((std::min)(len, reserve_cap));
|
||||
}
|
||||
|
||||
template<typename ArrayType>
|
||||
@@ -375,13 +342,6 @@ struct diagnostic_positions
|
||||
Holds whether a parse error occurred and whether errors are thrown, and
|
||||
implements the parse_error() and is_errored() members of the SAX interface for
|
||||
json_sax_dom_parser and json_sax_dom_callback_parser.
|
||||
|
||||
It also bounds what the parsers reserve for arrays that announce their size
|
||||
(reserve_count()): the announced size is not backed by input yet when the
|
||||
array opens, and the arrays that are open at the same time each keep what was
|
||||
reserved for them, so the binary_reader tells the parser where to find the
|
||||
number of characters it has read (set_input_position()), which the bound
|
||||
is derived from.
|
||||
*/
|
||||
class json_sax_dom_error_state
|
||||
{
|
||||
@@ -415,103 +375,7 @@ class json_sax_dom_error_state
|
||||
return errored;
|
||||
}
|
||||
|
||||
/// bind the number of characters the input has provided so far; see
|
||||
/// @ref reserve_count
|
||||
/// (nullptr: not known, which bounds only the size of each array)
|
||||
void set_input_position(const std::size_t* position) noexcept
|
||||
{
|
||||
input_position = position;
|
||||
credit_position = 0;
|
||||
reserve_credit = position != nullptr ? reserved_elements_base : (std::numeric_limits<std::size_t>::max)();
|
||||
}
|
||||
|
||||
/// what set_input_position() changes, so that a binding can be undone
|
||||
struct reserve_state
|
||||
{
|
||||
const std::size_t* position;
|
||||
std::size_t position_at_refill;
|
||||
std::size_t credit;
|
||||
};
|
||||
|
||||
reserve_state get_reserve_state() const noexcept
|
||||
{
|
||||
return {input_position, credit_position, reserve_credit};
|
||||
}
|
||||
|
||||
void set_reserve_state(const reserve_state& state) noexcept
|
||||
{
|
||||
input_position = state.position;
|
||||
credit_position = state.position_at_refill;
|
||||
reserve_credit = state.credit;
|
||||
}
|
||||
|
||||
protected:
|
||||
/*!
|
||||
@brief the number of elements to reserve for an array that announces @a len
|
||||
|
||||
The announced number is not backed by input when the array opens, and the
|
||||
arrays that are open at the same time each keep what was reserved for
|
||||
them, so what is reserved for all arrays together is bounded by the
|
||||
input read so far: @ref reserved_elements_base plus
|
||||
@ref reserved_elements_per_byte for every character. An array that
|
||||
announces more than what is left of that is given what is left, and
|
||||
grows as its elements arrive. Without the position of the input (the
|
||||
parser is not driven by a binary_reader that is running) nothing is
|
||||
bounded but the cap of reserve_array().
|
||||
|
||||
The common case, an array that announces no more than the credit left,
|
||||
costs a comparison and a subtraction. It is charged what it announces, which
|
||||
for an array of more than reserve_cap() elements is more than reserve_array()
|
||||
reserves for it, which its own elements refill as they are read.
|
||||
*/
|
||||
std::size_t reserve_count(const std::size_t len) noexcept
|
||||
{
|
||||
if (JSON_HEDLEY_LIKELY(len <= reserve_credit))
|
||||
{
|
||||
reserve_credit -= len;
|
||||
return len;
|
||||
}
|
||||
return reserve_count_slowly(len);
|
||||
}
|
||||
|
||||
private:
|
||||
JSON_HEDLEY_NEVER_INLINE
|
||||
std::size_t reserve_count_slowly(const std::size_t len) noexcept
|
||||
{
|
||||
constexpr std::size_t max_size = (std::numeric_limits<std::size_t>::max)();
|
||||
if (input_position == nullptr)
|
||||
{
|
||||
reserve_credit = max_size;
|
||||
return len;
|
||||
}
|
||||
|
||||
// refill the credit with 'reserved_elements_per_byte' for every
|
||||
// character read since the last refill; it saturates instead of
|
||||
// overflowing when size_t is 32 bits wide and a very large input has
|
||||
// been read (the position goes back by up to two in BON8, which has
|
||||
// no sizes)
|
||||
const std::size_t position = *input_position;
|
||||
if (position > credit_position)
|
||||
{
|
||||
const std::size_t bytes = position - credit_position;
|
||||
reserve_credit = bytes <= (max_size - reserve_credit) / reserved_elements_per_byte
|
||||
? reserve_credit + (reserved_elements_per_byte * bytes)
|
||||
: max_size;
|
||||
credit_position = position;
|
||||
}
|
||||
|
||||
std::size_t count = (std::min)(len, reserve_cap());
|
||||
count = (std::min)(count, reserve_credit);
|
||||
reserve_credit -= count;
|
||||
return count;
|
||||
}
|
||||
|
||||
/// the number of characters the input has provided so far, if known
|
||||
const std::size_t* input_position = nullptr;
|
||||
/// the number of characters the input had provided when the credit was refilled
|
||||
std::size_t credit_position = 0;
|
||||
/// the number of elements that may still be reserved
|
||||
std::size_t reserve_credit = reserved_elements_base;
|
||||
/// whether a syntax error occurred
|
||||
bool errored = false;
|
||||
/// whether to throw exceptions in case of errors
|
||||
@@ -1346,7 +1210,7 @@ class json_sax_dom_parser : public json_sax_dom_error_state
|
||||
|
||||
if (len != detail::unknown_size())
|
||||
{
|
||||
reserve_array(*ref->m_data.m_value.array, reserve_count(len), priority_tag<1> {});
|
||||
reserve_array(*ref->m_data.m_value.array, len, priority_tag<1> {});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1694,7 +1558,7 @@ class json_sax_dom_binary_parser : public json_sax_dom_error_state
|
||||
|
||||
if (len != detail::unknown_size())
|
||||
{
|
||||
reserve_array(*ref_stack.back()->m_data.m_value.array, reserve_count(len), priority_tag<1> {});
|
||||
reserve_array(*ref_stack.back()->m_data.m_value.array, len, priority_tag<1> {});
|
||||
}
|
||||
|
||||
return true;
|
||||
@@ -1986,7 +1850,7 @@ class json_sax_dom_callback_parser : public json_sax_dom_error_state
|
||||
|
||||
if (len != detail::unknown_size())
|
||||
{
|
||||
reserve_array(*ref_stack.back()->m_data.m_value.array, reserve_count(len), priority_tag<1> {});
|
||||
reserve_array(*ref_stack.back()->m_data.m_value.array, len, priority_tag<1> {});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -224,6 +224,12 @@
|
||||
#define JSON_NO_THREAD_LOCAL 1
|
||||
#endif
|
||||
|
||||
// JSON_RECURSION_DEPTH_LIMIT (the number of nesting levels the walks over a
|
||||
// value recurse into before they continue with an explicit stack; 128 by
|
||||
// default, 0 to 254 allowed) is defined and checked where it is used, in
|
||||
// detail/recursion_depth_limit.hpp. Like JSON_NO_THREAD_LOCAL it is neither part
|
||||
// of the ABI tag nor undefined at the end of json.hpp.
|
||||
|
||||
// allow disabling exceptions
|
||||
#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION)
|
||||
#define JSON_THROW(exception) throw exception
|
||||
@@ -917,14 +923,3 @@
|
||||
#ifndef JSON_DISABLE_TUPLE_REFERENCE_CONVERSION
|
||||
#define JSON_DISABLE_TUPLE_REFERENCE_CONVERSION 0
|
||||
#endif
|
||||
|
||||
// JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE: how many elements of optimized
|
||||
// UBJSON containers of type Z, T or F (which have no payload) a reader accepts
|
||||
// per byte of input, on top of a fixed base. The value is checked and converted
|
||||
// in binary_reader.hpp; the macro is documented in
|
||||
// docs/mkdocs/docs/api/macros/json_ubjson_valueless_elements_per_byte.md.
|
||||
// It is not part of the ABI tag: it must have the same value in every
|
||||
// translation unit of a program.
|
||||
#ifndef JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE
|
||||
#define JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE 4
|
||||
#endif
|
||||
@@ -60,7 +60,6 @@
|
||||
#undef JSON_USE_OBJECTS_FOR_ENUM_KEYED_MAPS
|
||||
#undef JSON_DISABLE_STREAM_BULK_SCAN
|
||||
#undef JSON_DISABLE_UNLOCKED_FILE_IO
|
||||
#undef JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE
|
||||
#endif
|
||||
|
||||
#include <nlohmann/thirdparty/hedley/hedley_undef.hpp>
|
||||
@@ -1204,29 +1204,20 @@ class binary_writer
|
||||
|
||||
// an optimized array of a valueless type carries no payload, so a
|
||||
// reader has nothing but the declared count to bound the allocation
|
||||
// by. It refuses one array of more than max_valueless_container_size
|
||||
// elements, and all such arrays of one input together when they add
|
||||
// up to more than valueless_elements_base plus
|
||||
// JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE per byte read. Write the
|
||||
// unoptimized form, at one byte per element, for an array that
|
||||
// would exceed the base with the arrays written so far, so that the
|
||||
// result can be read back: the limit of a reader is never below the
|
||||
// base, whatever the macro is set to, so the writer does not use
|
||||
// it (and a reader with a larger rate reads this output as well).
|
||||
// by and refuses an excessive one. Write the unoptimized form for
|
||||
// those, at one byte per element, so the result can be read back.
|
||||
// Objects are not affected: every element is preceded by its key.
|
||||
if (same_prefix && !(use_bjdata && is_bjdata_excluded_type_marker(first_prefix)))
|
||||
const bool valueless_type = (first_prefix == 'Z' || first_prefix == 'T' || first_prefix == 'F');
|
||||
const bool excessive_valueless = IsArray
|
||||
&& valueless_type
|
||||
&& size > detail::max_valueless_container_size;
|
||||
|
||||
if (same_prefix && !excessive_valueless
|
||||
&& !(use_bjdata && is_bjdata_excluded_type_marker(first_prefix)))
|
||||
{
|
||||
const bool valueless_type = IsArray && (first_prefix == 'Z' || first_prefix == 'T' || first_prefix == 'F');
|
||||
if (!valueless_type || size <= detail::valueless_elements_base - valueless_elements_written)
|
||||
{
|
||||
prefix_required = false;
|
||||
oa.write_character(to_char_type('$'));
|
||||
oa.write_character(first_prefix);
|
||||
if (valueless_type)
|
||||
{
|
||||
valueless_elements_written += size;
|
||||
}
|
||||
}
|
||||
prefix_required = false;
|
||||
oa.write_character(to_char_type('$'));
|
||||
oa.write_character(first_prefix);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5450,10 +5441,6 @@ class binary_writer
|
||||
/// how to treat a string value or object key that is not valid UTF-8
|
||||
/// (CBOR, MessagePack, UBJSON, BJData, and BSON; not BON8)
|
||||
const error_handler_t error_handler = binary_writer_default_error_handler();
|
||||
|
||||
/// UBJSON: the number of elements written so far in optimized arrays of a
|
||||
/// valueless type, which a reader adds up and bounds
|
||||
std::size_t valueless_elements_written = 0;
|
||||
};
|
||||
|
||||
} // namespace detail
|
||||
|
||||
@@ -9,9 +9,22 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstddef> // size_t
|
||||
#include <type_traits> // is_integral, is_same
|
||||
|
||||
#include <nlohmann/detail/abi_macros.hpp>
|
||||
|
||||
// JSON_RECURSION_DEPTH_LIMIT: the number of nesting levels the operations that
|
||||
// walk a value recurse into on the call stack before they continue with an
|
||||
// explicit stack (see detail::recursion_depth_limit() below).
|
||||
//
|
||||
// May be defined by the user before the first include of the library; it has
|
||||
// to be the same in every translation unit of a program (it is not part of the
|
||||
// ABI tag, see https://json.nlohmann.me/api/macros/json_recursion_depth_limit/).
|
||||
// Allowed: an integer constant expression from 0 to 254. The default is 128.
|
||||
#ifndef JSON_RECURSION_DEPTH_LIMIT
|
||||
#define JSON_RECURSION_DEPTH_LIMIT 128
|
||||
#endif
|
||||
|
||||
NLOHMANN_JSON_NAMESPACE_BEGIN
|
||||
namespace detail
|
||||
{
|
||||
@@ -20,21 +33,59 @@ namespace detail
|
||||
@brief the number of nesting levels an operation recurses into
|
||||
|
||||
Operations that walk a value (copying, comparing, serializing, hashing, merging,
|
||||
...) recurse once
|
||||
per nesting level, which is fastest, but a value nested deeply enough would
|
||||
exhaust the call stack. So they recurse only this many levels deep and finish
|
||||
whatever lies below with an explicit stack. All of them share this limit.
|
||||
diffing, converting, destroying, and writing the binary formats CBOR,
|
||||
MessagePack, UBJSON, BJData, and BON8) recurse once per nesting level, which is
|
||||
fastest, but a value nested deeply enough would exhaust the call stack. So they
|
||||
recurse only this many levels deep and finish whatever lies below with an
|
||||
explicit stack that lives on the heap. All of them share this limit and none has
|
||||
a constant of its own. It is the value of the macro JSON_RECURSION_DEPTH_LIMIT,
|
||||
128 by default. (Parsing, the binary readers, flatten(), unflatten(), and
|
||||
to_bson() always use an explicit stack and do not read it, and the lockstep
|
||||
comparison of json_view documents has a fixed bound of its own, 48 levels.)
|
||||
|
||||
Most of them pass the depth down as an argument. The copy constructor and the
|
||||
comparison operators cannot, as their signatures are fixed, so they count it
|
||||
in basic_json::nesting_depth() instead, a byte per thread; the limit must
|
||||
therefore stay below 255.
|
||||
Most of the operations pass the depth down as an argument. The copy constructor
|
||||
and the comparison operators cannot, as their signatures are fixed, so they
|
||||
count it in basic_json::nesting_depth() instead, a byte per thread. Only the
|
||||
levels that are descended into are counted, so the count never exceeds the limit.
|
||||
The limit must fit in the byte with some headroom, so that the count cannot wrap
|
||||
around: it must be below 255, and 254 is the largest value allowed. (More levels
|
||||
would hardly be useful: 254 levels need up to 576 KiB of stack in an unoptimized
|
||||
build already, twice the stack of the default.)
|
||||
|
||||
The smallest value is 0: every operation then uses its explicit stack from the
|
||||
first level on and no operation recurses over the nesting of a value at all.
|
||||
That is slower, but needs a constant amount of call stack. It works because
|
||||
every use of the limit compares a depth with it (`depth >= limit`, or `<` for
|
||||
the opposite), which holds for 0 as for any other value; nothing computes
|
||||
`limit - 1` or `limit + 1`, so nothing wraps around. Each operation's explicit
|
||||
stack handles any nesting level, including the first, so none of them relies on
|
||||
having recursed first.
|
||||
|
||||
The call stack that an operation needs is roughly a few KiB plus a cost per level
|
||||
times the limit. At the default of 128 levels, an unoptimized build needs up to
|
||||
about 290 KiB (`dump()` with Clang at -O0: about 2.3 KiB per level) and an
|
||||
optimized build up to about 48 KiB; with 254 levels it is up to 576 KiB and
|
||||
112 KiB, and with 0 it is 16 KiB or less. Lower the limit for threads with small
|
||||
stacks (RTOS tasks, fibers, musl threads), raise it to trade stack for speed on
|
||||
deeply nested values. All of this, with the measurements, is in the
|
||||
documentation of the macro.
|
||||
|
||||
@sa https://json.nlohmann.me/api/macros/json_recursion_depth_limit/
|
||||
@sa https://github.com/nlohmann/json/issues/5387
|
||||
*/
|
||||
constexpr std::size_t recursion_depth_limit() noexcept
|
||||
{
|
||||
return 128;
|
||||
// an integer: not a floating-point number, whose range check would pass for 1.5 or 254.0
|
||||
// (bool is an integer type, too, but 1 and 0 spelled true and false are no way to say a limit)
|
||||
static_assert(std::is_integral<decltype(JSON_RECURSION_DEPTH_LIMIT)>::value
|
||||
&& !std::is_same<decltype(JSON_RECURSION_DEPTH_LIMIT), bool>::value,
|
||||
"JSON_RECURSION_DEPTH_LIMIT must be an integer constant");
|
||||
static_assert(JSON_RECURSION_DEPTH_LIMIT >= 0 && JSON_RECURSION_DEPTH_LIMIT <= 254,
|
||||
"JSON_RECURSION_DEPTH_LIMIT must be between 0 and 254: the nesting depth is counted in a byte per thread "
|
||||
"and may exceed the limit by one (see https://json.nlohmann.me/api/macros/json_recursion_depth_limit/)");
|
||||
// (a value out of range has been reported by now; the remainder keeps it
|
||||
// below 255 for any value, so that basic_json does not report it again)
|
||||
return static_cast<std::size_t>(JSON_RECURSION_DEPTH_LIMIT) % 255;
|
||||
}
|
||||
|
||||
} // namespace detail
|
||||
|
||||
@@ -1256,8 +1256,12 @@ public:
|
||||
}
|
||||
|
||||
#ifndef JSON_NO_THREAD_LOCAL
|
||||
// nesting_depth() is a byte and may exceed the limit by one level
|
||||
static_assert(detail::recursion_depth_limit() < 255, "the nesting depth count must fit in a byte");
|
||||
// nesting_depth() is a byte that never exceeds the limit (JSON_RECURSION_DEPTH_LIMIT),
|
||||
// which is at most 254: the largest value stays one below the byte's maximum, so
|
||||
// that the count cannot wrap around to 0 and let a descent go on without a bound.
|
||||
// (recursion_depth_limit() checks the macro itself, with a message that
|
||||
// names it; this is the check that cannot be edited away unnoticed.)
|
||||
static_assert(detail::recursion_depth_limit() < 255, "the nesting depth count must fit in a byte: JSON_RECURSION_DEPTH_LIMIT must not exceed 254");
|
||||
|
||||
/*!
|
||||
@brief how many levels the operation going on in this thread has descended into
|
||||
@@ -1268,8 +1272,15 @@ public:
|
||||
user code nests them anyway, sharing the count only ends a descent sooner
|
||||
than it had to, which costs a little speed and is never wrong.
|
||||
|
||||
A byte is enough: the count never exceeds the limit by more than the single
|
||||
level that notices the limit has been reached.
|
||||
A byte is enough: only the levels that are descended into are counted
|
||||
(@ref nesting_depth_guard), so the count never exceeds the limit, and the
|
||||
limit is at most 254.
|
||||
|
||||
The count is per thread and not per value or per library configuration: all
|
||||
translation units of a program share it, even if they were compiled with
|
||||
different values of JSON_RECURSION_DEPTH_LIMIT (which is not supported, see
|
||||
the documentation of the macro); each of them then compares the shared count
|
||||
with its own limit.
|
||||
*/
|
||||
static std::uint8_t& nesting_depth() noexcept
|
||||
{
|
||||
@@ -1327,14 +1338,25 @@ public:
|
||||
#endif
|
||||
{
|
||||
#ifndef JSON_NO_THREAD_LOCAL
|
||||
++nesting_depth();
|
||||
// Only a level that is entered is counted, so that the count never
|
||||
// exceeds the limit. A level that is not entered is finished without
|
||||
// descending, and whatever runs while it is (a copy made by a custom
|
||||
// base class, say) sees the same count as the level itself did and
|
||||
// is not entered either.
|
||||
if (m_okay)
|
||||
{
|
||||
++nesting_depth();
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
~nesting_depth_guard()
|
||||
{
|
||||
#ifndef JSON_NO_THREAD_LOCAL
|
||||
--nesting_depth();
|
||||
if (m_okay)
|
||||
{
|
||||
--nesting_depth();
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -5246,7 +5268,10 @@ public:
|
||||
*/
|
||||
void update_members(const iterator& first, const iterator& last, const bool merge_objects, const std::size_t depth)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(depth >= detail::recursion_depth_limit()))
|
||||
// without merge_objects the loop below never descends, so there is no
|
||||
// call stack to save (and the explicit stack always merges); the
|
||||
// distinction matters for a limit of 0, which reaches this at depth 0
|
||||
if (JSON_HEDLEY_UNLIKELY(merge_objects && depth >= detail::recursion_depth_limit()))
|
||||
{
|
||||
update_members_iteratively(first, last);
|
||||
return;
|
||||
|
||||
Generated
+128
-415
@@ -2597,6 +2597,12 @@ JSON_HEDLEY_DIAGNOSTIC_POP
|
||||
#define JSON_NO_THREAD_LOCAL 1
|
||||
#endif
|
||||
|
||||
// JSON_RECURSION_DEPTH_LIMIT (the number of nesting levels the walks over a
|
||||
// value recurse into before they continue with an explicit stack; 128 by
|
||||
// default, 0 to 254 allowed) is defined and checked where it is used, in
|
||||
// detail/recursion_depth_limit.hpp. Like JSON_NO_THREAD_LOCAL it is neither part
|
||||
// of the ABI tag nor undefined at the end of json.hpp.
|
||||
|
||||
// allow disabling exceptions
|
||||
#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION)
|
||||
#define JSON_THROW(exception) throw exception
|
||||
@@ -3291,17 +3297,6 @@ JSON_HEDLEY_DIAGNOSTIC_POP
|
||||
#define JSON_DISABLE_TUPLE_REFERENCE_CONVERSION 0
|
||||
#endif
|
||||
|
||||
// JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE: how many elements of optimized
|
||||
// UBJSON containers of type Z, T or F (which have no payload) a reader accepts
|
||||
// per byte of input, on top of a fixed base. The value is checked and converted
|
||||
// in binary_reader.hpp; the macro is documented in
|
||||
// docs/mkdocs/docs/api/macros/json_ubjson_valueless_elements_per_byte.md.
|
||||
// It is not part of the ABI tag: it must have the same value in every
|
||||
// translation unit of a program.
|
||||
#ifndef JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE
|
||||
#define JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE 4
|
||||
#endif
|
||||
|
||||
#if JSON_HAS_THREE_WAY_COMPARISON
|
||||
#include <compare> // partial_ordering
|
||||
#endif
|
||||
@@ -8611,10 +8606,23 @@ NLOHMANN_JSON_NAMESPACE_END
|
||||
|
||||
|
||||
#include <cstddef> // size_t
|
||||
#include <type_traits> // is_integral, is_same
|
||||
|
||||
// #include <nlohmann/detail/abi_macros.hpp>
|
||||
|
||||
|
||||
// JSON_RECURSION_DEPTH_LIMIT: the number of nesting levels the operations that
|
||||
// walk a value recurse into on the call stack before they continue with an
|
||||
// explicit stack (see detail::recursion_depth_limit() below).
|
||||
//
|
||||
// May be defined by the user before the first include of the library; it has
|
||||
// to be the same in every translation unit of a program (it is not part of the
|
||||
// ABI tag, see https://json.nlohmann.me/api/macros/json_recursion_depth_limit/).
|
||||
// Allowed: an integer constant expression from 0 to 254. The default is 128.
|
||||
#ifndef JSON_RECURSION_DEPTH_LIMIT
|
||||
#define JSON_RECURSION_DEPTH_LIMIT 128
|
||||
#endif
|
||||
|
||||
NLOHMANN_JSON_NAMESPACE_BEGIN
|
||||
namespace detail
|
||||
{
|
||||
@@ -8623,21 +8631,59 @@ namespace detail
|
||||
@brief the number of nesting levels an operation recurses into
|
||||
|
||||
Operations that walk a value (copying, comparing, serializing, hashing, merging,
|
||||
...) recurse once
|
||||
per nesting level, which is fastest, but a value nested deeply enough would
|
||||
exhaust the call stack. So they recurse only this many levels deep and finish
|
||||
whatever lies below with an explicit stack. All of them share this limit.
|
||||
diffing, converting, destroying, and writing the binary formats CBOR,
|
||||
MessagePack, UBJSON, BJData, and BON8) recurse once per nesting level, which is
|
||||
fastest, but a value nested deeply enough would exhaust the call stack. So they
|
||||
recurse only this many levels deep and finish whatever lies below with an
|
||||
explicit stack that lives on the heap. All of them share this limit and none has
|
||||
a constant of its own. It is the value of the macro JSON_RECURSION_DEPTH_LIMIT,
|
||||
128 by default. (Parsing, the binary readers, flatten(), unflatten(), and
|
||||
to_bson() always use an explicit stack and do not read it, and the lockstep
|
||||
comparison of json_view documents has a fixed bound of its own, 48 levels.)
|
||||
|
||||
Most of them pass the depth down as an argument. The copy constructor and the
|
||||
comparison operators cannot, as their signatures are fixed, so they count it
|
||||
in basic_json::nesting_depth() instead, a byte per thread; the limit must
|
||||
therefore stay below 255.
|
||||
Most of the operations pass the depth down as an argument. The copy constructor
|
||||
and the comparison operators cannot, as their signatures are fixed, so they
|
||||
count it in basic_json::nesting_depth() instead, a byte per thread. Only the
|
||||
levels that are descended into are counted, so the count never exceeds the limit.
|
||||
The limit must fit in the byte with some headroom, so that the count cannot wrap
|
||||
around: it must be below 255, and 254 is the largest value allowed. (More levels
|
||||
would hardly be useful: 254 levels need up to 576 KiB of stack in an unoptimized
|
||||
build already, twice the stack of the default.)
|
||||
|
||||
The smallest value is 0: every operation then uses its explicit stack from the
|
||||
first level on and no operation recurses over the nesting of a value at all.
|
||||
That is slower, but needs a constant amount of call stack. It works because
|
||||
every use of the limit compares a depth with it (`depth >= limit`, or `<` for
|
||||
the opposite), which holds for 0 as for any other value; nothing computes
|
||||
`limit - 1` or `limit + 1`, so nothing wraps around. Each operation's explicit
|
||||
stack handles any nesting level, including the first, so none of them relies on
|
||||
having recursed first.
|
||||
|
||||
The call stack that an operation needs is roughly a few KiB plus a cost per level
|
||||
times the limit. At the default of 128 levels, an unoptimized build needs up to
|
||||
about 290 KiB (`dump()` with Clang at -O0: about 2.3 KiB per level) and an
|
||||
optimized build up to about 48 KiB; with 254 levels it is up to 576 KiB and
|
||||
112 KiB, and with 0 it is 16 KiB or less. Lower the limit for threads with small
|
||||
stacks (RTOS tasks, fibers, musl threads), raise it to trade stack for speed on
|
||||
deeply nested values. All of this, with the measurements, is in the
|
||||
documentation of the macro.
|
||||
|
||||
@sa https://json.nlohmann.me/api/macros/json_recursion_depth_limit/
|
||||
@sa https://github.com/nlohmann/json/issues/5387
|
||||
*/
|
||||
constexpr std::size_t recursion_depth_limit() noexcept
|
||||
{
|
||||
return 128;
|
||||
// an integer: not a floating-point number, whose range check would pass for 1.5 or 254.0
|
||||
// (bool is an integer type, too, but 1 and 0 spelled true and false are no way to say a limit)
|
||||
static_assert(std::is_integral<decltype(JSON_RECURSION_DEPTH_LIMIT)>::value
|
||||
&& !std::is_same<decltype(JSON_RECURSION_DEPTH_LIMIT), bool>::value,
|
||||
"JSON_RECURSION_DEPTH_LIMIT must be an integer constant");
|
||||
static_assert(JSON_RECURSION_DEPTH_LIMIT >= 0 && JSON_RECURSION_DEPTH_LIMIT <= 254,
|
||||
"JSON_RECURSION_DEPTH_LIMIT must be between 0 and 254: the nesting depth is counted in a byte per thread "
|
||||
"and may exceed the limit by one (see https://json.nlohmann.me/api/macros/json_recursion_depth_limit/)");
|
||||
// (a value out of range has been reported by now; the remainder keeps it
|
||||
// below 255 for any value, so that basic_json does not report it again)
|
||||
return static_cast<std::size_t>(JSON_RECURSION_DEPTH_LIMIT) % 255;
|
||||
}
|
||||
|
||||
} // namespace detail
|
||||
@@ -8877,7 +8923,6 @@ NLOHMANN_JSON_NAMESPACE_END
|
||||
#include <iterator> // back_inserter
|
||||
#include <limits> // numeric_limits
|
||||
#include <string> // char_traits, string
|
||||
#include <type_traits> // false_type, integral_constant, is_convertible, is_integral, is_same, remove_cv, remove_reference, true_type
|
||||
#include <utility> // make_pair, move
|
||||
#include <vector> // vector
|
||||
#ifdef __cpp_lib_byteswap
|
||||
@@ -16068,40 +16113,6 @@ constexpr std::size_t unknown_size()
|
||||
return (std::numeric_limits<std::size_t>::max)();
|
||||
}
|
||||
|
||||
/// the largest number of elements the DOM parsers reserve for one array upfront
|
||||
constexpr std::size_t reserve_cap()
|
||||
{
|
||||
return 16384;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief number of array elements a DOM parser may reserve upfront on top of
|
||||
@ref reserved_elements_per_byte
|
||||
|
||||
An array that announces its size is given storage for that many elements when
|
||||
it opens (at most @ref reserve_cap of them), before any input backs the
|
||||
announcement, and the arrays that are open at the same time keep their storage
|
||||
until the input is complete: 100000 nested headers of three bytes each that
|
||||
announce 16384 elements would otherwise need 1.7 GB. The elements reserved for
|
||||
all arrays of an input are therefore added up, and the sum must stay below
|
||||
this base plus @ref reserved_elements_per_byte for every byte read so far; an
|
||||
array that would go beyond reserves what is left, and grows as its elements
|
||||
arrive. The base is four arrays of the maximum size, so the first nested
|
||||
arrays of an input are not affected.
|
||||
*/
|
||||
JSON_INLINE_VARIABLE constexpr std::size_t reserved_elements_base = 4 * reserve_cap();
|
||||
|
||||
/*!
|
||||
@brief number of array elements a DOM parser may reserve upfront per byte of
|
||||
input, in addition to @ref reserved_elements_base
|
||||
|
||||
An element of an array takes at least one byte of input, so an array that
|
||||
completes has used what was reserved for it. At a rate of 2 elements per
|
||||
byte, an array that announces more than the input provides keeps at most
|
||||
32 bytes of memory per byte of input, plus the base, until the parse fails.
|
||||
*/
|
||||
JSON_INLINE_VARIABLE constexpr std::size_t reserved_elements_per_byte = 2;
|
||||
|
||||
/*!
|
||||
@brief reserve capacity for @a len elements in array @a arr
|
||||
|
||||
@@ -16117,7 +16128,8 @@ template<typename ArrayType>
|
||||
auto reserve_array(ArrayType& arr, std::size_t len, priority_tag<1> /*unused*/)
|
||||
-> decltype(arr.reserve(len), void())
|
||||
{
|
||||
arr.reserve((std::min)(len, reserve_cap()));
|
||||
constexpr std::size_t reserve_cap = 16384;
|
||||
arr.reserve((std::min)(len, reserve_cap));
|
||||
}
|
||||
|
||||
template<typename ArrayType>
|
||||
@@ -16284,13 +16296,6 @@ struct diagnostic_positions
|
||||
Holds whether a parse error occurred and whether errors are thrown, and
|
||||
implements the parse_error() and is_errored() members of the SAX interface for
|
||||
json_sax_dom_parser and json_sax_dom_callback_parser.
|
||||
|
||||
It also bounds what the parsers reserve for arrays that announce their size
|
||||
(reserve_count()): the announced size is not backed by input yet when the
|
||||
array opens, and the arrays that are open at the same time each keep what was
|
||||
reserved for them, so the binary_reader tells the parser where to find the
|
||||
number of characters it has read (set_input_position()), which the bound
|
||||
is derived from.
|
||||
*/
|
||||
class json_sax_dom_error_state
|
||||
{
|
||||
@@ -16324,103 +16329,7 @@ class json_sax_dom_error_state
|
||||
return errored;
|
||||
}
|
||||
|
||||
/// bind the number of characters the input has provided so far; see
|
||||
/// @ref reserve_count
|
||||
/// (nullptr: not known, which bounds only the size of each array)
|
||||
void set_input_position(const std::size_t* position) noexcept
|
||||
{
|
||||
input_position = position;
|
||||
credit_position = 0;
|
||||
reserve_credit = position != nullptr ? reserved_elements_base : (std::numeric_limits<std::size_t>::max)();
|
||||
}
|
||||
|
||||
/// what set_input_position() changes, so that a binding can be undone
|
||||
struct reserve_state
|
||||
{
|
||||
const std::size_t* position;
|
||||
std::size_t position_at_refill;
|
||||
std::size_t credit;
|
||||
};
|
||||
|
||||
reserve_state get_reserve_state() const noexcept
|
||||
{
|
||||
return {input_position, credit_position, reserve_credit};
|
||||
}
|
||||
|
||||
void set_reserve_state(const reserve_state& state) noexcept
|
||||
{
|
||||
input_position = state.position;
|
||||
credit_position = state.position_at_refill;
|
||||
reserve_credit = state.credit;
|
||||
}
|
||||
|
||||
protected:
|
||||
/*!
|
||||
@brief the number of elements to reserve for an array that announces @a len
|
||||
|
||||
The announced number is not backed by input when the array opens, and the
|
||||
arrays that are open at the same time each keep what was reserved for
|
||||
them, so what is reserved for all arrays together is bounded by the
|
||||
input read so far: @ref reserved_elements_base plus
|
||||
@ref reserved_elements_per_byte for every character. An array that
|
||||
announces more than what is left of that is given what is left, and
|
||||
grows as its elements arrive. Without the position of the input (the
|
||||
parser is not driven by a binary_reader that is running) nothing is
|
||||
bounded but the cap of reserve_array().
|
||||
|
||||
The common case, an array that announces no more than the credit left,
|
||||
costs a comparison and a subtraction. It is charged what it announces, which
|
||||
for an array of more than reserve_cap() elements is more than reserve_array()
|
||||
reserves for it, which its own elements refill as they are read.
|
||||
*/
|
||||
std::size_t reserve_count(const std::size_t len) noexcept
|
||||
{
|
||||
if (JSON_HEDLEY_LIKELY(len <= reserve_credit))
|
||||
{
|
||||
reserve_credit -= len;
|
||||
return len;
|
||||
}
|
||||
return reserve_count_slowly(len);
|
||||
}
|
||||
|
||||
private:
|
||||
JSON_HEDLEY_NEVER_INLINE
|
||||
std::size_t reserve_count_slowly(const std::size_t len) noexcept
|
||||
{
|
||||
constexpr std::size_t max_size = (std::numeric_limits<std::size_t>::max)();
|
||||
if (input_position == nullptr)
|
||||
{
|
||||
reserve_credit = max_size;
|
||||
return len;
|
||||
}
|
||||
|
||||
// refill the credit with 'reserved_elements_per_byte' for every
|
||||
// character read since the last refill; it saturates instead of
|
||||
// overflowing when size_t is 32 bits wide and a very large input has
|
||||
// been read (the position goes back by up to two in BON8, which has
|
||||
// no sizes)
|
||||
const std::size_t position = *input_position;
|
||||
if (position > credit_position)
|
||||
{
|
||||
const std::size_t bytes = position - credit_position;
|
||||
reserve_credit = bytes <= (max_size - reserve_credit) / reserved_elements_per_byte
|
||||
? reserve_credit + (reserved_elements_per_byte * bytes)
|
||||
: max_size;
|
||||
credit_position = position;
|
||||
}
|
||||
|
||||
std::size_t count = (std::min)(len, reserve_cap());
|
||||
count = (std::min)(count, reserve_credit);
|
||||
reserve_credit -= count;
|
||||
return count;
|
||||
}
|
||||
|
||||
/// the number of characters the input has provided so far, if known
|
||||
const std::size_t* input_position = nullptr;
|
||||
/// the number of characters the input had provided when the credit was refilled
|
||||
std::size_t credit_position = 0;
|
||||
/// the number of elements that may still be reserved
|
||||
std::size_t reserve_credit = reserved_elements_base;
|
||||
/// whether a syntax error occurred
|
||||
bool errored = false;
|
||||
/// whether to throw exceptions in case of errors
|
||||
@@ -17255,7 +17164,7 @@ class json_sax_dom_parser : public json_sax_dom_error_state
|
||||
|
||||
if (len != detail::unknown_size())
|
||||
{
|
||||
reserve_array(*ref->m_data.m_value.array, reserve_count(len), priority_tag<1> {});
|
||||
reserve_array(*ref->m_data.m_value.array, len, priority_tag<1> {});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17603,7 +17512,7 @@ class json_sax_dom_binary_parser : public json_sax_dom_error_state
|
||||
|
||||
if (len != detail::unknown_size())
|
||||
{
|
||||
reserve_array(*ref_stack.back()->m_data.m_value.array, reserve_count(len), priority_tag<1> {});
|
||||
reserve_array(*ref_stack.back()->m_data.m_value.array, len, priority_tag<1> {});
|
||||
}
|
||||
|
||||
return true;
|
||||
@@ -17895,7 +17804,7 @@ class json_sax_dom_callback_parser : public json_sax_dom_error_state
|
||||
|
||||
if (len != detail::unknown_size())
|
||||
{
|
||||
reserve_array(*ref_stack.back()->m_data.m_value.array, reserve_count(len), priority_tag<1> {});
|
||||
reserve_array(*ref_stack.back()->m_data.m_value.array, len, priority_tag<1> {});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18500,124 +18409,13 @@ the input.
|
||||
|
||||
This is a sanity bound rather than a security boundary, and it is far above
|
||||
any container met in practice. @ref binary_writer falls back to the
|
||||
unoptimized encoding for longer containers, and for containers that together
|
||||
with the ones before them would exceed @ref valueless_elements_base, so that a
|
||||
value serialized by this library can always be read back.
|
||||
unoptimized encoding for longer containers, so that a value serialized by
|
||||
this library can always be read back.
|
||||
|
||||
@sa https://github.com/nlohmann/json/issues/2793
|
||||
*/
|
||||
JSON_INLINE_VARIABLE constexpr std::size_t max_valueless_container_size = 1 << 20;
|
||||
|
||||
/*!
|
||||
@brief number of elements of valueless UBJSON containers accepted on top of
|
||||
@ref valueless_elements_per_byte
|
||||
|
||||
The limit @ref max_valueless_container_size applies to each container on its
|
||||
own, so on its own it lets 9 bytes announce 1048576 values, and every further
|
||||
8 bytes (a row of an optimized array of arrays) another 1048576. The elements of all containers of a valueless type
|
||||
in one input are therefore added up, and the sum must stay below this base
|
||||
plus @ref valueless_elements_per_byte for every byte read so far. The base is
|
||||
the per-container limit, so a container that was accepted on its own before
|
||||
the sum was introduced is still accepted at the start of an input.
|
||||
*/
|
||||
JSON_INLINE_VARIABLE constexpr std::size_t valueless_elements_base = max_valueless_container_size;
|
||||
|
||||
/// the value of JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE as a std::size_t:
|
||||
/// saturated at the largest one, not truncated, for a value that is wider
|
||||
template<typename T>
|
||||
constexpr std::size_t valueless_elements_rate(const T value) noexcept
|
||||
{
|
||||
return static_cast<std::uintmax_t>(value) > (std::numeric_limits<std::size_t>::max)()
|
||||
? (std::numeric_limits<std::size_t>::max)()
|
||||
: static_cast<std::size_t>(value);
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief number of elements of valueless UBJSON containers accepted per byte of
|
||||
input, in addition to @ref valueless_elements_base
|
||||
|
||||
This is the value of the macro JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE (4 by
|
||||
default), see docs/mkdocs/docs/api/macros/json_ubjson_valueless_elements_per_byte.md.
|
||||
|
||||
An element of a value that has a payload costs at least one byte of input, but
|
||||
an element of a valueless type costs none, so these are the only elements whose
|
||||
number the input does not bound. They are accepted at this rate per byte read,
|
||||
which bounds the memory they take in a DOM: a value takes 16 bytes there, so
|
||||
the arrays at the limit take about 20 times this rate bytes of memory per byte
|
||||
of input (16 for the value, the rest is allocator rounding and the arrays
|
||||
themselves; measured on macOS, DOM, with containers of 9 times the rate
|
||||
elements: 25 bytes per byte at a rate of 1, 81 at 4, 312 at 16, 1226 at 64 and
|
||||
5493 at 256), plus about 33 MB for the first elements of the base (16 bytes for
|
||||
each of the 1048576 elements, and a vector that grows by reallocation, as only
|
||||
16384 elements are reserved upfront, needs the old and the new storage at once).
|
||||
|
||||
The bound is meant for input written by other encoders and by versions before
|
||||
3.13.0, which wrote long arrays of this type optimized without a limit: input
|
||||
written by @ref binary_writer never needs more than the base, because the
|
||||
writer falls back to the unoptimized form once the arrays written so far and
|
||||
the next one add up to more than that, whatever the rate is.
|
||||
|
||||
Range: a positive integer. A value that is not one is a compile-time error (a
|
||||
reader with a rate of 0 would accept nothing beyond the base, and a negative or
|
||||
fractional rate has no meaning). A value that does not fit into std::size_t is
|
||||
used as the largest std::size_t: the rate is so high then that the bound is
|
||||
out of reach anyway, and a project can use one spelling (for example
|
||||
`0xFFFFFFFFFFFFFFFFu`) for 32 and 64 bit targets. The type of the value must
|
||||
be an integer type that is not wider than std::uintmax_t and not bool or a
|
||||
character type. A rate of at least 2^20 / 8 = 131072 effectively switches the
|
||||
whole-input bound off: an array of more than 32767 elements needs a count of
|
||||
type `l`, so its shortest header is `$Z#l` and four bytes (8 bytes) inside an
|
||||
optimized array of arrays (9 bytes, `[$Z#l`, at the top level), and a shorter
|
||||
header (`$Z#I` and two bytes) announces at most 32767. Every container thus has
|
||||
at most (rate * header bytes) elements, which never exceeds the limit. With a
|
||||
rate of 131071 an input that is refused exists (rows of 1048576 elements gain
|
||||
8 elements each); the per-container limit still applies.
|
||||
|
||||
The value must be the same in every translation unit of a program (it is
|
||||
compiled into member functions of binary_reader, which are inline), and it is
|
||||
not part of the ABI tag of the namespace.
|
||||
*/
|
||||
JSON_INLINE_VARIABLE constexpr std::size_t valueless_elements_per_byte =
|
||||
// a value that does not fit into std::size_t is the largest one (see above);
|
||||
// the comparison is made in the widest unsigned type, so that nothing is
|
||||
// truncated before it is made. A function template, so that the casts are
|
||||
// not "useless" (-Wuseless-cast) where the macro already has the type
|
||||
valueless_elements_rate((JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE));
|
||||
|
||||
/// the type of the macro's value is a character or boolean type
|
||||
template<typename T>
|
||||
struct valueless_elements_rate_is_char_or_bool : std::integral_constant < bool,
|
||||
std::is_same<T, bool>::value || std::is_same<T, char>::value || std::is_same<T, signed char>::value
|
||||
|| std::is_same<T, unsigned char>::value || std::is_same<T, wchar_t>::value
|
||||
|| std::is_same<T, char16_t>::value || std::is_same<T, char32_t>::value >
|
||||
{};
|
||||
|
||||
// these messages are what a user sees for a wrong value of the macro. The
|
||||
// integer test comes first because a fractional value would also fail the
|
||||
// positivity test; a value wider than std::uintmax_t would be truncated by
|
||||
// the conversion above, so it is rejected
|
||||
static_assert(std::is_integral<typename std::remove_reference<decltype((JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE))>::type>::value
|
||||
&& !valueless_elements_rate_is_char_or_bool<typename std::remove_cv<typename std::remove_reference<decltype((JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE))>::type>::type>::value,
|
||||
"JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE must be an integer (not bool or a character type)");
|
||||
static_assert(sizeof(decltype((JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE))) <= sizeof(std::uintmax_t),
|
||||
"JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE must not be wider than std::uintmax_t");
|
||||
static_assert((JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE) > 0,
|
||||
"JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE must be a positive integer");
|
||||
|
||||
/*!
|
||||
@brief the number of bytes read up to which
|
||||
@ref valueless_elements_base plus @ref valueless_elements_per_byte times
|
||||
that number can be computed in std::size_t
|
||||
|
||||
For more bytes than this the limit of the valueless elements is the largest
|
||||
std::size_t, which is saturation: the sum is not computed, because it would wrap
|
||||
around, which happens for a rate that is large (the largest std::size_t as the
|
||||
rate wraps already at the first byte) or an input that is large (more than
|
||||
about a billion bytes for the default rate with a 32-bit std::size_t). It is a
|
||||
constant of the build, so no division is made while reading.
|
||||
*/
|
||||
JSON_INLINE_VARIABLE constexpr std::size_t valueless_elements_saturation_chars =
|
||||
((std::numeric_limits<std::size_t>::max)() - valueless_elements_base) / (valueless_elements_per_byte == 0 ? 1 : valueless_elements_per_byte);
|
||||
///////////////////
|
||||
// binary reader //
|
||||
///////////////////
|
||||
@@ -18685,8 +18483,6 @@ class binary_reader
|
||||
sax = sax_;
|
||||
container_stack.clear();
|
||||
bon8_pushback_size = 0;
|
||||
// the position is only known to the DOM parser while this reader lives
|
||||
const position_binding binding(bound_parser(sax, std::is_convertible<SAX*, json_sax_dom_error_state*> {}), &chars_read);
|
||||
bool result = false;
|
||||
|
||||
switch (input_format)
|
||||
@@ -18744,61 +18540,6 @@ class binary_reader
|
||||
}
|
||||
|
||||
private:
|
||||
/// the DOM parser of @a s whose reserved storage is bounded by the input
|
||||
/// read so far; nullptr for any other SAX parser, which does not reserve,
|
||||
/// and for one that derives from a DOM parser privately or protectedly
|
||||
/// (or ambiguously), which the reader cannot reach
|
||||
static json_sax_dom_error_state* bound_parser(SAX* s, std::true_type /*is a DOM parser*/) noexcept
|
||||
{
|
||||
return s;
|
||||
}
|
||||
|
||||
static json_sax_dom_error_state* bound_parser(SAX* /*s*/, std::false_type /*is a DOM parser*/) noexcept
|
||||
{
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief tells a DOM parser where to find the number of characters read
|
||||
while sax_parse() runs, and puts back what it knew before when
|
||||
sax_parse() returns or throws
|
||||
|
||||
The reader is a local of the caller and is gone then, so the parser must
|
||||
not keep its position. A parser that was bound by a reader that is still
|
||||
running (a parse inside a parse) gets that binding back; that reader is
|
||||
still alive, as it encloses this one.
|
||||
*/
|
||||
class position_binding
|
||||
{
|
||||
public:
|
||||
position_binding(json_sax_dom_error_state* parser_, const std::size_t* position) noexcept
|
||||
: parser(parser_), saved()
|
||||
{
|
||||
if (parser != nullptr)
|
||||
{
|
||||
saved = parser->get_reserve_state();
|
||||
parser->set_input_position(position);
|
||||
}
|
||||
}
|
||||
|
||||
position_binding(const position_binding&) = delete;
|
||||
position_binding(position_binding&&) = delete;
|
||||
position_binding& operator=(const position_binding&) = delete;
|
||||
position_binding& operator=(position_binding&&) = delete;
|
||||
|
||||
~position_binding()
|
||||
{
|
||||
if (parser != nullptr)
|
||||
{
|
||||
parser->set_reserve_state(saved);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
json_sax_dom_error_state* parser;
|
||||
json_sax_dom_error_state::reserve_state saved;
|
||||
};
|
||||
|
||||
////////////////////////
|
||||
// nested containers //
|
||||
////////////////////////
|
||||
@@ -22295,38 +22036,6 @@ class binary_reader
|
||||
return emit_unsigned(number);
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief add the elements of a container of a valueless type to the total of
|
||||
the input and check the total against the input read so far
|
||||
|
||||
A container of type 'Z', 'T' or 'F' has no payload, so its elements are the
|
||||
only ones for which the input does not bound the count. The check is made
|
||||
once per container, not per element.
|
||||
|
||||
@param[in] count number of elements the container declares; at most
|
||||
@ref max_valueless_container_size
|
||||
@return false if the total is more than @ref valueless_elements_base plus
|
||||
@ref valueless_elements_per_byte for every byte read so far; the
|
||||
total is not changed then
|
||||
*/
|
||||
bool charge_valueless_elements(const std::size_t count) noexcept
|
||||
{
|
||||
// limit = base + rate * chars_read, saturating at the largest size_t
|
||||
// instead of wrapping around, which would make a huge rate or input
|
||||
// reject everything. valueless_elements never exceeds the limit of an
|
||||
// earlier call, and the limit does not decrease, so the difference
|
||||
// below cannot underflow.
|
||||
const std::size_t limit = chars_read <= valueless_elements_saturation_chars
|
||||
? valueless_elements_base + (valueless_elements_per_byte * chars_read)
|
||||
: (std::numeric_limits<std::size_t>::max)();
|
||||
if (JSON_HEDLEY_UNLIKELY(count > limit - valueless_elements))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
valueless_elements += count;
|
||||
return true;
|
||||
}
|
||||
|
||||
/*!
|
||||
@return whether array creation completed
|
||||
*/
|
||||
@@ -22412,27 +22121,24 @@ class binary_reader
|
||||
// declared count alone decides how much is allocated; the check is
|
||||
// made before the start event so that no container is opened that
|
||||
// is then abandoned. See @ref max_valueless_container_size.
|
||||
if (JSON_HEDLEY_UNLIKELY(size_and_type.second == 'Z' || size_and_type.second == 'T' || size_and_type.second == 'F'))
|
||||
if (JSON_HEDLEY_UNLIKELY((size_and_type.second == 'Z' || size_and_type.second == 'T' || size_and_type.second == 'F')
|
||||
&& size_and_type.first > max_valueless_container_size))
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(size_and_type.first > max_valueless_container_size))
|
||||
{
|
||||
return out_of_range_error(exception_id::container_too_large, "excessive array size", "size");
|
||||
}
|
||||
if (JSON_HEDLEY_UNLIKELY(!charge_valueless_elements(size_and_type.first)))
|
||||
{
|
||||
return out_of_range_error(exception_id::container_too_large, "excessive number of elements without payload", "size");
|
||||
}
|
||||
return out_of_range_error(exception_id::container_too_large, "excessive array size", "size");
|
||||
}
|
||||
|
||||
// a no-op is not a value, so an array of them is empty whatever
|
||||
// count it declares: the SAX parser is told so, instead of being
|
||||
// made to reserve storage for a count that no input backs
|
||||
const std::size_t announced = size_and_type.second == 'N' ? 0 : size_and_type.first;
|
||||
if (JSON_HEDLEY_UNLIKELY(!enter_array(announced, size_and_type.second)))
|
||||
if (JSON_HEDLEY_UNLIKELY(!enter_array(size_and_type.first, size_and_type.second)))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (size_and_type.second == 'N')
|
||||
{
|
||||
// a no-op is not a value, so a container of them holds none;
|
||||
// the declared size has already been passed to the SAX parser
|
||||
container_stack.back().remaining = 0;
|
||||
}
|
||||
|
||||
return size_and_type.second == 0
|
||||
|| parse_ubjson_typed_run(container_stack.back().remaining, size_and_type.second, std::integral_constant<bool, bulk_scan> {});
|
||||
}
|
||||
@@ -23744,10 +23450,6 @@ class binary_reader
|
||||
/// the number of characters read
|
||||
std::size_t chars_read = 0;
|
||||
|
||||
/// UBJSON: the sum of the element counts of the containers of a valueless
|
||||
/// type read so far; see @ref charge_valueless_elements
|
||||
std::size_t valueless_elements = 0;
|
||||
|
||||
/// whether we can assume little endianness
|
||||
const bool is_little_endian = little_endianness();
|
||||
|
||||
@@ -30901,29 +30603,20 @@ class binary_writer
|
||||
|
||||
// an optimized array of a valueless type carries no payload, so a
|
||||
// reader has nothing but the declared count to bound the allocation
|
||||
// by. It refuses one array of more than max_valueless_container_size
|
||||
// elements, and all such arrays of one input together when they add
|
||||
// up to more than valueless_elements_base plus
|
||||
// JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE per byte read. Write the
|
||||
// unoptimized form, at one byte per element, for an array that
|
||||
// would exceed the base with the arrays written so far, so that the
|
||||
// result can be read back: the limit of a reader is never below the
|
||||
// base, whatever the macro is set to, so the writer does not use
|
||||
// it (and a reader with a larger rate reads this output as well).
|
||||
// by and refuses an excessive one. Write the unoptimized form for
|
||||
// those, at one byte per element, so the result can be read back.
|
||||
// Objects are not affected: every element is preceded by its key.
|
||||
if (same_prefix && !(use_bjdata && is_bjdata_excluded_type_marker(first_prefix)))
|
||||
const bool valueless_type = (first_prefix == 'Z' || first_prefix == 'T' || first_prefix == 'F');
|
||||
const bool excessive_valueless = IsArray
|
||||
&& valueless_type
|
||||
&& size > detail::max_valueless_container_size;
|
||||
|
||||
if (same_prefix && !excessive_valueless
|
||||
&& !(use_bjdata && is_bjdata_excluded_type_marker(first_prefix)))
|
||||
{
|
||||
const bool valueless_type = IsArray && (first_prefix == 'Z' || first_prefix == 'T' || first_prefix == 'F');
|
||||
if (!valueless_type || size <= detail::valueless_elements_base - valueless_elements_written)
|
||||
{
|
||||
prefix_required = false;
|
||||
oa.write_character(to_char_type('$'));
|
||||
oa.write_character(first_prefix);
|
||||
if (valueless_type)
|
||||
{
|
||||
valueless_elements_written += size;
|
||||
}
|
||||
}
|
||||
prefix_required = false;
|
||||
oa.write_character(to_char_type('$'));
|
||||
oa.write_character(first_prefix);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35147,10 +34840,6 @@ class binary_writer
|
||||
/// how to treat a string value or object key that is not valid UTF-8
|
||||
/// (CBOR, MessagePack, UBJSON, BJData, and BSON; not BON8)
|
||||
const error_handler_t error_handler = binary_writer_default_error_handler();
|
||||
|
||||
/// UBJSON: the number of elements written so far in optimized arrays of a
|
||||
/// valueless type, which a reader adds up and bounds
|
||||
std::size_t valueless_elements_written = 0;
|
||||
};
|
||||
|
||||
} // namespace detail
|
||||
@@ -40774,8 +40463,12 @@ public:
|
||||
}
|
||||
|
||||
#ifndef JSON_NO_THREAD_LOCAL
|
||||
// nesting_depth() is a byte and may exceed the limit by one level
|
||||
static_assert(detail::recursion_depth_limit() < 255, "the nesting depth count must fit in a byte");
|
||||
// nesting_depth() is a byte that never exceeds the limit (JSON_RECURSION_DEPTH_LIMIT),
|
||||
// which is at most 254: the largest value stays one below the byte's maximum, so
|
||||
// that the count cannot wrap around to 0 and let a descent go on without a bound.
|
||||
// (recursion_depth_limit() checks the macro itself, with a message that
|
||||
// names it; this is the check that cannot be edited away unnoticed.)
|
||||
static_assert(detail::recursion_depth_limit() < 255, "the nesting depth count must fit in a byte: JSON_RECURSION_DEPTH_LIMIT must not exceed 254");
|
||||
|
||||
/*!
|
||||
@brief how many levels the operation going on in this thread has descended into
|
||||
@@ -40786,8 +40479,15 @@ public:
|
||||
user code nests them anyway, sharing the count only ends a descent sooner
|
||||
than it had to, which costs a little speed and is never wrong.
|
||||
|
||||
A byte is enough: the count never exceeds the limit by more than the single
|
||||
level that notices the limit has been reached.
|
||||
A byte is enough: only the levels that are descended into are counted
|
||||
(@ref nesting_depth_guard), so the count never exceeds the limit, and the
|
||||
limit is at most 254.
|
||||
|
||||
The count is per thread and not per value or per library configuration: all
|
||||
translation units of a program share it, even if they were compiled with
|
||||
different values of JSON_RECURSION_DEPTH_LIMIT (which is not supported, see
|
||||
the documentation of the macro); each of them then compares the shared count
|
||||
with its own limit.
|
||||
*/
|
||||
static std::uint8_t& nesting_depth() noexcept
|
||||
{
|
||||
@@ -40845,14 +40545,25 @@ public:
|
||||
#endif
|
||||
{
|
||||
#ifndef JSON_NO_THREAD_LOCAL
|
||||
++nesting_depth();
|
||||
// Only a level that is entered is counted, so that the count never
|
||||
// exceeds the limit. A level that is not entered is finished without
|
||||
// descending, and whatever runs while it is (a copy made by a custom
|
||||
// base class, say) sees the same count as the level itself did and
|
||||
// is not entered either.
|
||||
if (m_okay)
|
||||
{
|
||||
++nesting_depth();
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
~nesting_depth_guard()
|
||||
{
|
||||
#ifndef JSON_NO_THREAD_LOCAL
|
||||
--nesting_depth();
|
||||
if (m_okay)
|
||||
{
|
||||
--nesting_depth();
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -44764,7 +44475,10 @@ public:
|
||||
*/
|
||||
void update_members(const iterator& first, const iterator& last, const bool merge_objects, const std::size_t depth)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(depth >= detail::recursion_depth_limit()))
|
||||
// without merge_objects the loop below never descends, so there is no
|
||||
// call stack to save (and the explicit stack always merges); the
|
||||
// distinction matters for a limit of 0, which reaches this at depth 0
|
||||
if (JSON_HEDLEY_UNLIKELY(merge_objects && depth >= detail::recursion_depth_limit()))
|
||||
{
|
||||
update_members_iteratively(first, last);
|
||||
return;
|
||||
@@ -48134,7 +47848,6 @@ struct formatter<nlohmann::NLOHMANN_BASIC_JSON_TPL, char> // NOLINT(cert-dcl58-c
|
||||
#undef JSON_USE_OBJECTS_FOR_ENUM_KEYED_MAPS
|
||||
#undef JSON_DISABLE_STREAM_BULK_SCAN
|
||||
#undef JSON_DISABLE_UNLOCKED_FILE_IO
|
||||
#undef JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE
|
||||
#endif
|
||||
|
||||
// #include <nlohmann/thirdparty/hedley/hedley_undef.hpp>
|
||||
|
||||
+14
-28
@@ -355,6 +355,20 @@ json_test_add_test_for(src/unit-comparison.cpp
|
||||
MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force}
|
||||
)
|
||||
|
||||
# test the walks over deeply nested values with the extreme settings of
|
||||
# JSON_RECURSION_DEPTH_LIMIT: 0 (every walk uses its explicit stack from the
|
||||
# first level on), 1, and 254 (the largest limit that fits the byte that counts
|
||||
# the nesting of a copy)
|
||||
foreach(recursion_depth_limit 0 1 254)
|
||||
json_test_set_test_options(test-recursion_depth_limit_${recursion_depth_limit}
|
||||
COMPILE_DEFINITIONS JSON_RECURSION_DEPTH_LIMIT=${recursion_depth_limit}
|
||||
)
|
||||
json_test_add_test_for(src/unit-recursion_depth_limit.cpp
|
||||
NAME test-recursion_depth_limit_${recursion_depth_limit}
|
||||
MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force}
|
||||
)
|
||||
endforeach()
|
||||
|
||||
# test the parser again with JSON_DIAGNOSTIC_POSITIONS enabled
|
||||
json_test_set_test_options(test-class_parser_diagnostic_positions
|
||||
COMPILE_DEFINITIONS JSON_DIAGNOSTIC_POSITIONS=1
|
||||
@@ -410,34 +424,6 @@ json_test_add_test_for(src/unit-stream-bulk-scan.cpp
|
||||
MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force}
|
||||
)
|
||||
|
||||
# test the budget for the elements of UBJSON containers of a valueless type again
|
||||
# with other values of JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE: 1 (the smallest),
|
||||
# 1000000 (so large that the budget cannot be reached), and the largest value of
|
||||
# a 64-bit integer (which saturates the limit already at the first byte, and is
|
||||
# clamped to the largest std::size_t where that has 32 bits). The file does not
|
||||
# mention JSON_HAS_CPP_<n>, so only C++11 is built unless the standards are forced
|
||||
json_test_set_test_options(test-ubjson_valueless_rate_1
|
||||
COMPILE_DEFINITIONS JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE=1
|
||||
)
|
||||
json_test_add_test_for(src/unit-ubjson_valueless.cpp
|
||||
NAME test-ubjson_valueless_rate_1
|
||||
MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force}
|
||||
)
|
||||
json_test_set_test_options(test-ubjson_valueless_rate_1000000
|
||||
COMPILE_DEFINITIONS JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE=1000000
|
||||
)
|
||||
json_test_add_test_for(src/unit-ubjson_valueless.cpp
|
||||
NAME test-ubjson_valueless_rate_1000000
|
||||
MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force}
|
||||
)
|
||||
json_test_set_test_options(test-ubjson_valueless_rate_max
|
||||
COMPILE_DEFINITIONS JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE=0xFFFFFFFFFFFFFFFFu
|
||||
)
|
||||
json_test_add_test_for(src/unit-ubjson_valueless.cpp
|
||||
NAME test-ubjson_valueless_rate_max
|
||||
MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force}
|
||||
)
|
||||
|
||||
# test reading from a FILE* again with the locking stdio functions
|
||||
json_test_set_test_options(test-file_input_unlocked_io_disabled
|
||||
COMPILE_DEFINITIONS JSON_DISABLE_UNLOCKED_FILE_IO=1
|
||||
|
||||
@@ -1,86 +0,0 @@
|
||||
// __ _____ _____ _____
|
||||
// __| | __| | | | JSON for Modern C++ (supporting code)
|
||||
// | | |__ | | | | | | version 3.12.0
|
||||
// |_____|_____|_____|_|___| https://github.com/nlohmann/json
|
||||
//
|
||||
// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm> // max
|
||||
#include <cstddef> // size_t
|
||||
#include <cstdint> // uint8_t
|
||||
#include <memory> // allocator
|
||||
#include <vector> // vector
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace utils
|
||||
{
|
||||
/// the allocations of the values of a JSON type that uses counting_allocator
|
||||
struct allocation_counts
|
||||
{
|
||||
/// bytes allocated and not yet deallocated
|
||||
std::size_t current = 0;
|
||||
/// the largest value current had
|
||||
std::size_t peak = 0;
|
||||
/// number of allocations
|
||||
std::size_t allocations = 0;
|
||||
};
|
||||
|
||||
/// the counts of all counting_allocators; the tests are single-threaded
|
||||
inline allocation_counts& counts()
|
||||
{
|
||||
static allocation_counts c; // NOLINT(misc-const-correctness)
|
||||
return c;
|
||||
}
|
||||
|
||||
/// an allocator that counts the bytes and the number of allocations of a
|
||||
/// JSON type, so that a test can check how much memory a parse holds at any
|
||||
/// time without measuring the memory of the process
|
||||
template<class T>
|
||||
struct counting_allocator : std::allocator<T>
|
||||
{
|
||||
using std::allocator<T>::allocator;
|
||||
|
||||
counting_allocator() = default;
|
||||
template<class U> counting_allocator(const counting_allocator<U>& /*unused*/) noexcept {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions)
|
||||
|
||||
template <class U>
|
||||
struct rebind
|
||||
{
|
||||
using other = counting_allocator<U>;
|
||||
};
|
||||
|
||||
T* allocate(const std::size_t n)
|
||||
{
|
||||
T* const result = std::allocator<T>::allocate(n);
|
||||
allocation_counts& c = counts();
|
||||
c.current += n * sizeof(T);
|
||||
c.peak = (std::max)(c.peak, c.current);
|
||||
++c.allocations;
|
||||
return result;
|
||||
}
|
||||
|
||||
void deallocate(T* p, const std::size_t n)
|
||||
{
|
||||
counts().current -= n * sizeof(T);
|
||||
std::allocator<T>::deallocate(p, n);
|
||||
}
|
||||
};
|
||||
|
||||
using counted_json = nlohmann::json::with_allocator_t<counting_allocator>;
|
||||
|
||||
/// @a count times the bytes of @a unit
|
||||
inline std::vector<std::uint8_t> repeated(const std::vector<std::uint8_t>& unit, const std::size_t count)
|
||||
{
|
||||
std::vector<std::uint8_t> result;
|
||||
result.reserve(unit.size() * count);
|
||||
for (std::size_t i = 0; i < count; ++i)
|
||||
{
|
||||
result.insert(result.end(), unit.begin(), unit.end());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
} // namespace utils
|
||||
@@ -1,125 +0,0 @@
|
||||
// __ _____ _____ _____
|
||||
// __| | __| | | | JSON for Modern C++ (supporting code)
|
||||
// | | |__ | | | | | | version 3.12.0
|
||||
// |_____|_____|_____|_|___| https://github.com/nlohmann/json
|
||||
//
|
||||
// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstddef> // size_t
|
||||
#include <cstdint> // uint8_t, uint32_t
|
||||
#include <limits> // numeric_limits
|
||||
#include <string> // string
|
||||
#include <vector> // vector
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
// helpers of the tests of the budget for the elements of UBJSON containers of
|
||||
// a valueless type (unit-ubjson.cpp and unit-ubjson_valueless.cpp)
|
||||
#if defined(__clang__)
|
||||
#pragma clang diagnostic push
|
||||
#pragma clang diagnostic ignored "-Wweak-vtables"
|
||||
#endif
|
||||
|
||||
namespace utils
|
||||
{
|
||||
/// SAX consumer that counts the values and keeps the id of a parse error
|
||||
struct ValuelessEventCounter : public nlohmann::json_sax<nlohmann::json>
|
||||
{
|
||||
bool null() override
|
||||
{
|
||||
return ++values < stop_after;
|
||||
}
|
||||
bool boolean(bool /*val*/) override
|
||||
{
|
||||
return ++values < stop_after;
|
||||
}
|
||||
bool number_integer(nlohmann::json::number_integer_t /*val*/) override
|
||||
{
|
||||
++values;
|
||||
return true;
|
||||
}
|
||||
bool number_unsigned(nlohmann::json::number_unsigned_t /*val*/) override
|
||||
{
|
||||
++values;
|
||||
return true;
|
||||
}
|
||||
bool number_float(nlohmann::json::number_float_t /*val*/, const std::string& /*s*/) override
|
||||
{
|
||||
++values;
|
||||
return true;
|
||||
}
|
||||
bool string(std::string& /*val*/) override
|
||||
{
|
||||
++values;
|
||||
return true;
|
||||
}
|
||||
bool binary(nlohmann::json::binary_t& /*val*/) override
|
||||
{
|
||||
++values;
|
||||
return true;
|
||||
}
|
||||
bool start_object(std::size_t /*elements*/) override
|
||||
{
|
||||
return true;
|
||||
}
|
||||
bool key(std::string& /*val*/) override
|
||||
{
|
||||
return true;
|
||||
}
|
||||
bool end_object() override
|
||||
{
|
||||
return true;
|
||||
}
|
||||
bool start_array(std::size_t /*elements*/) override
|
||||
{
|
||||
return true;
|
||||
}
|
||||
bool end_array() override
|
||||
{
|
||||
return true;
|
||||
}
|
||||
bool parse_error(std::size_t /*position*/, const std::string& /*last_token*/, const nlohmann::json::exception& ex) override
|
||||
{
|
||||
error_id = ex.id;
|
||||
return false;
|
||||
}
|
||||
|
||||
/// the handler refuses the value that makes this many
|
||||
std::size_t stop_after = (std::numeric_limits<std::size_t>::max)();
|
||||
std::size_t values = 0;
|
||||
int error_id = 0;
|
||||
};
|
||||
|
||||
/// appends `[$<marker>#l<count>` (nine bytes) to @a v
|
||||
inline void append_valueless_array(std::vector<std::uint8_t>& v, const char marker, const std::uint32_t count)
|
||||
{
|
||||
v.push_back('[');
|
||||
v.push_back('$');
|
||||
v.push_back(static_cast<std::uint8_t>(marker));
|
||||
v.push_back('#');
|
||||
v.push_back('l');
|
||||
v.push_back(static_cast<std::uint8_t>(count >> 24));
|
||||
v.push_back(static_cast<std::uint8_t>(count >> 16));
|
||||
v.push_back(static_cast<std::uint8_t>(count >> 8));
|
||||
v.push_back(static_cast<std::uint8_t>(count));
|
||||
}
|
||||
|
||||
/// `[` followed by one `[$<marker>#l<count>` per entry of @a counts and `]`
|
||||
inline std::vector<std::uint8_t> valueless_arrays(const char marker, const std::vector<std::uint32_t>& counts)
|
||||
{
|
||||
std::vector<std::uint8_t> v = {'['};
|
||||
for (const auto count : counts)
|
||||
{
|
||||
append_valueless_array(v, marker, count);
|
||||
}
|
||||
v.push_back(']');
|
||||
return v;
|
||||
}
|
||||
} // namespace utils
|
||||
|
||||
#if defined(__clang__)
|
||||
#pragma clang diagnostic pop
|
||||
#endif
|
||||
@@ -236,7 +236,8 @@ TEST_CASE("controlled bad_alloc")
|
||||
|
||||
next_construct_fails = false;
|
||||
|
||||
// deeper than the 128 levels the copy constructor descends into
|
||||
// deeper than the levels the copy constructor descends into
|
||||
// (recursion_depth_limit(): 128 by default, 254 at most)
|
||||
const std::size_t depth = 300;
|
||||
|
||||
my_json j = 1;
|
||||
@@ -300,8 +301,9 @@ struct nth_alloc_fails_allocator : std::allocator<T>
|
||||
};
|
||||
};
|
||||
|
||||
// builds a value nested more than 128 levels deep - the bound the copy
|
||||
// constructor descends into before it continues without the call stack - and
|
||||
// builds a value nested more than recursion_depth_limit() (128 by default)
|
||||
// levels deep - the bound the copy constructor descends into before it
|
||||
// continues without the call stack - and
|
||||
// checks that a copy survives any single allocation of it failing: every
|
||||
// attempt either throws std::bad_alloc, without crashing or leaving the
|
||||
// source altered, or completes the copy
|
||||
@@ -312,9 +314,9 @@ void check_deep_copy_survives_failing_allocation(bool nest_objects)
|
||||
|
||||
fail_at_alloc_call = -1;
|
||||
|
||||
// [[[ ... [1] ... ]]], or the same nesting with objects, 130 levels deep
|
||||
// [[[ ... [1] ... ]]], or the same nesting with objects, two levels deeper than the limit
|
||||
BasicJsonType src = 1;
|
||||
for (std::size_t i = 0; i < 130; ++i)
|
||||
for (std::size_t i = 0; i < nlohmann::detail::recursion_depth_limit() + 2; ++i)
|
||||
{
|
||||
if (nest_objects)
|
||||
{
|
||||
@@ -440,8 +442,9 @@ TEST_CASE("deep copy uses the provided allocator")
|
||||
{
|
||||
using counting_json = nlohmann::json::with_allocator_t<scratch_counting_allocator>;
|
||||
|
||||
// deeper than the 128 levels the copy constructor descends into, so the
|
||||
// innermost objects are copied by the iterative deep copy
|
||||
// deeper than the levels the copy constructor descends into (128 by
|
||||
// default, 254 at most), so the innermost objects are copied by the
|
||||
// iterative deep copy
|
||||
counting_json j = 1;
|
||||
for (std::size_t i = 0; i < 300; ++i)
|
||||
{
|
||||
@@ -499,11 +502,12 @@ TEST_CASE("converting a deeply nested value from another specialization fails cl
|
||||
#if !(defined(_MSC_VER) && _MSC_VER < 1910 && defined(_ITERATOR_DEBUG_LEVEL) && _ITERATOR_DEBUG_LEVEL > 0)
|
||||
using countdown_json = nlohmann::json::with_allocator_t<countdown_allocator>;
|
||||
|
||||
// deeper than the 128 levels the converting constructor descends into, so
|
||||
// that failures land on both sides of the bound - or, built with
|
||||
// JSON_NO_THREAD_LOCAL, all in the iterative conversion
|
||||
// deeper than the levels the converting constructor descends into
|
||||
// (recursion_depth_limit(), 128 by default), so that failures land on both
|
||||
// sides of the bound - or, built with JSON_NO_THREAD_LOCAL, all in the
|
||||
// iterative conversion
|
||||
json j = {1, "two", {{"three", 3}}};
|
||||
for (std::size_t i = 0; i < 150; ++i)
|
||||
for (std::size_t i = 0; i < nlohmann::detail::recursion_depth_limit() + 22; ++i)
|
||||
{
|
||||
j = json{{"a", json::array({j, "sibling"})}};
|
||||
}
|
||||
|
||||
@@ -31,7 +31,6 @@ using nlohmann::ordered_json;
|
||||
#include "make_test_data_available.hpp"
|
||||
#include "test_utils.hpp"
|
||||
#include "sax_countdown.hpp"
|
||||
#include "allocation_counter.hpp"
|
||||
#include "sax_event_loggers.hpp"
|
||||
using utils::SaxCountdown;
|
||||
using utils::SaxEventLogger;
|
||||
@@ -4031,60 +4030,6 @@ TEST_CASE("issue #5405 - array reserve for definite-length BJData arrays")
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("BJData - open arrays reserve only what the input has backed")
|
||||
{
|
||||
// an array that announces its size gets storage for (up to 16384 of) its
|
||||
// elements when it opens; the arrays that are open at the same time must
|
||||
// not hold more than the input read so far can back
|
||||
SECTION("nested headers that announce 16384 elements and are never completed")
|
||||
{
|
||||
// 200 levels would hold 200 * 16384 * sizeof(json),
|
||||
// if each of them reserved what it announces
|
||||
const auto input = utils::repeated({'[', '#', 'I', 0x00, 0x40}, 200);
|
||||
|
||||
utils::counts() = utils::allocation_counts{};
|
||||
const auto j = utils::counted_json::from_bjdata(input, true, false);
|
||||
CHECK(j.is_discarded());
|
||||
CAPTURE(utils::counts().peak)
|
||||
// the credit is 65,536 values; the base peaks at more than 3 million
|
||||
CHECK(utils::counts().peak < sizeof(utils::counted_json) * 128 * 1024);
|
||||
}
|
||||
|
||||
SECTION("a completed array is still reserved in one allocation")
|
||||
{
|
||||
json array = json::array();
|
||||
for (int i = 0; i < 10000; ++i)
|
||||
{
|
||||
array.push_back(i % 100);
|
||||
}
|
||||
|
||||
// an array without a size is not reserved, so only these two
|
||||
{
|
||||
const auto packed = json::to_bjdata(array, true);
|
||||
|
||||
utils::counts() = utils::allocation_counts{};
|
||||
const auto j = utils::counted_json::from_bjdata(packed);
|
||||
CHECK(j.size() == 10000);
|
||||
// the array and its storage; growing the storage as the elements
|
||||
// arrive would take more than a dozen allocations
|
||||
CAPTURE(utils::counts().allocations)
|
||||
CHECK(utils::counts().allocations <= 4);
|
||||
}
|
||||
|
||||
{
|
||||
const auto packed = json::to_bjdata(array, true, true);
|
||||
|
||||
utils::counts() = utils::allocation_counts{};
|
||||
const auto j = utils::counted_json::from_bjdata(packed);
|
||||
CHECK(j.size() == 10000);
|
||||
// the array and its storage; growing the storage as the elements
|
||||
// arrive would take more than a dozen allocations
|
||||
CAPTURE(utils::counts().allocations)
|
||||
CHECK(utils::counts().allocations <= 4);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("Universal Binary JSON Specification Examples 1")
|
||||
{
|
||||
SECTION("Null Value")
|
||||
|
||||
@@ -29,7 +29,6 @@ using nlohmann::json;
|
||||
#include "round_trip_corpus.hpp"
|
||||
#include "test_utils.hpp"
|
||||
#include "sax_countdown.hpp"
|
||||
#include "allocation_counter.hpp"
|
||||
#include "sax_event_loggers.hpp"
|
||||
using utils::SaxCountdown;
|
||||
using utils::SaxEventLogger;
|
||||
@@ -2710,289 +2709,6 @@ TEST_CASE("issue #5405 - array reserve for definite-length CBOR arrays")
|
||||
}
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
// NOLINTBEGIN(bugprone-derived-method-shadowing-base-method)
|
||||
using dom_parser = nlohmann::detail::json_sax_dom_parser<json, nlohmann::detail::input_stream_adapter>;
|
||||
|
||||
// handlers that wrap a DOM parser, as users of the SAX interface do
|
||||
struct public_wrapper : dom_parser
|
||||
{
|
||||
explicit public_wrapper(json& result) : dom_parser(result, false)
|
||||
{}
|
||||
};
|
||||
|
||||
/// parses another input with itself while the first parse is still running
|
||||
struct reentrant_wrapper : dom_parser
|
||||
{
|
||||
explicit reentrant_wrapper(json& result) : dom_parser(result, false)
|
||||
{}
|
||||
|
||||
bool start_array(std::size_t len)
|
||||
{
|
||||
if (!inner_done)
|
||||
{
|
||||
inner_done = true;
|
||||
const std::vector<std::uint8_t> input = {0x80};
|
||||
json::sax_parse(input, this, json::input_format_t::cbor);
|
||||
}
|
||||
return dom_parser::start_array(len);
|
||||
}
|
||||
|
||||
bool inner_done = false;
|
||||
};
|
||||
|
||||
class private_wrapper : private dom_parser
|
||||
{
|
||||
public:
|
||||
explicit private_wrapper(json& result) : dom_parser(result, false)
|
||||
{}
|
||||
|
||||
using dom_parser::null;
|
||||
using dom_parser::boolean;
|
||||
using dom_parser::number_integer;
|
||||
using dom_parser::number_unsigned;
|
||||
using dom_parser::number_float;
|
||||
using dom_parser::string;
|
||||
using dom_parser::binary;
|
||||
using dom_parser::start_array;
|
||||
using dom_parser::end_array;
|
||||
using dom_parser::parse_error;
|
||||
|
||||
// the DOM parser has private overloads of these, which a using
|
||||
// declaration cannot expose; the input of the test has no objects
|
||||
static bool start_object(std::size_t /*len*/)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool key(std::string& /*val*/)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool end_object()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
class protected_wrapper : protected dom_parser
|
||||
{
|
||||
public:
|
||||
explicit protected_wrapper(json& result) : dom_parser(result, false)
|
||||
{}
|
||||
|
||||
using dom_parser::null;
|
||||
using dom_parser::boolean;
|
||||
using dom_parser::number_integer;
|
||||
using dom_parser::number_unsigned;
|
||||
using dom_parser::number_float;
|
||||
using dom_parser::string;
|
||||
using dom_parser::binary;
|
||||
using dom_parser::start_array;
|
||||
using dom_parser::end_array;
|
||||
using dom_parser::parse_error;
|
||||
|
||||
// the DOM parser has private overloads of these, which a using
|
||||
// declaration cannot expose; the input of the test has no objects
|
||||
static bool start_object(std::size_t /*len*/)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool key(std::string& /*val*/)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool end_object()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
// NOLINTEND(bugprone-derived-method-shadowing-base-method)
|
||||
|
||||
/// the capacity of the fifth array of five that are open
|
||||
std::size_t innermost_capacity(const json& result)
|
||||
{
|
||||
const json* node = &result;
|
||||
for (int level = 0; level < 4; ++level)
|
||||
{
|
||||
node = &node->front();
|
||||
}
|
||||
return node->get_ref<const json::array_t&>().capacity();
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST_CASE("CBOR - open arrays reserve only what the input has backed")
|
||||
{
|
||||
// an array that announces its size gets storage for (up to 16384 of) its
|
||||
// elements when it opens; the arrays that are open at the same time must
|
||||
// not hold more than the input read so far can back
|
||||
SECTION("nested headers that announce 16384 elements and are never completed")
|
||||
{
|
||||
// 200 levels would hold 200 * 16384 * sizeof(json),
|
||||
// if each of them reserved what it announces
|
||||
const auto input = utils::repeated({0x99, 0x40, 0x00}, 200);
|
||||
|
||||
utils::counts() = utils::allocation_counts{};
|
||||
const auto j = utils::counted_json::from_cbor(input, true, false);
|
||||
CHECK(j.is_discarded());
|
||||
CAPTURE(utils::counts().peak)
|
||||
// the credit is 65,536 values; the base peaks at more than 3 million
|
||||
CHECK(utils::counts().peak < sizeof(utils::counted_json) * 128 * 1024);
|
||||
}
|
||||
|
||||
SECTION("a completed array is still reserved in one allocation")
|
||||
{
|
||||
json array = json::array();
|
||||
for (int i = 0; i < 10000; ++i)
|
||||
{
|
||||
array.push_back(i % 100);
|
||||
}
|
||||
|
||||
{
|
||||
const auto packed = json::to_cbor(array);
|
||||
|
||||
utils::counts() = utils::allocation_counts{};
|
||||
const auto j = utils::counted_json::from_cbor(packed);
|
||||
CHECK(j.size() == 10000);
|
||||
// the array and its storage; growing the storage as the elements
|
||||
// arrive would take more than a dozen allocations
|
||||
CAPTURE(utils::counts().allocations)
|
||||
CHECK(utils::counts().allocations <= 4);
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("an array that is given less than it announces is still decoded")
|
||||
{
|
||||
// five levels that announce 16384 elements each, the first of which
|
||||
// is the next level: they are more than the input could back when
|
||||
// they open
|
||||
constexpr std::size_t levels = 5;
|
||||
std::vector<std::uint8_t> input;
|
||||
for (std::size_t level = 0; level < levels; ++level)
|
||||
{
|
||||
input.insert(input.end(), {0x99, 0x40, 0x00});
|
||||
}
|
||||
input.insert(input.end(), 16384, 0x01);
|
||||
for (std::size_t level = 1; level < levels; ++level)
|
||||
{
|
||||
input.insert(input.end(), 16383, 0x02);
|
||||
}
|
||||
|
||||
const auto j = json::from_cbor(input);
|
||||
const json* node = &j;
|
||||
for (std::size_t level = 0; level < levels; ++level)
|
||||
{
|
||||
REQUIRE(node->is_array());
|
||||
CHECK(node->size() == 16384);
|
||||
node = &node->front();
|
||||
}
|
||||
CHECK(*node == 1);
|
||||
CHECK(j.back() == 2);
|
||||
}
|
||||
|
||||
SECTION("a DOM parser that a parse has finished with is not bounded by the input of that parse")
|
||||
{
|
||||
// the DOM parser learns where the reader counts its characters, and
|
||||
// must forget it when the parse is over: a handler may be driven by
|
||||
// other code afterwards, while the reader no longer exists
|
||||
// five levels announce more than the credit of one input, which a
|
||||
// parse of one byte has hardly refilled
|
||||
const auto check_unbounded = [](dom_parser & parser, json & result)
|
||||
{
|
||||
for (int level = 0; level < 5; ++level)
|
||||
{
|
||||
CHECK(parser.start_array(16384));
|
||||
}
|
||||
|
||||
const json* node = &result;
|
||||
for (int level = 0; level < 4; ++level)
|
||||
{
|
||||
REQUIRE(node->is_array());
|
||||
REQUIRE(node->size() == 1);
|
||||
node = &node->front();
|
||||
}
|
||||
REQUIRE(node->is_array());
|
||||
CHECK(node->get_ref<const json::array_t&>().capacity() == 16384);
|
||||
};
|
||||
|
||||
SECTION("after a parse that succeeded")
|
||||
{
|
||||
json result;
|
||||
dom_parser parser(result, false);
|
||||
const std::vector<std::uint8_t> input = {0x80};
|
||||
CHECK(json::sax_parse(input, &parser, json::input_format_t::cbor));
|
||||
check_unbounded(parser, result);
|
||||
}
|
||||
|
||||
#if !defined(JSON_NOEXCEPTION)
|
||||
SECTION("after a parse that threw")
|
||||
{
|
||||
json result;
|
||||
dom_parser parser(result, true);
|
||||
const std::vector<std::uint8_t> input = {0x1C}; // not a CBOR value
|
||||
CHECK_THROWS_AS(json::sax_parse(input, &parser, json::input_format_t::cbor), json::parse_error&);
|
||||
check_unbounded(parser, result);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
SECTION("a handler that wraps a DOM parser is bound if the reader can reach the parser")
|
||||
{
|
||||
// five arrays announce 16384 elements: the credit of one input covers
|
||||
// four of them, and a parse of 15 bytes has hardly refilled it
|
||||
const auto input = utils::repeated({0x99, 0x40, 0x00}, 5);
|
||||
|
||||
SECTION("public base")
|
||||
{
|
||||
json result;
|
||||
public_wrapper handler(result);
|
||||
CHECK_FALSE(json::sax_parse(input, &handler, json::input_format_t::cbor));
|
||||
CHECK(innermost_capacity(result) < 16384);
|
||||
}
|
||||
|
||||
SECTION("public base, a parse inside the parse")
|
||||
{
|
||||
json result;
|
||||
reentrant_wrapper handler(result);
|
||||
CHECK_FALSE(json::sax_parse(input, &handler, json::input_format_t::cbor));
|
||||
CHECK(handler.inner_done);
|
||||
CHECK(innermost_capacity(result) < 16384);
|
||||
}
|
||||
|
||||
// the reader cannot convert these to the DOM parser: they are not bound
|
||||
SECTION("private base")
|
||||
{
|
||||
json result;
|
||||
private_wrapper handler(result);
|
||||
CHECK_FALSE(json::sax_parse(input, &handler, json::input_format_t::cbor));
|
||||
CHECK(innermost_capacity(result) == 16384);
|
||||
}
|
||||
|
||||
SECTION("protected base")
|
||||
{
|
||||
json result;
|
||||
protected_wrapper handler(result);
|
||||
CHECK_FALSE(json::sax_parse(input, &handler, json::input_format_t::cbor));
|
||||
CHECK(innermost_capacity(result) == 16384);
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("SAX events report the announced size")
|
||||
{
|
||||
SaxEventLogger logger;
|
||||
const std::vector<std::uint8_t> input = {0x99, 0x40, 0x00};
|
||||
CHECK_FALSE(json::sax_parse(input, &logger, json::input_format_t::cbor));
|
||||
REQUIRE(logger.events.size() == 2);
|
||||
CHECK(logger.events[0] == "start_array(16384)");
|
||||
CHECK(logger.events[1] == "parse_error(4)");
|
||||
}
|
||||
}
|
||||
|
||||
#if !defined(JSON_NOEXCEPTION) // corpus values that do not survive the round trip are skipped by catching the exception
|
||||
TEST_CASE("CBOR round-trip invariants")
|
||||
{
|
||||
|
||||
@@ -62,6 +62,15 @@ template<> struct StringMaker<std::partial_ordering>
|
||||
|
||||
namespace
|
||||
{
|
||||
// nesting depths on either side of detail::recursion_depth_limit(): none, the
|
||||
// last one within the limit, the first one past it, and well past it (0, 127,
|
||||
// 128, and 200 by default)
|
||||
std::vector<std::size_t> depths_around_recursion_limit()
|
||||
{
|
||||
const std::size_t limit = nlohmann::detail::recursion_depth_limit();
|
||||
return {0, limit > 0 ? limit - 1 : 0, limit, limit + 72};
|
||||
}
|
||||
|
||||
// helper function to check std::less<json::value_t>
|
||||
// see https://en.cppreference.com/w/cpp/utility/functional/less
|
||||
template <typename A, typename B, typename U = std::less<json::value_t>>
|
||||
@@ -968,7 +977,7 @@ TEST_CASE("copying an object preserves its comparator's state")
|
||||
object["a"] = 3;
|
||||
REQUIRE(object.dump() == R"({"B":2,"a":3,"b":1})");
|
||||
|
||||
for (const std::size_t depth : std::vector<std::size_t> {0, 127, 128, 200})
|
||||
for (const std::size_t depth : depths_around_recursion_limit())
|
||||
{
|
||||
CAPTURE(depth)
|
||||
|
||||
@@ -1011,7 +1020,7 @@ TEST_CASE("equality of an object whose comparator treats different keys as equiv
|
||||
// sanity check: the object type's own comparison already disagrees
|
||||
CHECK_FALSE(a.get_ref<const ci_json::object_t&>() == b.get_ref<const ci_json::object_t&>());
|
||||
|
||||
for (const std::size_t depth : std::vector<std::size_t> {0, 127, 128, 200})
|
||||
for (const std::size_t depth : depths_around_recursion_limit())
|
||||
{
|
||||
CAPTURE(depth)
|
||||
|
||||
@@ -1135,9 +1144,10 @@ TEST_CASE("operator<=> of binary values with a different subtype does not depend
|
||||
return result;
|
||||
};
|
||||
|
||||
// 127 levels stay within nesting_depth_limit() (128); 128 and 200 do not,
|
||||
// and must still agree with the levels that do
|
||||
for (const std::size_t depth : std::vector<std::size_t> {0, 127, 128, 200})
|
||||
// one level less than the recursion depth limit (127 by default) stays within it;
|
||||
// the limit itself and more (128 and 200) do not, and must still agree with the
|
||||
// levels that do
|
||||
for (const std::size_t depth : depths_around_recursion_limit())
|
||||
{
|
||||
CAPTURE(depth)
|
||||
const json x = deep(a, depth);
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
#include <nlohmann/json.hpp>
|
||||
using nlohmann::json;
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <deque>
|
||||
#include <map>
|
||||
@@ -217,16 +218,17 @@ TEST_CASE("array type with std::deque nested beyond the recursion limit")
|
||||
// recursion_depth_limit() with an explicit stack of frames; with the
|
||||
// iterators of std::deque, these frames are copied and moved through
|
||||
// their own special members
|
||||
const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 72; // 200 by default
|
||||
deque_json j = 1;
|
||||
for (int i = 0; i < 200; ++i)
|
||||
for (std::size_t i = 0; i < depth; ++i)
|
||||
{
|
||||
deque_json outer = deque_json::array();
|
||||
outer.push_back(std::move(j));
|
||||
j = std::move(outer);
|
||||
}
|
||||
|
||||
// 200 arrays of one element each (0x81) around the integer 1
|
||||
std::vector<std::uint8_t> expected(200, 0x81);
|
||||
// `depth` arrays of one element each (0x81) around the integer 1
|
||||
std::vector<std::uint8_t> expected(depth, 0x81);
|
||||
expected.push_back(0x01);
|
||||
CHECK(deque_json::to_cbor(j) == expected);
|
||||
}
|
||||
@@ -475,10 +475,10 @@ TEST_CASE("Regression test for issue #5674 - copy construction must not require
|
||||
|
||||
SECTION("nested deeper than the copy constructor's descent bound")
|
||||
{
|
||||
// beyond nesting_depth_limit() (128) levels, the copy constructor
|
||||
// beyond recursion_depth_limit() (128 by default) levels, the copy constructor
|
||||
// copies without the call stack (copy_iteratively / copy_array_level),
|
||||
// which used to assign the base class of every element it created
|
||||
const std::size_t depth = 300;
|
||||
const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 172;
|
||||
|
||||
const json_with_const_base j = make_nested_array(depth);
|
||||
const json_with_const_base copy = j; // NOLINT(performance-unnecessary-copy-initialization)
|
||||
|
||||
@@ -128,16 +128,19 @@ TEST_CASE("Better diagnostics with positions")
|
||||
check_copy(depth, true);
|
||||
};
|
||||
|
||||
// around detail::recursion_depth_limit() (127, 128, and 129 by default)
|
||||
const std::size_t limit = nlohmann::detail::recursion_depth_limit();
|
||||
|
||||
check_arrays(1);
|
||||
check_arrays(127);
|
||||
check_arrays(128);
|
||||
check_arrays(129);
|
||||
check_arrays(limit > 0 ? limit - 1 : 0);
|
||||
check_arrays(limit);
|
||||
check_arrays(limit + 1);
|
||||
check_arrays(300);
|
||||
|
||||
check_objects(1);
|
||||
check_objects(127);
|
||||
check_objects(128);
|
||||
check_objects(129);
|
||||
check_objects(limit > 0 ? limit - 1 : 0);
|
||||
check_objects(limit);
|
||||
check_objects(limit + 1);
|
||||
check_objects(300);
|
||||
}
|
||||
|
||||
@@ -186,10 +189,13 @@ TEST_CASE("Better diagnostics with positions")
|
||||
}
|
||||
};
|
||||
|
||||
// around detail::recursion_depth_limit() (127, 128, and 129 by default)
|
||||
const std::size_t limit = nlohmann::detail::recursion_depth_limit();
|
||||
|
||||
check_conversion(1);
|
||||
check_conversion(127);
|
||||
check_conversion(128);
|
||||
check_conversion(129);
|
||||
check_conversion(limit > 0 ? limit - 1 : 0);
|
||||
check_conversion(limit);
|
||||
check_conversion(limit + 1);
|
||||
check_conversion(300);
|
||||
}
|
||||
|
||||
|
||||
@@ -515,9 +515,11 @@ TEST_CASE("Regression tests for extended diagnostics")
|
||||
|
||||
// update(j, true) around its descent bound, where the nested vectors
|
||||
// grow while the objects are merged without recursing
|
||||
const std::size_t limit = nlohmann::detail::recursion_depth_limit();
|
||||
// (the limit can be 0, where "one level less" is no level)
|
||||
for (const std::size_t depth :
|
||||
{
|
||||
nlohmann::detail::recursion_depth_limit() - 1, nlohmann::detail::recursion_depth_limit(), nlohmann::detail::recursion_depth_limit() + 2
|
||||
limit > 0 ? limit - 1 : 0, limit, limit + 2
|
||||
})
|
||||
{
|
||||
ordered_json j = {{"z", {{"x", 1}}}};
|
||||
|
||||
@@ -1975,14 +1975,15 @@ TEST_CASE("JSON patch: diff of deeply nested values")
|
||||
SECTION("the diff reproduces the target at every depth")
|
||||
{
|
||||
// depths on either side of the nesting depth up to which diff()
|
||||
// recurses (detail::recursion_depth_limit(), 128); not every depth up
|
||||
// recurses (detail::recursion_depth_limit(), 128 by default); not every depth up
|
||||
// to 300, as the test would then time out under Valgrind
|
||||
std::vector<std::size_t> depths;
|
||||
for (std::size_t depth = 0; depth <= 16; ++depth)
|
||||
{
|
||||
depths.push_back(depth);
|
||||
}
|
||||
for (std::size_t depth = 120; depth <= 136; ++depth)
|
||||
const std::size_t limit = nlohmann::detail::recursion_depth_limit();
|
||||
for (std::size_t depth = limit > 8 ? limit - 8 : 0; depth <= limit + 8; ++depth)
|
||||
{
|
||||
depths.push_back(depth);
|
||||
}
|
||||
@@ -2524,7 +2525,7 @@ TEST_CASE("JSON patch - diff of leaf values below the bound of the recursive des
|
||||
// the values at the innermost level are diffed without recursion: equal
|
||||
// leaves yield nothing, leaves of different types and different leaves
|
||||
// are replaced
|
||||
const std::size_t depth = 200;
|
||||
const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 72; // 200 by default
|
||||
|
||||
json source = {1, "text", 3, {{"x", 1}, {"y", 2}}, true, nullptr};
|
||||
json target = {1, 2, 4, {{"x", 1}, {"y", 3}}, true, nullptr};
|
||||
|
||||
@@ -50,13 +50,18 @@ json nested_object(const std::size_t depth, json leaf)
|
||||
|
||||
TEST_CASE("issue #5392 - binary writers on deeply nested values")
|
||||
{
|
||||
// 200 is past the point where the writers stop recursing, and still
|
||||
// shallow enough that from_* and operator== (which still recurse) are fine.
|
||||
const json deep_array = nested_array(200, json(0));
|
||||
const json deep_object = nested_object(200, json("x"));
|
||||
const json empty_array = nested_array(200, json::array());
|
||||
const json empty_object = nested_object(200, json::object());
|
||||
const json mixed = nested_object(80, nested_array(80, json(true)));
|
||||
// The limit (128 by default) of the levels that the writers recurse into.
|
||||
const std::size_t limit = nlohmann::detail::recursion_depth_limit();
|
||||
|
||||
// 72 levels more than the limit (200 by default) is past the point where
|
||||
// the writers stop recursing, and still shallow enough that from_* (which
|
||||
// does not recurse) and operator== (which does, up to the limit) are quick.
|
||||
const json deep_array = nested_array(limit + 72, json(0));
|
||||
const json deep_object = nested_object(limit + 72, json("x"));
|
||||
const json empty_array = nested_array(limit + 72, json::array());
|
||||
const json empty_object = nested_object(limit + 72, json::object());
|
||||
// 2 * (limit / 2 + 16) levels: 160 by default, and more than the limit for every limit
|
||||
const json mixed = nested_object((limit / 2) + 16, nested_array((limit / 2) + 16, json(true)));
|
||||
|
||||
SECTION("roundtrip past the recursion bound")
|
||||
{
|
||||
@@ -95,7 +100,7 @@ TEST_CASE("issue #5392 - binary writers on deeply nested values")
|
||||
|
||||
SECTION("the two ways of writing a value meet at the bound")
|
||||
{
|
||||
for (std::size_t depth = 120; depth <= 140; ++depth)
|
||||
for (std::size_t depth = limit > 8 ? limit - 8 : 0; depth <= limit + 12; ++depth)
|
||||
{
|
||||
CAPTURE(depth)
|
||||
|
||||
@@ -118,8 +123,8 @@ TEST_CASE("issue #5392 - binary writers on deeply nested values")
|
||||
const json ndarray = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2, 3}}, {"_ArrayData_", {1, 2, 3, 4, 5, 6}}});
|
||||
const json invalid = json({{"_ArrayType_", "nope"}, {"_ArraySize_", {1}}, {"_ArrayData_", {1}}});
|
||||
|
||||
const json deep_ndarray = nested_array(140, ndarray);
|
||||
const json deep_invalid = nested_array(140, invalid);
|
||||
const json deep_ndarray = nested_array(limit + 12, ndarray);
|
||||
const json deep_invalid = nested_array(limit + 12, invalid);
|
||||
|
||||
CHECK(json::from_bjdata(json::to_bjdata(deep_ndarray)) == deep_ndarray);
|
||||
CHECK(json::from_bjdata(json::to_bjdata(deep_invalid)) == deep_invalid);
|
||||
@@ -131,10 +136,10 @@ TEST_CASE("issue #5392 - binary writers on deeply nested values")
|
||||
// nested one-element arrays around the recursion bound: the exact
|
||||
// bytes a writer produces do not depend on whether it stayed on the
|
||||
// call stack or moved to the heap one partway through
|
||||
// (the limit can be 0, where "one level less" is no level)
|
||||
for (const std::size_t depth :
|
||||
{
|
||||
nlohmann::detail::recursion_depth_limit() - 1, nlohmann::detail::recursion_depth_limit(),
|
||||
nlohmann::detail::recursion_depth_limit() + 1, nlohmann::detail::recursion_depth_limit() + 2
|
||||
limit > 0 ? limit - 1 : 0, limit, limit + 1, limit + 2
|
||||
})
|
||||
{
|
||||
CAPTURE(depth)
|
||||
|
||||
@@ -115,7 +115,7 @@ TEST_CASE("issue #5650 - stack overflow converting between specializations")
|
||||
// members that the two object types enumerate in different orders.
|
||||
const auto bury = [](nlohmann::ordered_json value)
|
||||
{
|
||||
for (std::size_t i = 0; i < 200; ++i)
|
||||
for (std::size_t i = 0; i < nlohmann::detail::recursion_depth_limit() + 72; ++i)
|
||||
{
|
||||
value = nlohmann::ordered_json::array({std::move(value)});
|
||||
}
|
||||
@@ -124,7 +124,7 @@ TEST_CASE("issue #5650 - stack overflow converting between specializations")
|
||||
const auto dig = [](const json & value)
|
||||
{
|
||||
const json* current = &value;
|
||||
for (std::size_t i = 0; i < 200; ++i)
|
||||
for (std::size_t i = 0; i < nlohmann::detail::recursion_depth_limit() + 72; ++i)
|
||||
{
|
||||
current = ¤t->at(0);
|
||||
}
|
||||
|
||||
@@ -930,14 +930,15 @@ TEST_CASE("JSON Merge Patch diff of deeply nested values")
|
||||
SECTION("the patch is the same past the descent bound")
|
||||
{
|
||||
// depths on either side of the nesting depth up to which merge_diff()
|
||||
// recurses (detail::recursion_depth_limit(), 128); not every depth up
|
||||
// recurses (detail::recursion_depth_limit(), 128 by default); not every depth up
|
||||
// to 300, to keep the test fast under Valgrind
|
||||
std::vector<std::size_t> depths;
|
||||
for (std::size_t depth = 0; depth <= 16; ++depth)
|
||||
{
|
||||
depths.push_back(depth);
|
||||
}
|
||||
for (std::size_t depth = 120; depth <= 136; ++depth)
|
||||
const std::size_t limit = nlohmann::detail::recursion_depth_limit();
|
||||
for (std::size_t depth = limit > 8 ? limit - 8 : 0; depth <= limit + 8; ++depth)
|
||||
{
|
||||
depths.push_back(depth);
|
||||
}
|
||||
|
||||
@@ -302,7 +302,7 @@ TEST_CASE("JSON Merge Patch on deeply nested values")
|
||||
SECTION("patching past the descent bound gives the same result")
|
||||
{
|
||||
// every depth on either side of where the iterative version takes
|
||||
// over (detail::recursion_depth_limit(), 128)
|
||||
// over (detail::recursion_depth_limit(), 128 by default)
|
||||
for (std::size_t depth = 0; depth <= 300; ++depth)
|
||||
{
|
||||
CAPTURE(depth)
|
||||
@@ -396,7 +396,7 @@ TEST_CASE("merge_patch() with an argument that aliases *this (#5641)")
|
||||
SECTION("a patch nested past the iterative descent bound aliases *this")
|
||||
{
|
||||
// every depth on either side of where the iterative version takes
|
||||
// over (detail::recursion_depth_limit(), 128); patching *this with
|
||||
// over (detail::recursion_depth_limit(), 128 by default); patching *this with
|
||||
// itself is idempotent, aliased or not
|
||||
for (const std::size_t depth :
|
||||
{
|
||||
|
||||
@@ -1124,7 +1124,7 @@ TEST_CASE("update() on deeply nested values")
|
||||
SECTION("merging past the descent bound gives the same result")
|
||||
{
|
||||
// every depth on either side of where the iterative version takes
|
||||
// over (detail::recursion_depth_limit(), 128)
|
||||
// over (detail::recursion_depth_limit(), 128 by default)
|
||||
for (std::size_t depth = 0; depth <= 300; ++depth)
|
||||
{
|
||||
CAPTURE(depth)
|
||||
@@ -1215,7 +1215,7 @@ TEST_CASE("update() with an argument that aliases *this (#5641)")
|
||||
SECTION("nested past the iterative descent bound aliases *this")
|
||||
{
|
||||
// every depth on either side of where the iterative version takes
|
||||
// over (detail::recursion_depth_limit(), 128); merging *this into
|
||||
// over (detail::recursion_depth_limit(), 128 by default); merging *this into
|
||||
// itself is idempotent, aliased or not
|
||||
for (const std::size_t depth :
|
||||
{
|
||||
|
||||
@@ -33,7 +33,6 @@ using nlohmann::json;
|
||||
#include "round_trip_corpus.hpp"
|
||||
#include "test_utils.hpp"
|
||||
#include "sax_countdown.hpp"
|
||||
#include "allocation_counter.hpp"
|
||||
#include "sax_event_loggers.hpp"
|
||||
using utils::SaxCountdown;
|
||||
using utils::SaxEventLogger;
|
||||
@@ -1760,47 +1759,6 @@ TEST_CASE("issue #5405 - array reserve for definite-length MessagePack arrays")
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("MessagePack - open arrays reserve only what the input has backed")
|
||||
{
|
||||
// an array that announces its size gets storage for (up to 16384 of) its
|
||||
// elements when it opens; the arrays that are open at the same time must
|
||||
// not hold more than the input read so far can back
|
||||
SECTION("nested headers that announce 16384 elements and are never completed")
|
||||
{
|
||||
// 200 levels would hold 200 * 16384 * sizeof(json),
|
||||
// if each of them reserved what it announces
|
||||
const auto input = utils::repeated({0xDC, 0x40, 0x00}, 200);
|
||||
|
||||
utils::counts() = utils::allocation_counts{};
|
||||
const auto j = utils::counted_json::from_msgpack(input, true, false);
|
||||
CHECK(j.is_discarded());
|
||||
CAPTURE(utils::counts().peak)
|
||||
// the credit is 65,536 values; the base peaks at more than 3 million
|
||||
CHECK(utils::counts().peak < sizeof(utils::counted_json) * 128 * 1024);
|
||||
}
|
||||
|
||||
SECTION("a completed array is still reserved in one allocation")
|
||||
{
|
||||
json array = json::array();
|
||||
for (int i = 0; i < 10000; ++i)
|
||||
{
|
||||
array.push_back(i % 100);
|
||||
}
|
||||
|
||||
{
|
||||
const auto packed = json::to_msgpack(array);
|
||||
|
||||
utils::counts() = utils::allocation_counts{};
|
||||
const auto j = utils::counted_json::from_msgpack(packed);
|
||||
CHECK(j.size() == 10000);
|
||||
// the array and its storage; growing the storage as the elements
|
||||
// arrive would take more than a dozen allocations
|
||||
CAPTURE(utils::counts().allocations)
|
||||
CHECK(utils::counts().allocations <= 4);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("regression test - MessagePack ext type rejects a subtype that doesn't fit a single byte")
|
||||
{
|
||||
// subtype 0-255 must still round-trip correctly (regression guard, pre-existing behavior)
|
||||
|
||||
@@ -0,0 +1,600 @@
|
||||
// __ _____ _____ _____
|
||||
// __| | __| | | | JSON for Modern C++ (supporting code)
|
||||
// | | |__ | | | | | | version 3.12.0
|
||||
// |_____|_____|_____|_|___| https://github.com/nlohmann/json
|
||||
//
|
||||
// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
#include "doctest_compatibility.h"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
using nlohmann::json;
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <iomanip>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
// JSON_RECURSION_DEPTH_LIMIT changes only how much call stack the walks over a
|
||||
// value use, never what they return. This file checks both: the macro is
|
||||
// honoured (the number of levels the containers handle themselves is the limit)
|
||||
// and every walk returns what it returns with the default limit. The expected
|
||||
// results are built here, without the library, so that the same file passes for
|
||||
// every limit; tests/CMakeLists.txt builds it with limits 0, 1, and 254 besides
|
||||
// the default. Values are nested up to 300 levels (and past every limit, so that
|
||||
// the explicit stack of every walk is exercised, even for the limit of 254) and
|
||||
// are built and taken apart without recursion. Keep the depths moderate: walks
|
||||
// like diff and dump with indentation are quadratic in the depth.
|
||||
|
||||
namespace
|
||||
{
|
||||
constexpr std::size_t limit()
|
||||
{
|
||||
return nlohmann::detail::recursion_depth_limit();
|
||||
}
|
||||
|
||||
// a value nested as @a kinds says, outermost level first: 'a' is an array of
|
||||
// one element, 'o' an object with the single key "a"; @a leaf is innermost
|
||||
json build(const std::string& kinds, json leaf)
|
||||
{
|
||||
json j = std::move(leaf);
|
||||
for (std::size_t i = kinds.size(); i-- > 0;)
|
||||
{
|
||||
if (kinds[i] == 'a')
|
||||
{
|
||||
json a = json::array();
|
||||
a.push_back(std::move(j));
|
||||
j = std::move(a);
|
||||
}
|
||||
else
|
||||
{
|
||||
json o = json::object();
|
||||
o["a"] = std::move(j);
|
||||
j = std::move(o);
|
||||
}
|
||||
}
|
||||
return j;
|
||||
}
|
||||
|
||||
std::string repeat(const std::string& s, std::size_t n)
|
||||
{
|
||||
std::string result;
|
||||
for (std::size_t i = 0; i < n; ++i)
|
||||
{
|
||||
result += s;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// the nesting of the given shape and depth: arrays, objects, or both alternating
|
||||
std::string make_kinds(char shape, std::size_t depth)
|
||||
{
|
||||
std::string kinds;
|
||||
for (std::size_t i = 0; i < depth; ++i)
|
||||
{
|
||||
if (shape == 'm')
|
||||
{
|
||||
kinds += i % 2 == 0 ? 'a' : 'o';
|
||||
}
|
||||
else
|
||||
{
|
||||
kinds += shape;
|
||||
}
|
||||
}
|
||||
return kinds;
|
||||
}
|
||||
|
||||
std::string expected_compact(const std::string& kinds, const std::string& leaf)
|
||||
{
|
||||
std::string s;
|
||||
for (const char kind : kinds)
|
||||
{
|
||||
s += kind == 'a' ? "[" : "{\"a\":";
|
||||
}
|
||||
s += leaf;
|
||||
for (std::size_t i = kinds.size(); i-- > 0;)
|
||||
{
|
||||
s += kinds[i] == 'a' ? "]" : "}";
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
// dump(1)
|
||||
std::string expected_indented(const std::string& kinds, const std::string& leaf)
|
||||
{
|
||||
std::string s;
|
||||
for (std::size_t i = 0; i < kinds.size(); ++i)
|
||||
{
|
||||
s += kinds[i] == 'a' ? "[\n" : "{\n";
|
||||
s += std::string(i + 1, ' ');
|
||||
if (kinds[i] == 'o')
|
||||
{
|
||||
s += "\"a\": ";
|
||||
}
|
||||
}
|
||||
s += leaf;
|
||||
for (std::size_t i = kinds.size(); i-- > 0;)
|
||||
{
|
||||
s += "\n" + std::string(i, ' ') + (kinds[i] == 'a' ? "]" : "}");
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
// the JSON pointer to the innermost value
|
||||
std::string pointer(const std::string& kinds)
|
||||
{
|
||||
std::string s;
|
||||
for (const char kind : kinds)
|
||||
{
|
||||
s += kind == 'a' ? "/0" : "/a";
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
// the binary formats: the bytes that open and close a level, and the leaf 1
|
||||
std::vector<std::uint8_t> expected_cbor(const std::string& kinds)
|
||||
{
|
||||
std::vector<std::uint8_t> bytes;
|
||||
for (const char kind : kinds)
|
||||
{
|
||||
if (kind == 'a')
|
||||
{
|
||||
bytes.push_back(0x81);
|
||||
}
|
||||
else
|
||||
{
|
||||
bytes.push_back(0xA1);
|
||||
bytes.push_back(0x61);
|
||||
bytes.push_back('a');
|
||||
}
|
||||
}
|
||||
bytes.push_back(0x01);
|
||||
return bytes;
|
||||
}
|
||||
|
||||
std::vector<std::uint8_t> expected_msgpack(const std::string& kinds)
|
||||
{
|
||||
std::vector<std::uint8_t> bytes;
|
||||
for (const char kind : kinds)
|
||||
{
|
||||
if (kind == 'a')
|
||||
{
|
||||
bytes.push_back(0x91);
|
||||
}
|
||||
else
|
||||
{
|
||||
bytes.push_back(0x81);
|
||||
bytes.push_back(0xA1);
|
||||
bytes.push_back('a');
|
||||
}
|
||||
}
|
||||
bytes.push_back(0x01);
|
||||
return bytes;
|
||||
}
|
||||
|
||||
std::vector<std::uint8_t> expected_ubjson(const std::string& kinds)
|
||||
{
|
||||
std::vector<std::uint8_t> bytes;
|
||||
for (const char kind : kinds)
|
||||
{
|
||||
if (kind == 'a')
|
||||
{
|
||||
bytes.push_back('[');
|
||||
}
|
||||
else
|
||||
{
|
||||
bytes.push_back('{');
|
||||
bytes.push_back('i');
|
||||
bytes.push_back(1);
|
||||
bytes.push_back('a');
|
||||
}
|
||||
}
|
||||
bytes.push_back('i');
|
||||
bytes.push_back(1);
|
||||
for (std::size_t i = kinds.size(); i-- > 0;)
|
||||
{
|
||||
bytes.push_back(kinds[i] == 'a' ? ']' : '}');
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
// the depths worth trying for the limit: either side of it, twice it (if that
|
||||
// is not too deep to be quick), and a value nested well past all limits
|
||||
std::vector<std::size_t> depths()
|
||||
{
|
||||
std::vector<std::size_t> result;
|
||||
result.push_back(1);
|
||||
result.push_back(2);
|
||||
result.push_back(limit() > 0 ? limit() - 1 : 3);
|
||||
result.push_back(limit());
|
||||
result.push_back(limit() + 1);
|
||||
if ((2 * limit()) + 5 <= 300)
|
||||
{
|
||||
result.push_back((2 * limit()) + 5);
|
||||
}
|
||||
result.push_back(300);
|
||||
std::sort(result.begin(), result.end());
|
||||
result.erase(std::unique(result.begin(), result.end()), result.end());
|
||||
result.erase(std::remove(result.begin(), result.end(), static_cast<std::size_t>(0)), result.end());
|
||||
return result;
|
||||
}
|
||||
|
||||
// the call for every shape and depth
|
||||
void for_each_value(const std::function<void(const std::string&)>& check)
|
||||
{
|
||||
const std::string shapes = "aom";
|
||||
for (const char shape : shapes)
|
||||
{
|
||||
for (const std::size_t depth : depths())
|
||||
{
|
||||
CAPTURE(shape)
|
||||
CAPTURE(depth)
|
||||
check(make_kinds(shape, depth));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An array type that counts how many of its copies are being made at once. A
|
||||
// copy of a value copies one level at a time through the array type until the
|
||||
// limit has been reached, so the peak is the number of levels that were copied
|
||||
// with the call stack.
|
||||
std::size_t copies_going_on = 0;
|
||||
std::size_t copies_peak = 0;
|
||||
std::size_t equal_going_on = 0;
|
||||
std::size_t equal_peak = 0;
|
||||
|
||||
template<class T, class Allocator = std::allocator<T>>
|
||||
class counting_vector : public std::vector<T, Allocator>
|
||||
{
|
||||
public:
|
||||
counting_vector() = default;
|
||||
|
||||
template<class InputIt>
|
||||
counting_vector(InputIt first, InputIt last) : std::vector<T, Allocator>(first, last) {}
|
||||
|
||||
counting_vector(const counting_vector& other) : std::vector<T, Allocator>()
|
||||
{
|
||||
++copies_going_on;
|
||||
copies_peak = (std::max)(copies_peak, copies_going_on);
|
||||
this->std::vector<T, Allocator>::operator=(static_cast<const std::vector<T, Allocator>&>(other));
|
||||
--copies_going_on;
|
||||
}
|
||||
|
||||
counting_vector(counting_vector&&) = default;
|
||||
counting_vector& operator=(const counting_vector&) = default;
|
||||
counting_vector& operator=(counting_vector&&) = default;
|
||||
~counting_vector() = default;
|
||||
|
||||
friend bool operator==(const counting_vector& lhs, const counting_vector& rhs)
|
||||
{
|
||||
++equal_going_on;
|
||||
equal_peak = (std::max)(equal_peak, equal_going_on);
|
||||
const bool result = static_cast<const std::vector<T, Allocator>&>(lhs) == static_cast<const std::vector<T, Allocator>&>(rhs);
|
||||
--equal_going_on;
|
||||
return result;
|
||||
}
|
||||
};
|
||||
|
||||
using counting_json = json::with_array_t<counting_vector>;
|
||||
|
||||
// A base class whose copy constructor copies a whole value of the same
|
||||
// specialization when it copies a marked element, as user code may do from a
|
||||
// hook. That copy runs while the copy that it is part of is going on.
|
||||
struct copy_hook // NOLINT(cppcoreguidelines-special-member-functions,hicpp-special-member-functions)
|
||||
{
|
||||
int mark = 0;
|
||||
|
||||
copy_hook() = default;
|
||||
copy_hook(const copy_hook& other);
|
||||
copy_hook(copy_hook&&) = default;
|
||||
copy_hook& operator=(const copy_hook&) = default;
|
||||
copy_hook& operator=(copy_hook&&) = default;
|
||||
};
|
||||
|
||||
using hook_json = nlohmann::basic_json<std::map, counting_vector, std::string, bool, std::int64_t, std::uint64_t, double,
|
||||
std::allocator, nlohmann::adl_serializer, std::vector<std::uint8_t>, copy_hook>;
|
||||
|
||||
const hook_json* hook_copy_source = nullptr;
|
||||
int hook_copies_going_on = 0;
|
||||
|
||||
copy_hook::copy_hook(const copy_hook& other) : mark(other.mark)
|
||||
{
|
||||
// the value that is copied is marked, too: its copy copies it again, once
|
||||
if (other.mark == 1 && hook_copy_source != nullptr && hook_copies_going_on < 2)
|
||||
{
|
||||
++hook_copies_going_on;
|
||||
const hook_json nested = *hook_copy_source; // NOLINT(performance-unnecessary-copy-initialization)
|
||||
static_cast<void>(nested);
|
||||
--hook_copies_going_on;
|
||||
}
|
||||
}
|
||||
|
||||
// an array nested @a depth levels around the integer 1, which is marked if @a marked
|
||||
hook_json hook_nested(std::size_t depth, bool marked)
|
||||
{
|
||||
hook_json j = 1;
|
||||
if (marked)
|
||||
{
|
||||
static_cast<copy_hook&>(j).mark = 1;
|
||||
}
|
||||
for (std::size_t i = 0; i < depth; ++i)
|
||||
{
|
||||
hook_json a = hook_json::array();
|
||||
a.push_back(std::move(j));
|
||||
j = std::move(a);
|
||||
}
|
||||
return j;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
// the macro is the limit, and is in the range the library supports
|
||||
static_assert(nlohmann::detail::recursion_depth_limit() == static_cast<std::size_t>(JSON_RECURSION_DEPTH_LIMIT),
|
||||
"the library does not use JSON_RECURSION_DEPTH_LIMIT");
|
||||
static_assert(nlohmann::detail::recursion_depth_limit() <= 254, "the nesting depth must fit in a byte");
|
||||
|
||||
TEST_CASE("JSON_RECURSION_DEPTH_LIMIT is honoured")
|
||||
{
|
||||
SECTION("copying and comparing stop descending at the limit")
|
||||
{
|
||||
// copy and compare a value nested deeper than the limit, with the array type that counts
|
||||
counting_json inner = 1;
|
||||
for (std::size_t i = 0; i < (2 * limit()) + 5; ++i)
|
||||
{
|
||||
counting_json a = counting_json::array();
|
||||
a.push_back(std::move(inner));
|
||||
inner = std::move(a);
|
||||
}
|
||||
|
||||
copies_going_on = copies_peak = 0;
|
||||
const counting_json copy = inner;
|
||||
CHECK(copies_going_on == 0);
|
||||
|
||||
equal_going_on = equal_peak = 0;
|
||||
CHECK((inner == copy));
|
||||
CHECK(equal_going_on == 0);
|
||||
|
||||
#ifdef JSON_NO_THREAD_LOCAL
|
||||
// without the counters of the levels, no level is descended into
|
||||
CHECK(copies_peak == 0);
|
||||
CHECK(equal_peak == 0);
|
||||
#else
|
||||
// copying descends into as many levels as the limit says, and nothing further
|
||||
CHECK(copies_peak == limit());
|
||||
CHECK(equal_peak == limit());
|
||||
#endif
|
||||
|
||||
// a value that is not nested deeper than the limit is not cut short
|
||||
const std::size_t shallow = limit() > 0 ? limit() : 1;
|
||||
counting_json small = 1;
|
||||
for (std::size_t i = 0; i < shallow; ++i)
|
||||
{
|
||||
counting_json a = counting_json::array();
|
||||
a.push_back(std::move(small));
|
||||
small = std::move(a);
|
||||
}
|
||||
copies_going_on = copies_peak = 0;
|
||||
const counting_json small_copy = small;
|
||||
CHECK(copies_going_on == 0);
|
||||
#ifndef JSON_NO_THREAD_LOCAL
|
||||
CHECK(copies_peak == limit());
|
||||
#endif
|
||||
CHECK(small_copy.dump() == expected_compact(make_kinds('a', shallow), "1"));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("a copy made while a copy is going on is bounded, too")
|
||||
{
|
||||
// The innermost element of the value is marked, and copying it (which happens
|
||||
// after limit() levels have been entered, when the copy continues without the
|
||||
// call stack) copies the value again, and that copy once more. Those copies
|
||||
// must not be given limit() levels of their own: the count of the levels entered
|
||||
// never exceeds the limit, not even by one, which for the largest limit would
|
||||
// wrap the byte that holds it around to 0.
|
||||
const hook_json marked = hook_nested((2 * limit()) + 5, true);
|
||||
hook_copy_source = &marked;
|
||||
|
||||
copies_going_on = copies_peak = 0;
|
||||
const hook_json copy = marked; // NOLINT(performance-unnecessary-copy-initialization)
|
||||
hook_copy_source = nullptr;
|
||||
CHECK(copies_going_on == 0);
|
||||
CHECK(static_cast<const copy_hook&>(copy).mark == 0); // (the top level is not marked)
|
||||
|
||||
#ifdef JSON_NO_THREAD_LOCAL
|
||||
CHECK(copies_peak == 0);
|
||||
#else
|
||||
CHECK(copies_peak == limit());
|
||||
#endif
|
||||
}
|
||||
|
||||
TEST_CASE("walks over values nested around the recursion depth limit")
|
||||
{
|
||||
SECTION("copying and comparing")
|
||||
{
|
||||
for_each_value([](const std::string & kinds)
|
||||
{
|
||||
const json a = build(kinds, 1);
|
||||
const json b = a; // NOLINT(performance-unnecessary-copy-initialization)
|
||||
CHECK((a == b));
|
||||
CHECK_FALSE((a != b));
|
||||
CHECK_FALSE((a < b));
|
||||
CHECK((a <= b));
|
||||
CHECK((a >= b));
|
||||
CHECK_FALSE((a > b));
|
||||
|
||||
json assigned;
|
||||
assigned = a;
|
||||
CHECK((assigned == a));
|
||||
|
||||
// differing only in the innermost value
|
||||
const json c = build(kinds, 2);
|
||||
CHECK((a != c));
|
||||
CHECK((a < c));
|
||||
CHECK((c > a));
|
||||
CHECK_FALSE((c < a));
|
||||
CHECK_FALSE((a == c));
|
||||
|
||||
// a same-shape array or object that is one level shallower
|
||||
const json shallower = build(kinds.substr(1), 1);
|
||||
CHECK((shallower != a));
|
||||
const json outer = build(kinds.substr(0, 1), 1);
|
||||
CHECK((outer != a) == (kinds.size() > 1));
|
||||
|
||||
const json moved = std::move(assigned);
|
||||
CHECK((moved == a));
|
||||
});
|
||||
}
|
||||
|
||||
SECTION("serializing")
|
||||
{
|
||||
for_each_value([](const std::string & kinds)
|
||||
{
|
||||
const json a = build(kinds, 1);
|
||||
const std::string compact = expected_compact(kinds, "1");
|
||||
const std::string indented = expected_indented(kinds, "1");
|
||||
|
||||
CHECK(a.dump() == compact);
|
||||
CHECK(a.dump(1) == indented);
|
||||
|
||||
std::ostringstream compact_stream;
|
||||
compact_stream << a;
|
||||
CHECK(compact_stream.str() == compact);
|
||||
std::ostringstream indented_stream;
|
||||
indented_stream << std::setw(1) << a;
|
||||
CHECK(indented_stream.str() == indented);
|
||||
|
||||
CHECK((json::parse(compact) == a));
|
||||
CHECK((json::parse(indented) == a));
|
||||
});
|
||||
}
|
||||
|
||||
SECTION("hashing")
|
||||
{
|
||||
for_each_value([](const std::string & kinds)
|
||||
{
|
||||
const json a = build(kinds, 1);
|
||||
const std::hash<json> hash;
|
||||
CHECK(hash(a) == hash(json(a)));
|
||||
CHECK(hash(a) == hash(json::parse(expected_compact(kinds, "1"))));
|
||||
CHECK(hash(a) != hash(build(kinds, 2)));
|
||||
CHECK(hash(a) != hash(build(kinds.substr(1), 1)));
|
||||
});
|
||||
}
|
||||
|
||||
SECTION("diff and patch")
|
||||
{
|
||||
for_each_value([](const std::string & kinds)
|
||||
{
|
||||
const json a = build(kinds, 1);
|
||||
const json b = build(kinds, 2);
|
||||
|
||||
const json difference = json::diff(a, b);
|
||||
CHECK(difference.dump() == "[{\"op\":\"replace\",\"path\":\"" + pointer(kinds) + "\",\"value\":2}]");
|
||||
CHECK((a.patch(difference) == b));
|
||||
CHECK((json::diff(a, a) == json::array()));
|
||||
});
|
||||
}
|
||||
|
||||
SECTION("flatten and unflatten")
|
||||
{
|
||||
for_each_value([](const std::string & kinds)
|
||||
{
|
||||
const json a = build(kinds, 1);
|
||||
const json flat = a.flatten();
|
||||
CHECK(flat.dump() == "{\"" + pointer(kinds) + "\":1}");
|
||||
CHECK((flat.unflatten() == a));
|
||||
});
|
||||
}
|
||||
|
||||
SECTION("merge_patch and update")
|
||||
{
|
||||
for (const std::size_t depth : depths())
|
||||
{
|
||||
CAPTURE(depth)
|
||||
const std::string kinds = make_kinds('o', depth);
|
||||
const std::string all_but_leaf = repeat("{\"a\":", depth - 1);
|
||||
|
||||
// the patch removes the innermost member, or replaces it
|
||||
json target = build(kinds, 1);
|
||||
target.merge_patch(build(kinds, json(nullptr)));
|
||||
CHECK(target.dump() == all_but_leaf + "{}" + repeat("}", depth - 1));
|
||||
|
||||
target = build(kinds, 1);
|
||||
target.merge_patch(build(kinds, 2));
|
||||
CHECK(target.dump() == expected_compact(kinds, "2"));
|
||||
|
||||
// merging the objects of two values nested alike, and replacing the first level
|
||||
const json base = build(kinds.substr(1), json::parse("{\"x\":1}"));
|
||||
const json other = build(kinds.substr(1), json::parse("{\"y\":2}"));
|
||||
json merged = json{{"a", base}};
|
||||
merged.update(json{{"a", other}}, true);
|
||||
CHECK(merged.dump() == expected_compact(kinds, "{\"x\":1,\"y\":2}"));
|
||||
|
||||
json replaced = json{{"a", base}};
|
||||
replaced.update(json{{"a", other}});
|
||||
CHECK(replaced.dump() == expected_compact(kinds, "{\"y\":2}"));
|
||||
|
||||
// update from a range
|
||||
json from_range = json{{"a", base}};
|
||||
const json source = json{{"a", other}};
|
||||
from_range.update(source.begin(), source.end(), true);
|
||||
CHECK((from_range == merged));
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("binary formats")
|
||||
{
|
||||
for_each_value([](const std::string & kinds)
|
||||
{
|
||||
const json a = build(kinds, 1);
|
||||
|
||||
const std::vector<std::uint8_t> cbor = json::to_cbor(a);
|
||||
CHECK(cbor == expected_cbor(kinds));
|
||||
CHECK((json::from_cbor(cbor) == a));
|
||||
|
||||
const std::vector<std::uint8_t> msgpack = json::to_msgpack(a);
|
||||
CHECK(msgpack == expected_msgpack(kinds));
|
||||
CHECK((json::from_msgpack(msgpack) == a));
|
||||
|
||||
const std::vector<std::uint8_t> ubjson = json::to_ubjson(a);
|
||||
CHECK(ubjson == expected_ubjson(kinds));
|
||||
CHECK((json::from_ubjson(ubjson) == a));
|
||||
|
||||
CHECK((json::from_bjdata(json::to_bjdata(a)) == a));
|
||||
CHECK((json::from_bon8(json::to_bon8(a)) == a));
|
||||
|
||||
// BSON documents are objects
|
||||
const json document = json{{"a", a}};
|
||||
CHECK((json::from_bson(json::to_bson(document)) == document));
|
||||
});
|
||||
}
|
||||
|
||||
SECTION("conversion to another specialization")
|
||||
{
|
||||
for_each_value([](const std::string & kinds)
|
||||
{
|
||||
const json a = build(kinds, 1);
|
||||
const nlohmann::ordered_json ordered = a;
|
||||
CHECK(ordered.dump() == expected_compact(kinds, "1"));
|
||||
const json back = ordered;
|
||||
CHECK((back == a));
|
||||
});
|
||||
}
|
||||
|
||||
SECTION("destroying")
|
||||
{
|
||||
for_each_value([](const std::string & kinds)
|
||||
{
|
||||
json a = build(kinds, 1);
|
||||
a = nullptr;
|
||||
CHECK(a.is_null());
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1554,8 +1554,10 @@ TEST_CASE("regression tests 1")
|
||||
SECTION("a missing member of an object")
|
||||
{
|
||||
my_json j = make();
|
||||
#ifdef JSON_NO_THREAD_LOCAL
|
||||
// (set by the user, or by the library for clang with MinGW)
|
||||
#if defined(JSON_NO_THREAD_LOCAL) || JSON_RECURSION_DEPTH_LIMIT < 2
|
||||
// (set by the user, or by the library for clang with MinGW; or fewer than two
|
||||
// levels are copied by the object type itself, as for a recursion depth limit
|
||||
// of 0 or 1: the objects below the outermost one are the second level)
|
||||
// the objects below the outermost one are not copied by their
|
||||
// object_t here, but member by member, which keeps their
|
||||
// timestamps correct: "c" is not mistaken for "b" and is inserted
|
||||
|
||||
@@ -27,9 +27,7 @@ using nlohmann::json;
|
||||
#include "round_trip_corpus.hpp"
|
||||
#include "test_utils.hpp"
|
||||
#include "sax_countdown.hpp"
|
||||
#include "allocation_counter.hpp"
|
||||
#include "sax_event_loggers.hpp"
|
||||
#include "ubjson_valueless.hpp"
|
||||
using utils::SaxCountdown;
|
||||
using utils::SaxEventLogger;
|
||||
|
||||
@@ -2438,155 +2436,6 @@ TEST_CASE("UBJSON optimized arrays of a valueless type are bounded")
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("UBJSON the writer keeps the valueless elements of an input within the base")
|
||||
{
|
||||
// The budget of the reader is 1048576 plus a rate per byte read, see
|
||||
// unit-ubjson_valueless.cpp, which tests it for several values of the rate.
|
||||
// The writer does not use the rate and stays within the base
|
||||
json _;
|
||||
|
||||
SECTION("to_ubjson writes the typed form up to the base only")
|
||||
{
|
||||
// 1048 rows of 1000 values fit under the base of 1048576, the 1049th
|
||||
// would not: its elements are written one by one. Written all typed,
|
||||
// 1100 rows of 6 bytes would be refused by a reader with the default
|
||||
// rate of the macro JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE (1074 rows
|
||||
// are the most it accepts); the writer does not follow the macro, since
|
||||
// the limit of a reader is never below the base. The rows hold null,
|
||||
// true or false, and the fallback does not depend on which
|
||||
std::vector<std::uint8_t> encoded;
|
||||
{
|
||||
json table = json::array();
|
||||
for (int row = 0; row < 1100; ++row)
|
||||
{
|
||||
const json cell = row % 3 == 0 ? json(nullptr) : json(row % 3 == 1);
|
||||
table.push_back(json(1000, cell));
|
||||
}
|
||||
encoded = json::to_ubjson(table, true, true);
|
||||
}
|
||||
// outer array: `[$[#I` and the count (7 bytes), then 1048 typed rows of 6
|
||||
// bytes (`$Z#I` and two bytes) and 52 rows of `#I`, two bytes, and 1000 `Z`
|
||||
CHECK(encoded.size() == 7 + (1048 * 6) + (52 * (4 + 1000)));
|
||||
|
||||
// the output is read back
|
||||
utils::ValuelessEventCounter sax;
|
||||
CHECK(json::sax_parse(encoded, &sax, json::input_format_t::ubjson));
|
||||
CHECK(sax.error_id == 0);
|
||||
CHECK(sax.values == 1100u * 1000u);
|
||||
}
|
||||
|
||||
SECTION("BJData is not affected")
|
||||
{
|
||||
// BJData has no optimized arrays of these types, and the writer does
|
||||
// not charge them
|
||||
json table = json::array();
|
||||
for (int row = 0; row < 3; ++row)
|
||||
{
|
||||
table.push_back(json(10, nullptr));
|
||||
}
|
||||
const auto encoded = json::to_bjdata(table, true, true);
|
||||
CHECK(json::from_bjdata(encoded) == table);
|
||||
|
||||
const std::vector<std::uint8_t> input = {'[', '$', 'Z', '#', 'l', 0x00, 0x10, 0x00, 0x00};
|
||||
CHECK_THROWS_AS(_ = json::from_bjdata(input), json::parse_error&);
|
||||
}
|
||||
|
||||
SECTION("an object of a valueless type needs a key for every value")
|
||||
{
|
||||
// an object member costs a key, so the count is bounded by the input
|
||||
// and is not charged to the budget
|
||||
const std::vector<std::uint8_t> input = {'{', '$', 'Z', '#', 'l', 0x00, 0x10, 0x00, 0x00};
|
||||
utils::ValuelessEventCounter sax;
|
||||
CHECK(!json::sax_parse(input, &sax, json::input_format_t::ubjson));
|
||||
CHECK(sax.error_id == 110);
|
||||
CHECK(sax.values == 0);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("UBJSON an optimized array of no-ops is empty whatever its count")
|
||||
{
|
||||
// A no-op ('N') is not a value, so `[$N#<n>` is an array without elements
|
||||
// for every n. The count used to be passed on to the SAX parser, which
|
||||
// told it to expect n elements, and the DOM parser reserved storage for
|
||||
// them: 16384 values (256 KB) for a count of 2^31 in 9 bytes of input.
|
||||
SECTION("the SAX parser is told that the array is empty")
|
||||
{
|
||||
for (const std::vector<std::uint8_t>& input :
|
||||
{
|
||||
std::vector<std::uint8_t> {'[', '$', 'N', '#', 'i', 2},
|
||||
std::vector<std::uint8_t> {'[', '$', 'N', '#', 'I', 0x7F, 0xFF},
|
||||
std::vector<std::uint8_t> {'[', '$', 'N', '#', 'l', 0x7F, 0xFF, 0xFF, 0xFF},
|
||||
std::vector<std::uint8_t> {'[', '$', 'N', '#', 'i', 0}
|
||||
})
|
||||
{
|
||||
SaxEventLogger logger;
|
||||
CHECK(json::sax_parse(input, &logger, json::input_format_t::ubjson));
|
||||
CHECK(logger.events == std::vector<std::string>({"start_array(0)", "end_array()"}));
|
||||
}
|
||||
|
||||
// between other values, the next value is read as usual
|
||||
const std::vector<std::uint8_t> nested = {'[', '[', '$', 'N', '#', 'l', 0x7F, 0xFF, 0xFF, 0xFF, 'T', '[', '$', 'N', '#', 'i', 3, ']'};
|
||||
SaxEventLogger logger;
|
||||
CHECK(json::sax_parse(nested, &logger, json::input_format_t::ubjson));
|
||||
CHECK(logger.events == std::vector<std::string>({"start_array()", "start_array(0)", "end_array()", "boolean(true)", "start_array(0)", "end_array()", "end_array()"}));
|
||||
}
|
||||
|
||||
SECTION("the DOM parser reserves nothing")
|
||||
{
|
||||
// 2000 arrays of 9 bytes that announce 2^31 elements each: with a
|
||||
// reservation of 16384 values per array this took about 500 MB
|
||||
std::vector<std::uint8_t> input = {'['};
|
||||
for (int i = 0; i < 2000; ++i)
|
||||
{
|
||||
input.insert(input.end(), {'[', '$', 'N', '#', 'l', 0x7F, 0xFF, 0xFF, 0xFF});
|
||||
}
|
||||
input.push_back(']');
|
||||
|
||||
const json j = json::from_ubjson(input);
|
||||
REQUIRE(j.size() == 2000);
|
||||
std::size_t capacity = 0;
|
||||
for (const auto& element : j)
|
||||
{
|
||||
CHECK(element.is_array());
|
||||
CHECK(element.empty());
|
||||
capacity += element.get_ref<const json::array_t&>().capacity();
|
||||
}
|
||||
CHECK(capacity == 0);
|
||||
|
||||
// the 7-byte form inside an optimized array of arrays
|
||||
std::vector<std::uint8_t> typed = {'[', '$', '[', '#', 'I', 0x07, 0xD0};
|
||||
for (int i = 0; i < 2000; ++i)
|
||||
{
|
||||
typed.insert(typed.end(), {'$', 'N', '#', 'I', 0x7F, 0xFF});
|
||||
}
|
||||
const json k = json::from_ubjson(typed);
|
||||
REQUIRE(k.size() == 2000);
|
||||
capacity = 0;
|
||||
for (const auto& element : k)
|
||||
{
|
||||
CHECK(element.empty());
|
||||
capacity += element.get_ref<const json::array_t&>().capacity();
|
||||
}
|
||||
CHECK(capacity == 0);
|
||||
}
|
||||
|
||||
SECTION("a count that no container could hold is no longer an error")
|
||||
{
|
||||
// the array is empty, so nothing has to fit: 0x7FFFFFFF used to be
|
||||
// reported as out_of_range.408 where it exceeds the maximum size of
|
||||
// an array, and is accepted now
|
||||
const std::vector<std::uint8_t> input = {'[', '$', 'N', '#', 'l', 0x7F, 0xFF, 0xFF, 0xFF};
|
||||
CHECK(json::from_ubjson(input) == json::array());
|
||||
|
||||
// a count that does not fit into std::size_t still is
|
||||
#if SIZE_MAX == 0xffffffff
|
||||
const std::vector<std::uint8_t> too_large = {'[', '$', 'N', '#', 'L', 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00};
|
||||
json _;
|
||||
CHECK_THROWS_AS(_ = json::from_ubjson(too_large), json::out_of_range&);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("issue #5405 - array reserve for definite-length UBJSON arrays")
|
||||
{
|
||||
#if !defined(JSON_NOEXCEPTION)
|
||||
@@ -2692,60 +2541,6 @@ TEST_CASE("issue #5405 - array reserve for definite-length UBJSON arrays")
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("UBJSON - open arrays reserve only what the input has backed")
|
||||
{
|
||||
// an array that announces its size gets storage for (up to 16384 of) its
|
||||
// elements when it opens; the arrays that are open at the same time must
|
||||
// not hold more than the input read so far can back
|
||||
SECTION("nested headers that announce 16384 elements and are never completed")
|
||||
{
|
||||
// 200 levels would hold 200 * 16384 * sizeof(json),
|
||||
// if each of them reserved what it announces
|
||||
const auto input = utils::repeated({'[', '#', 'I', 0x40, 0x00}, 200);
|
||||
|
||||
utils::counts() = utils::allocation_counts{};
|
||||
const auto j = utils::counted_json::from_ubjson(input, true, false);
|
||||
CHECK(j.is_discarded());
|
||||
CAPTURE(utils::counts().peak)
|
||||
// the credit is 65,536 values; the base peaks at more than 3 million
|
||||
CHECK(utils::counts().peak < sizeof(utils::counted_json) * 128 * 1024);
|
||||
}
|
||||
|
||||
SECTION("a completed array is still reserved in one allocation")
|
||||
{
|
||||
json array = json::array();
|
||||
for (int i = 0; i < 10000; ++i)
|
||||
{
|
||||
array.push_back(i % 100);
|
||||
}
|
||||
|
||||
// an array without a size is not reserved, so only these two
|
||||
{
|
||||
const auto packed = json::to_ubjson(array, true);
|
||||
|
||||
utils::counts() = utils::allocation_counts{};
|
||||
const auto j = utils::counted_json::from_ubjson(packed);
|
||||
CHECK(j.size() == 10000);
|
||||
// the array and its storage; growing the storage as the elements
|
||||
// arrive would take more than a dozen allocations
|
||||
CAPTURE(utils::counts().allocations)
|
||||
CHECK(utils::counts().allocations <= 4);
|
||||
}
|
||||
|
||||
{
|
||||
const auto packed = json::to_ubjson(array, true, true);
|
||||
|
||||
utils::counts() = utils::allocation_counts{};
|
||||
const auto j = utils::counted_json::from_ubjson(packed);
|
||||
CHECK(j.size() == 10000);
|
||||
// the array and its storage; growing the storage as the elements
|
||||
// arrive would take more than a dozen allocations
|
||||
CAPTURE(utils::counts().allocations)
|
||||
CHECK(utils::counts().allocations <= 4);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
TEST_CASE("Universal Binary JSON Specification Examples 1")
|
||||
{
|
||||
|
||||
@@ -1,310 +0,0 @@
|
||||
// __ _____ _____ _____
|
||||
// __| | __| | | | JSON for Modern C++ (supporting code)
|
||||
// | | |__ | | | | | | version 3.12.0
|
||||
// |_____|_____|_____|_|___| https://github.com/nlohmann/json
|
||||
//
|
||||
// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
// The budget for the elements of the UBJSON containers of a valueless type
|
||||
// ('Z', 'T', 'F') of a whole input: 1048576 plus
|
||||
// JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE for every byte read (see
|
||||
// docs/mkdocs/docs/api/macros/json_ubjson_valueless_elements_per_byte.md).
|
||||
//
|
||||
// The file is built several times, with the macro defined to other values (see
|
||||
// tests/CMakeLists.txt), and every boundary below is computed from the macro.
|
||||
// What it can test depends on the value: with a rate of at most 10000 the
|
||||
// budget can be reached with the few bytes of a test, so the tests exceed it
|
||||
// at the exact byte; with a rate of 131072 or more the budget cannot be reached
|
||||
// by an input of any size (see the macro's page: 2^20 / 8 = 131072 elements per
|
||||
// byte is what the shortest header of a container of the largest size allows), and
|
||||
// the tests check that nothing is refused, in particular that the limit does
|
||||
// not wrap around for a rate near the largest integer. The input is read with
|
||||
// a SAX handler that counts the values, because a DOM of a million values is
|
||||
// what makes such a test slow under Valgrind.
|
||||
|
||||
#include "doctest_compatibility.h"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
using nlohmann::json;
|
||||
|
||||
#include <cstddef> // size_t
|
||||
#include <cstdio> // FILE, tmpfile, fwrite, fseek, fclose
|
||||
#include <cstdint> // uint8_t, uint32_t
|
||||
#include <list>
|
||||
#include <sstream>
|
||||
#include <vector>
|
||||
|
||||
#include "ubjson_valueless.hpp"
|
||||
using utils::ValuelessEventCounter;
|
||||
using utils::valueless_arrays;
|
||||
|
||||
#ifndef JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE
|
||||
#error "this test needs the macro: compile it with JSON_TEST_KEEP_MACROS (see tests/CMakeLists.txt)"
|
||||
#endif
|
||||
|
||||
namespace
|
||||
{
|
||||
/// the base of the budget and the limit of a single container: both 2^20
|
||||
constexpr std::uint32_t budget_base = 1048576;
|
||||
|
||||
#if JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE <= 10000 || JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE >= 131072
|
||||
/// what version 3.12.0 wrote for a table of @a rows (at most 32767) rows of
|
||||
/// @a width nulls with to_ubjson(table, true, true): `[$[#I` and the count of
|
||||
/// the rows (7 bytes), then a row as `$Z#` and its count (`U` and 1 byte up to
|
||||
/// 255, `I` and 2 bytes up to 32767, `l` and 4 bytes above): 5, 6 or 8 bytes
|
||||
std::vector<std::uint8_t> rows_3_12_0(const std::uint32_t rows, const std::uint32_t width)
|
||||
{
|
||||
std::vector<std::uint8_t> v = {'[', '$', '[', '#', 'I', static_cast<std::uint8_t>(rows >> 8), static_cast<std::uint8_t>(rows)};
|
||||
for (std::uint32_t row = 0; row < rows; ++row)
|
||||
{
|
||||
v.push_back('$');
|
||||
v.push_back('Z');
|
||||
v.push_back('#');
|
||||
if (width <= 255)
|
||||
{
|
||||
v.push_back('U');
|
||||
}
|
||||
else if (width <= 32767)
|
||||
{
|
||||
v.push_back('I');
|
||||
v.push_back(static_cast<std::uint8_t>(width >> 8));
|
||||
}
|
||||
else
|
||||
{
|
||||
v.push_back('l');
|
||||
v.push_back(static_cast<std::uint8_t>(width >> 24));
|
||||
v.push_back(static_cast<std::uint8_t>(width >> 16));
|
||||
v.push_back(static_cast<std::uint8_t>(width >> 8));
|
||||
}
|
||||
v.push_back(static_cast<std::uint8_t>(width));
|
||||
}
|
||||
return v;
|
||||
}
|
||||
#endif
|
||||
} // namespace
|
||||
|
||||
TEST_CASE("UBJSON the limit of a single container does not depend on the macro")
|
||||
{
|
||||
#if !defined(JSON_NOEXCEPTION)
|
||||
json _;
|
||||
|
||||
SECTION("a container of more than 1048576 elements is refused")
|
||||
{
|
||||
// the first check of every container; its message differs from that of
|
||||
// the budget, so that it is clear which limit was exceeded
|
||||
for (const char marker :
|
||||
{'Z', 'T', 'F'
|
||||
})
|
||||
{
|
||||
const auto input = valueless_arrays(marker, {budget_base + 1});
|
||||
CHECK_THROWS_WITH_AS(_ = json::from_ubjson(input), "[json.exception.out_of_range.408] syntax error while parsing UBJSON size: excessive array size", json::out_of_range&);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
SECTION("a container of 1048576 elements is accepted at the start of an input")
|
||||
{
|
||||
// whatever the rate: the base is the per-container limit. The handler
|
||||
// ends the parse at the first value, so that the million values are
|
||||
// not read
|
||||
const std::vector<std::uint8_t> single = {'[', '$', 'Z', '#', 'l', 0x00, 0x10, 0x00, 0x00};
|
||||
ValuelessEventCounter sax;
|
||||
sax.stop_after = 1;
|
||||
CHECK(!json::sax_parse(single, &sax, json::input_format_t::ubjson));
|
||||
CHECK(sax.error_id == 0);
|
||||
CHECK(sax.values == 1);
|
||||
}
|
||||
}
|
||||
|
||||
#if JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE <= 10000
|
||||
|
||||
namespace
|
||||
{
|
||||
/// the rate of the build; at most 10000 here, which the boundaries rely on
|
||||
constexpr std::uint32_t rate = JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE;
|
||||
|
||||
/// elements of the first of two containers; the second one is then refused at
|
||||
/// its header, which is read at byte 1 + 9 + 9 = 19
|
||||
constexpr std::uint32_t first_count = 524288;
|
||||
|
||||
/// the largest count the second container may have at byte 19: base plus rate
|
||||
/// for each of the 19 bytes, less what the first one used. For a rate of at
|
||||
/// most 27594, the largest that keeps this within the limit of one container
|
||||
constexpr std::uint32_t second_allowed = budget_base - first_count + (19 * rate);
|
||||
} // namespace
|
||||
|
||||
TEST_CASE("UBJSON the valueless elements of a whole input are bounded")
|
||||
{
|
||||
// The per-container limit alone let nine bytes (`[$Z#l` and a count of
|
||||
// 1048576) announce a million values, and every further nine bytes another
|
||||
// million. The elements of all containers of type 'Z', 'T' or 'F' in one
|
||||
// input are added up: they may not exceed 1048576 plus
|
||||
// JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE for every byte read when the
|
||||
// container's header has been read.
|
||||
#if !defined(JSON_NOEXCEPTION)
|
||||
json _;
|
||||
const char* const budget_message = "[json.exception.out_of_range.408] syntax error while parsing UBJSON size: excessive number of elements without payload";
|
||||
#endif
|
||||
|
||||
SECTION("the input of the original report is rejected")
|
||||
{
|
||||
// the second array is the first one that is refused, by one element
|
||||
#if !defined(JSON_NOEXCEPTION)
|
||||
CHECK_THROWS_WITH_AS(_ = json::from_ubjson(valueless_arrays('Z', {first_count, second_allowed + 1})), budget_message, json::out_of_range&);
|
||||
#endif
|
||||
|
||||
// and is accepted with one element less
|
||||
ValuelessEventCounter sax;
|
||||
CHECK(json::sax_parse(valueless_arrays('Z', {first_count, second_allowed}), &sax, json::input_format_t::ubjson));
|
||||
CHECK(sax.values == first_count + second_allowed);
|
||||
}
|
||||
|
||||
SECTION("every input adapter applies the budget")
|
||||
{
|
||||
const auto input = valueless_arrays('Z', {first_count, second_allowed + 1});
|
||||
|
||||
// stream; not strict, which does not matter
|
||||
std::stringstream stream;
|
||||
stream.write(reinterpret_cast<const char*>(input.data()), static_cast<std::streamsize>(input.size()));
|
||||
ValuelessEventCounter from_stream;
|
||||
CHECK(!json::sax_parse(stream, &from_stream, json::input_format_t::ubjson, false));
|
||||
CHECK(from_stream.error_id == 408);
|
||||
|
||||
// a container that is not contiguous; the counts of arrays of
|
||||
// different types are added up
|
||||
std::vector<std::uint8_t> mixed = {'['};
|
||||
utils::append_valueless_array(mixed, 'T', first_count);
|
||||
utils::append_valueless_array(mixed, 'F', second_allowed + 1);
|
||||
mixed.push_back(']');
|
||||
const std::list<std::uint8_t> as_list(mixed.begin(), mixed.end());
|
||||
ValuelessEventCounter from_list;
|
||||
CHECK(!json::sax_parse(as_list.begin(), as_list.end(), &from_list, json::input_format_t::ubjson));
|
||||
CHECK(from_list.error_id == 408);
|
||||
CHECK(from_list.values == first_count);
|
||||
|
||||
// FILE*
|
||||
FILE* const file = std::tmpfile(); // NOLINT(cppcoreguidelines-owning-memory)
|
||||
REQUIRE(file != nullptr);
|
||||
CHECK(std::fwrite(input.data(), 1, input.size(), file) == input.size());
|
||||
CHECK(std::fseek(file, 0, SEEK_SET) == 0);
|
||||
ValuelessEventCounter from_file;
|
||||
CHECK(!json::sax_parse(file, &from_file, json::input_format_t::ubjson));
|
||||
CHECK(from_file.error_id == 408);
|
||||
std::fclose(file); // NOLINT(cert-err33-c,cppcoreguidelines-owning-memory)
|
||||
}
|
||||
|
||||
SECTION("the limit is exact and the bytes read raise it")
|
||||
{
|
||||
// [ [$Z#l first_count] [$U#l payload + payload bytes] [$Z#l count] ]:
|
||||
// chars_read is 1 + 9 + (9 + payload) + 9 = 28 + payload when the last
|
||||
// header has been read, so the last array may hold
|
||||
// rate * (28 + payload) values on top of the base less the
|
||||
// first_count of the first array. The payload is as large as the
|
||||
// limit of one container allows, but at most 4096
|
||||
constexpr std::uint32_t payload_limit = (524288 / rate) - 40;
|
||||
constexpr std::uint32_t payload = payload_limit < 4096 ? payload_limit : 4096;
|
||||
constexpr std::uint32_t last_allowed = budget_base - first_count + (rate * (28 + payload));
|
||||
static_assert(last_allowed < budget_base, "the count must be accepted by the limit of one container");
|
||||
|
||||
for (const std::uint32_t count :
|
||||
{
|
||||
last_allowed, last_allowed + 1
|
||||
})
|
||||
{
|
||||
CAPTURE(count)
|
||||
std::vector<std::uint8_t> input = {'['};
|
||||
utils::append_valueless_array(input, 'Z', first_count);
|
||||
input.insert(input.end(), {'[', '$', 'U', '#', 'l',
|
||||
static_cast<std::uint8_t>(payload >> 24), static_cast<std::uint8_t>(payload >> 16),
|
||||
static_cast<std::uint8_t>(payload >> 8), static_cast<std::uint8_t>(payload)
|
||||
});
|
||||
input.insert(input.end(), payload, 0x01);
|
||||
utils::append_valueless_array(input, 'Z', count);
|
||||
input.push_back(']');
|
||||
ValuelessEventCounter sax;
|
||||
const bool accepted = count == last_allowed;
|
||||
CHECK(json::sax_parse(input, &sax, json::input_format_t::ubjson) == accepted);
|
||||
CHECK(sax.error_id == (accepted ? 0 : 408));
|
||||
CHECK(sax.values == first_count + payload + (accepted ? count : 0u));
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("a table written by 3.12.0")
|
||||
{
|
||||
// Version 3.12.0 wrote a table of rows of nulls with one `$Z#` header
|
||||
// per row (see the macro's page). A row of `width` elements has a
|
||||
// header of 6 bytes up to 32767 elements (5 up to 255), so the budget
|
||||
// grows by 6 * rate per row, and the row brings `width`. The 7 bytes
|
||||
// of the outer header are read first. Row r is accepted while
|
||||
// r * width <= 1048576 + rate * (7 + 6 * r),
|
||||
// which is r <= (1048576 + 7 * rate) / (width - 6 * rate) for a width
|
||||
// above 6 * rate, and always true otherwise.
|
||||
constexpr std::uint32_t width = 1000;
|
||||
constexpr std::uint32_t rows = 1200;
|
||||
constexpr std::uint32_t credit = 6 * rate;
|
||||
constexpr std::uint32_t accepted_rows = width > credit ? (budget_base + (7 * rate)) / (width - credit) : rows;
|
||||
constexpr std::uint32_t read_rows = accepted_rows < rows ? accepted_rows : rows;
|
||||
|
||||
ValuelessEventCounter sax;
|
||||
CHECK(json::sax_parse(rows_3_12_0(rows, width), &sax, json::input_format_t::ubjson) == (accepted_rows >= rows));
|
||||
CHECK(sax.error_id == (accepted_rows >= rows ? 0 : 408));
|
||||
CHECK(sax.values == static_cast<std::size_t>(read_rows) * width);
|
||||
}
|
||||
}
|
||||
|
||||
#elif JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE >= 131072
|
||||
|
||||
TEST_CASE("UBJSON a rate that cannot be reached switches the budget off")
|
||||
{
|
||||
// The rate is so large (2^20 / 8 = 131072 elements per byte or more)
|
||||
// that no input can exceed the budget: the shortest header of a container
|
||||
// of the largest size is 8 bytes (`$Z#l` and a count of 4 bytes, in an
|
||||
// array of arrays). The limit is base + rate * bytes read, which wraps
|
||||
// around for a rate near the largest integer in 64 and in 32
|
||||
// bits already at the first byte, unless it saturates: a wrapped limit
|
||||
// would refuse the second container below (at byte 19 or at byte 10, where
|
||||
// a limit of 2^20 + 10 * rate wraps for a 32-bit rate of 0x7FFFFFFF).
|
||||
#if !defined(JSON_NOEXCEPTION)
|
||||
json _;
|
||||
#endif
|
||||
|
||||
SECTION("containers of the largest size are accepted one after the other")
|
||||
{
|
||||
// the first container is accepted by the base alone, the second one is
|
||||
// not: it needs the rate. The handler ends the parse at its first
|
||||
// value, so that the second million values are not read
|
||||
const auto input = valueless_arrays('Z', {budget_base, budget_base});
|
||||
ValuelessEventCounter sax;
|
||||
sax.stop_after = static_cast<std::size_t>(budget_base) + 1;
|
||||
CHECK(!json::sax_parse(input, &sax, json::input_format_t::ubjson));
|
||||
CHECK(sax.error_id == 0);
|
||||
CHECK(sax.values == static_cast<std::size_t>(budget_base) + 1);
|
||||
}
|
||||
|
||||
SECTION("the limit of a single container still applies")
|
||||
{
|
||||
const auto input = valueless_arrays('T', {budget_base, budget_base + 1});
|
||||
ValuelessEventCounter sax;
|
||||
CHECK(!json::sax_parse(input, &sax, json::input_format_t::ubjson));
|
||||
CHECK(sax.error_id == 408);
|
||||
CHECK(sax.values == budget_base);
|
||||
#if !defined(JSON_NOEXCEPTION)
|
||||
CHECK_THROWS_WITH_AS(_ = json::from_ubjson(valueless_arrays('F', {budget_base + 1})), "[json.exception.out_of_range.408] syntax error while parsing UBJSON size: excessive array size", json::out_of_range&);
|
||||
#endif
|
||||
}
|
||||
|
||||
SECTION("a table written by 3.12.0 is read completely")
|
||||
{
|
||||
// 1200 rows of 1000 values are refused after 1074 rows at the default
|
||||
// rate and are accepted with every rate of 167 and more
|
||||
ValuelessEventCounter sax;
|
||||
CHECK(json::sax_parse(rows_3_12_0(1200, 1000), &sax, json::input_format_t::ubjson));
|
||||
CHECK(sax.error_id == 0);
|
||||
CHECK(sax.values == 1200u * 1000u);
|
||||
}
|
||||
}
|
||||
|
||||
// (rates from 10001 to 131071 can be reached, but not with the few bytes of a
|
||||
// test: there is nothing to test for them here)
|
||||
#endif
|
||||
Reference in new issue
Block a user