starred/json
1
0
Fork 0
mirror of https://github.com/nlohmann/json.git synced 2026-09-27 06:10:44 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Keep BJData ndarray annotations that would not survive a round trip as objects (#5542)

Browse Source 
write_bjdata_ndarray() encoded a JData-annotated object as a BJData
ND-array whenever its dimensions' product matched _ArrayData_.size(),
which lost information in two ways:

- _ArrayData_ was never required to be an array. null has size 0, any
  other scalar has size 1, and iterating an object visits its values, so
  e.g. {"_ArraySize_":[1],"_ArrayData_":5} was written as the array [5],
  and an object _ArrayData_ came back as an array.

- The reader only restores an annotated object from an ND-array with at
  least two non-zero dimensions that is not a 1xN row vector; an empty,
  1-D, row-vector, or zero-sized shape is read back as a plain array. The
  writer nonetheless emitted ND-array headers for these shapes, so the
  annotation was silently dropped.

OSS-Fuzz issue 563659413 hit this in parse_bjdata_fuzzer: an empty binary
_ArraySize_ is written as a plain object and read back as an empty array,
after which {"_ArrayType_":"int16","_ArraySize_":[],"_ArrayData_":null}
was encoded as the ND-array header "[$I#[]" and re-read as [], failing the
harness's value-stability check.

Such objects now fall back to a plain object encoding, which round-trips.
Genuine ND-arrays (two or more positive dimensions, not a 1xN row vector)
are encoded exactly as before. Existing fallback tests that used 1-D
shapes are moved to 2-D shapes so they keep exercising the check they
were written for, and the BJData documentation is updated.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
This commit is contained in:
Niels Lohmann
2026-09-24 17:01:31 +02:00
committed by GitHub
parent f3768d6868
commit 918da64657
4 changed files with 156 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/src/unit-bjdata.cpp
+93 -19
View File
@@ -2604,25 +2604,25 @@ TEST_CASE("BJData")
// that still round-trips.
// string data declared as a uint64 array
json const j_str = json({{"_ArrayType_", "uint64"}, {"_ArraySize_", {1}}, {"_ArrayData_", {"pointer"}}});
json const j_str = json({{"_ArrayType_", "uint64"}, {"_ArraySize_", {2, 1}}, {"_ArrayData_", {"pointer", "value"}}});
const auto out_str = json::to_bjdata(j_str);
CHECK(out_str.at(0) == '{');
CHECK(json::from_bjdata(out_str) == j_str);
// integer data declared as a double array
json const j_float = json({{"_ArrayType_", "double"}, {"_ArraySize_", {2}}, {"_ArrayData_", {1, 2}}});
json const j_float = json({{"_ArrayType_", "double"}, {"_ArraySize_", {2, 1}}, {"_ArrayData_", {1, 2}}});
const auto out_float = json::to_bjdata(j_float);
CHECK(out_float.at(0) == '{');
CHECK(json::from_bjdata(out_float) == j_float);
// a non-integer shape entry is likewise not treated as an ndarray
json const j_size = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {"x"}}, {"_ArrayData_", {1}}});
json const j_size = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {"x", 1}}, {"_ArrayData_", {1}}});
const auto out_size = json::to_bjdata(j_size);
CHECK(out_size.at(0) == '{');
CHECK(json::from_bjdata(out_size) == j_size);
// a negative shape entry is not a usable dimension either
json const j_neg = json::parse(R"({"_ArrayType_":"uint8","_ArraySize_":[-1],"_ArrayData_":[1]})");
json const j_neg = json::parse(R"({"_ArrayType_":"uint8","_ArraySize_":[-1,1],"_ArrayData_":[1]})");
const auto out_neg = json::to_bjdata(j_neg);
CHECK(out_neg.at(0) == '{');
CHECK(json::from_bjdata(out_neg) == j_neg);
@@ -2657,14 +2657,14 @@ TEST_CASE("BJData")
}
// negative values under a signed type behave the same way
const auto from_neg = json::to_bjdata(json::parse(R"({"_ArrayType_":"int32","_ArraySize_":[2],"_ArrayData_":[-5,7]})"));
const auto from_neg = json::to_bjdata(json::parse(R"({"_ArrayType_":"int32","_ArraySize_":[2,1],"_ArrayData_":[-5,7]})"));
CHECK(from_neg.at(0) == '[');
CHECK(from_neg == json::to_bjdata(json({{"_ArrayType_", "int32"}, {"_ArraySize_", {2}}, {"_ArrayData_", {-5, 7}}})));
CHECK(from_neg == json::to_bjdata(json({{"_ArrayType_", "int32"}, {"_ArraySize_", {2, 1}}, {"_ArrayData_", {-5, 7}}})));
// and so do the floating point types
const auto from_float = json::to_bjdata(json::parse(R"({"_ArrayType_":"double","_ArraySize_":[2],"_ArrayData_":[1.5,2.5]})"));
const auto from_float = json::to_bjdata(json::parse(R"({"_ArrayType_":"double","_ArraySize_":[2,1],"_ArrayData_":[1.5,2.5]})"));
CHECK(from_float.at(0) == '[');
CHECK(from_float == json::to_bjdata(json({{"_ArrayType_", "double"}, {"_ArraySize_", {2}}, {"_ArrayData_", {1.5, 2.5}}})));
CHECK(from_float == json::to_bjdata(json({{"_ArrayType_", "double"}, {"_ArraySize_", {2, 1}}, {"_ArrayData_", {1.5, 2.5}}})));
}
SECTION("optimized ndarray (type and vector-size as 1D array)")
@@ -2835,7 +2835,7 @@ TEST_CASE("BJData")
// a single dimension that does not fit into std::size_t is
// rejected for the same reason (only observable where
// std::size_t is narrower than 64 bit)
json j_huge = json({{"_ArrayData_", json::array()}, {"_ArraySize_", {18446744073709551615ull}}, {"_ArrayType_", "uint8"}});
json j_huge = json({{"_ArrayData_", json::array()}, {"_ArraySize_", {18446744073709551615ull, 2}}, {"_ArrayType_", "uint8"}});
CHECK(json::from_bjdata(json::to_bjdata(j_huge), true, true) == j_huge);
// a well-formed ndarray is still encoded as one
@@ -2878,42 +2878,116 @@ TEST_CASE("BJData")
// object encoding that still round-trips (see GitHub issue #5403)
// an unsigned element that does not fit uint8
json const j_uint8 = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2}}, {"_ArrayData_", {1, 256}}});
json const j_uint8 = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2, 1}}, {"_ArrayData_", {1, 256}}});
const auto out_uint8 = json::to_bjdata(j_uint8);
CHECK(out_uint8.at(0) == '{');
CHECK(json::from_bjdata(out_uint8) == j_uint8);
// a signed element that does not fit int8
json const j_int8 = json({{"_ArrayType_", "int8"}, {"_ArraySize_", {2}}, {"_ArrayData_", {1, 200}}});
json const j_int8 = json({{"_ArrayType_", "int8"}, {"_ArraySize_", {2, 1}}, {"_ArrayData_", {1, 200}}});
const auto out_int8 = json::to_bjdata(j_int8);
CHECK(out_int8.at(0) == '{');
CHECK(json::from_bjdata(out_int8) == j_int8);
// a negative element is likewise out of range for an
// unsigned _ArrayType_
json const j_uint16_neg = json({{"_ArrayType_", "uint16"}, {"_ArraySize_", {2}}, {"_ArrayData_", {1, -1}}});
json const j_uint16_neg = json({{"_ArrayType_", "uint16"}, {"_ArraySize_", {2, 1}}, {"_ArrayData_", {1, -1}}});
const auto out_uint16_neg = json::to_bjdata(j_uint16_neg);
CHECK(out_uint16_neg.at(0) == '{');
CHECK(json::from_bjdata(out_uint16_neg) == j_uint16_neg);
// a double element that overflows to infinity when narrowed
// to the "single" (float) precision named by _ArrayType_
json const j_single = json({{"_ArrayType_", "single"}, {"_ArraySize_", {2}}, {"_ArrayData_", {1.5, 1e40}}});
json const j_single = json({{"_ArrayType_", "single"}, {"_ArraySize_", {2, 1}}, {"_ArrayData_", {1.5, 1e40}}});
const auto out_single = json::to_bjdata(j_single);
CHECK(out_single.at(0) == '{');
CHECK(json::from_bjdata(out_single) == j_single);
// in-range boundary values still use the compact ndarray encoding
json const j_uint8_ok = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2}}, {"_ArrayData_", {0, 255}}});
CHECK(json::to_bjdata(j_uint8_ok) == std::vector<uint8_t>({'[', '$', 'U', '#', '[', 'i', 2, ']', 0, 255}));
json const j_uint8_ok = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2, 1}}, {"_ArrayData_", {0, 255}}});
CHECK(json::to_bjdata(j_uint8_ok) == std::vector<uint8_t>({'[', '$', 'U', '#', '[', 'i', 2, 'i', 1, ']', 0, 255}));
json const j_int8_ok = json({{"_ArrayType_", "int8"}, {"_ArraySize_", {2}}, {"_ArrayData_", {-128, 127}}});
CHECK(json::to_bjdata(j_int8_ok) == std::vector<uint8_t>({'[', '$', 'i', '#', '[', 'i', 2, ']', 0x80, 0x7F}));
json const j_int8_ok = json({{"_ArrayType_", "int8"}, {"_ArraySize_", {2, 1}}, {"_ArrayData_", {-128, 127}}});
CHECK(json::to_bjdata(j_int8_ok) == std::vector<uint8_t>({'[', '$', 'i', '#', '[', 'i', 2, 'i', 1, ']', 0x80, 0x7F}));
json const j_single_ok = json({{"_ArrayType_", "single"}, {"_ArraySize_", {1}}, {"_ArrayData_", {1.5}}});
json const j_single_ok = json({{"_ArrayType_", "single"}, {"_ArraySize_", {2, 1}}, {"_ArrayData_", {1.5, -1.5}}});
const auto out_single_ok = json::to_bjdata(j_single_ok);
CHECK(out_single_ok.at(0) == '[');
CHECK(json::from_bjdata(out_single_ok) == json({1.5f}));
CHECK(json::from_bjdata(out_single_ok) == json({{"_ArrayType_", "single"}, {"_ArraySize_", {2, 1}}, {"_ArrayData_", {1.5f, -1.5f}}}));
}
SECTION("ndarray that would not be read back as an annotated object stays as object")
{
// the reader only restores an annotated object from an ND-array
// with at least two non-zero dimensions that is not a 1xN row
// vector; any other shape is read back as a plain array. Writing
// such an object as an ND-array would drop its annotation, so it
// falls back to a plain object encoding that round-trips.
for (const char* text :
{
R"({"_ArrayType_":"int16","_ArraySize_":[],"_ArrayData_":[]})",
R"({"_ArrayType_":"int16","_ArraySize_":[2],"_ArrayData_":[1,2]})",
R"({"_ArrayType_":"int16","_ArraySize_":[1,2],"_ArrayData_":[1,2]})",
R"({"_ArrayType_":"int16","_ArraySize_":[0],"_ArrayData_":[]})",
R"({"_ArrayType_":"int16","_ArraySize_":[2,0],"_ArrayData_":[]})",
R"({"_ArrayType_":"int16","_ArraySize_":[0,2],"_ArrayData_":[]})"
})
{
CAPTURE(text);
const json j = json::parse(text);
for (const bool use_size :
{
false, true
})
{
const auto out = json::to_bjdata(j, use_size, use_size);
CHECK(out.at(0) == '{');
CHECK(json::from_bjdata(out) == j);
}
}
// a genuine ND-array still uses the compact encoding and round-trips
const json j_2d = json::parse(R"({"_ArrayType_":"int16","_ArraySize_":[2,1],"_ArrayData_":[1,2]})");
const auto out_2d = json::to_bjdata(j_2d);
CHECK(out_2d.at(0) == '[');
CHECK(json::from_bjdata(out_2d) == j_2d);
}
SECTION("ndarray with non-array _ArrayData_ stays as object")
{
// the elements are written from _ArrayData_ as a flat list, so it
// has to be an array: null has size 0, any other scalar has size 1,
// and iterating an object visits its values, so each of these could
// match the dimensions and be encoded as an unrelated ND-array
for (const char* text :
{
R"({"_ArrayType_":"int16","_ArraySize_":[2,1],"_ArrayData_":null})",
R"({"_ArrayType_":"int16","_ArraySize_":[2,1],"_ArrayData_":{"a":1,"b":2}})",
R"({"_ArrayType_":"int16","_ArraySize_":[1],"_ArrayData_":5})",
R"({"_ArrayType_":"int16","_ArraySize_":[],"_ArrayData_":null})"
})
{
CAPTURE(text);
const json j = json::parse(text);
const auto out = json::to_bjdata(j);
CHECK(out.at(0) == '{');
CHECK(json::from_bjdata(out) == j);
}
// OSS-Fuzz issue 563659413: an empty binary _ArraySize_ is written
// as a plain object and read back as an empty array, after which
// the object with a null _ArrayData_ was encoded as an empty
// ND-array and re-read as [], so a second round trip lost the value
const std::vector<uint8_t> input =
{
'{', 'U', 11, '_', 'A', 'r', 'r', 'a', 'y', 'D', 'a', 't', 'a', '_', 'Z',
'U', 11, '_', 'A', 'r', 'r', 'a', 'y', 'T', 'y', 'p', 'e', '_', 'S', 'i', 5, 'i', 'n', 't', '1', '6',
'U', 11, '_', 'A', 'r', 'r', 'a', 'y', 'S', 'i', 'z', 'e', '_', '[', '$', 'B', '#', '[', ']', '}'
};
const json j1 = json::from_bjdata(input);
const json j2 = json::from_bjdata(json::to_bjdata(j1, false, false));
CHECK(j2 == json::parse(R"({"_ArrayType_":"int16","_ArraySize_":[],"_ArrayData_":null})"));
CHECK(json::from_bjdata(json::to_bjdata(j2, false, false)) == j2);
}
SECTION("ndarray with _ArrayType_ \"byte\" is gated by the BJData draft version")