From b87a618151e574d04c593fb681eff8145a54e077 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Thu, 24 Sep 2026 09:48:18 +0200 Subject: [PATCH] Pin Google Benchmark to release 1.9.5 The benchmarks fetched Google Benchmark's main branch, so two builds on different days could measure with different library code, and CMake 3.30 and later warn that the single-argument FetchContent_Populate() is deprecated. Fetch the 1.9.5 release archive, verified by its SHA-256, with FetchContent_MakeAvailable() instead. That needs CMake 3.14; Google Benchmark itself already needed 3.13. Its -Werror is switched off, so a newer compiler's new warnings cannot break the pinned release, and its install rules are no longer added. Signed-off-by: Niels Lohmann --- tests/benchmarks/CMakeLists.txt | 29 +++++++++++++++-------------- tests/benchmarks/README.md | 14 ++++++++------ 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/tests/benchmarks/CMakeLists.txt b/tests/benchmarks/CMakeLists.txt index 1e835d120..4d7265145 100644 --- a/tests/benchmarks/CMakeLists.txt +++ b/tests/benchmarks/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.11...3.14) +cmake_minimum_required(VERSION 3.14) project(JSON_Benchmarks LANGUAGES CXX) # set compiler flags @@ -6,21 +6,22 @@ if((CMAKE_CXX_COMPILER_ID MATCHES GNU) OR (CMAKE_CXX_COMPILER_ID MATCHES Clang)) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto -DNDEBUG -O3") endif() -# configure Google Benchmarks +# configure Google Benchmark; a fixed release, so that results stay comparable +set(JSON_GOOGLE_BENCHMARK_VERSION 1.9.5) include(FetchContent) -FetchContent_Declare( - benchmark - GIT_REPOSITORY https://github.com/google/benchmark.git - GIT_TAG origin/main - GIT_SHALLOW TRUE -) -FetchContent_GetProperties(benchmark) -if(NOT benchmark_POPULATED) - FetchContent_Populate(benchmark) - set(BENCHMARK_ENABLE_TESTING OFF CACHE INTERNAL "" FORCE) - add_subdirectory(${benchmark_SOURCE_DIR} ${benchmark_BINARY_DIR}) -endif() +# only the library is needed; -Werror would break the pinned release as soon as +# a newer compiler adds a warning +set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "" FORCE) +set(BENCHMARK_ENABLE_INSTALL OFF CACHE BOOL "" FORCE) +set(BENCHMARK_ENABLE_WERROR OFF CACHE BOOL "" FORCE) + +FetchContent_Declare(benchmark + URL https://github.com/google/benchmark/archive/refs/tags/v${JSON_GOOGLE_BENCHMARK_VERSION}.tar.gz + URL_HASH SHA256=9631341c82bac4a288bef951f8b26b41f69021794184ece969f8473977eaa340 + DOWNLOAD_EXTRACT_TIMESTAMP TRUE +) +FetchContent_MakeAvailable(benchmark) # download test data set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../../cmake ${CMAKE_MODULE_PATH}) diff --git a/tests/benchmarks/README.md b/tests/benchmarks/README.md index a4e095ae0..aaf5abe46 100644 --- a/tests/benchmarks/README.md +++ b/tests/benchmarks/README.md @@ -23,10 +23,12 @@ The input files are those of [nativejson-benchmark](https://github.com/miloyip/n ## Requirements -- CMake 3.11 or later, a C++11 compiler, and Ninja for the `make` target. -- Network access on the first configure: CMake fetches Google Benchmark and downloads the - [test data](https://github.com/nlohmann/json_test_data) into the build directory. To reuse a download, pass - `-DJSON_TestDataDirectory=/test_files`. +- CMake 3.14 or later, a C++11 compiler, and Ninja for the `make` target. +- Network access on the first configure: CMake downloads Google Benchmark and the + [test data](https://github.com/nlohmann/json_test_data) into the build directory. To reuse a download of the test + data, pass `-DJSON_TestDataDirectory=/test_files`. +- Google Benchmark is pinned to a release (1.9.5), so that results from different days stay comparable. To update it, + change `JSON_GOOGLE_BENCHMARK_VERSION` and the archive's `URL_HASH` in `CMakeLists.txt` together. - The benchmarks include `single_include/nlohmann/json.hpp`, so run `make amalgamate` after changing anything in `include/`. @@ -99,8 +101,8 @@ build-venv/bin/pip install numpy scipy build-venv/bin/python build-current/_deps/benchmark-src/tools/compare.py -a benchmarks build-baseline/results.json build-current/results.json ``` -The tool's own `tools/requirements.txt` pins the newest NumPy and SciPy, which may need a newer Python than yours; -unpinned versions work as well. In its output: +The tool's own `tools/requirements.txt` pins NumPy and SciPy versions that need Python 3.11 or later; with an older +Python, unpinned versions work as well. In its output: - the `Time` and `CPU` columns are relative changes: `-0.35` means 35% faster, `+0.10` means 10% slower; - `_pvalue` lines report a Mann-Whitney U test of whether the two versions differ. It needs at least 9