mirror of
https://github.com/calibrain/shelfmark.git
synced 2026-11-14 10:08:24 +00:00
splitsec2
a430341a93
I went through what Shelfmark does per search and per cover and measured the pieces that showed up. Four of them are cheap to fix, so this is one PR. Shelfmark runs on a single gevent worker, so any of these blocks every other request while it runs, and I'm assuming a lot of people run it on small hardware (a Pi, say). Cover cache. Every cache hit rewrote the whole `cache_index.json` just to bump the entry's last-access time. On my box that is 3.7 ms of CPU per hit at 500 entries and about 100 ms at 15,000 (2.2 MB written each time), and a results page asks for around 40 covers. The access time now stays in memory and the index is written at most every 30 seconds, or whenever something else saves it. A crash inside that window loses some access times, which only changes eviction order. Settings reload. `config.refresh()` rebuilt the cache by reading and parsing a tab's JSON file once per field, so about 250 parses per refresh, 100 ms of CPU, and the metadata search, providers and config routes each trigger it. It now reads each file once per rebuild (several tabs share `settings.json`, so the memo is keyed on the file path). Library match. When the library check is on, each search result is compared against every library entry, and the title was split into words again for every entry. The words are now computed once per result. I measured 33 ms per result at 5,000 library entries and 106 ms at 20,000, which is a second or more per page of 40 results. The matching itself is unchanged. Cover URL probe. `is_covers_cache_enabled()` creates and deletes a file in the config directory to check it is writable, and it runs for every cover, queue task and history row. The answer is remembered for 60 seconds. The probe is 0.045 ms on its own, but it is called a few hundred times per request in the worst case. Each change has a test that fails without it (call counts on the index saves, file opens during a refresh, tokenisation calls, and probe calls). Ruff, basedpyright on the changed files and the non-e2e suite pass. The timings are from a dev machine with an empty config, so the real numbers depend on cache and library size.
47 lines
1.8 KiB
Python
47 lines
1.8 KiB
Python
"""Rebuilding the settings cache parses each tab's config file once."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from shelfmark.core import settings_registry
|
|
from shelfmark.core.config import config as app_config
|
|
|
|
|
|
def test_refresh_reads_each_config_file_once(tmp_path, monkeypatch):
|
|
plugins = tmp_path / "plugins"
|
|
plugins.mkdir()
|
|
(tmp_path / "settings.json").write_text(json.dumps({"SEARCH_MODE": "universal"}))
|
|
(plugins / "downloads.json").write_text(json.dumps({"MAX_CONCURRENT_DOWNLOADS": 3}))
|
|
(plugins / "hardcover.json").write_text(json.dumps({"HARDCOVER_ENABLED": False}))
|
|
|
|
opened: dict[str, int] = {}
|
|
real_open = Path.open
|
|
|
|
def counting_open(self, *args, **kwargs):
|
|
if self.suffix == ".json" and self.parent in (tmp_path, plugins):
|
|
opened[self.name] = opened.get(self.name, 0) + 1
|
|
return real_open(self, *args, **kwargs)
|
|
|
|
with monkeypatch.context() as patched:
|
|
patched.setattr(settings_registry, "_get_config_dir", lambda: tmp_path)
|
|
patched.setattr(Path, "open", counting_open)
|
|
app_config.refresh(force=True)
|
|
|
|
app_config.refresh(force=True)
|
|
assert opened, "no config file was read"
|
|
assert max(opened.values()) == 1, opened
|
|
|
|
|
|
def test_config_files_are_read_fresh_outside_the_block(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(settings_registry, "_get_config_dir", lambda: tmp_path)
|
|
(tmp_path / "settings.json").write_text(json.dumps({"A": 1}))
|
|
|
|
with settings_registry.config_files_read_once():
|
|
assert settings_registry.load_config_file("general") == {"A": 1}
|
|
(tmp_path / "settings.json").write_text(json.dumps({"A": 2}))
|
|
assert settings_registry.load_config_file("general") == {"A": 1}
|
|
|
|
assert settings_registry.load_config_file("general") == {"A": 2}
|