mirror of
https://github.com/calibrain/shelfmark.git
synced 2026-11-14 16:48:24 +00:00
splitsec2
a430341a93
I went through what Shelfmark does per search and per cover and measured the pieces that showed up. Four of them are cheap to fix, so this is one PR. Shelfmark runs on a single gevent worker, so any of these blocks every other request while it runs, and I'm assuming a lot of people run it on small hardware (a Pi, say). Cover cache. Every cache hit rewrote the whole `cache_index.json` just to bump the entry's last-access time. On my box that is 3.7 ms of CPU per hit at 500 entries and about 100 ms at 15,000 (2.2 MB written each time), and a results page asks for around 40 covers. The access time now stays in memory and the index is written at most every 30 seconds, or whenever something else saves it. A crash inside that window loses some access times, which only changes eviction order. Settings reload. `config.refresh()` rebuilt the cache by reading and parsing a tab's JSON file once per field, so about 250 parses per refresh, 100 ms of CPU, and the metadata search, providers and config routes each trigger it. It now reads each file once per rebuild (several tabs share `settings.json`, so the memo is keyed on the file path). Library match. When the library check is on, each search result is compared against every library entry, and the title was split into words again for every entry. The words are now computed once per result. I measured 33 ms per result at 5,000 library entries and 106 ms at 20,000, which is a second or more per page of 40 results. The matching itself is unchanged. Cover URL probe. `is_covers_cache_enabled()` creates and deletes a file in the config directory to check it is writable, and it runs for every cover, queue task and history row. The answer is remembered for 60 seconds. The probe is 0.045 ms on its own, but it is called a few hundred times per request in the worst case. Each change has a test that fails without it (call counts on the index saves, file opens during a refresh, tokenisation calls, and probe calls). Ruff, basedpyright on the changed files and the non-e2e suite pass. The timings are from a dev machine with an empty config, so the real numbers depend on cache and library size.
125 lines
3.8 KiB
Python
125 lines
3.8 KiB
Python
"""
|
|
Pytest configuration and shared fixtures.
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
|
|
# Set environment variables BEFORE importing the application
|
|
# These override the defaults that try to use system paths like /var/log
|
|
_temp_base = tempfile.mkdtemp(prefix="cwabd_test_")
|
|
|
|
# LOG_ROOT is the base - LOG_DIR is computed as LOG_ROOT / "shelfmark"
|
|
# So we set LOG_ROOT to our temp directory to get LOG_DIR = _temp_base/shelfmark
|
|
os.environ["LOG_ROOT"] = _temp_base
|
|
os.environ["CONFIG_DIR"] = os.path.join(_temp_base, "config")
|
|
os.environ["INGEST_DIR"] = os.path.join(_temp_base, "ingest")
|
|
os.environ["TMP_DIR"] = os.path.join(_temp_base, "tmp")
|
|
|
|
# Create the directories that will be used
|
|
os.makedirs(os.path.join(_temp_base, "shelfmark"), exist_ok=True) # LOG_DIR
|
|
os.makedirs(os.path.join(_temp_base, "config"), exist_ok=True)
|
|
os.makedirs(os.path.join(_temp_base, "ingest"), exist_ok=True)
|
|
os.makedirs(os.path.join(_temp_base, "tmp"), exist_ok=True)
|
|
|
|
# Add the project root to Python path
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
|
|
import pytest
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _reset_config_dir_writable_cache():
|
|
"""The writable-config-dir probe is remembered; each test starts without it."""
|
|
from shelfmark.config.env import reset_config_dir_writable_cache
|
|
|
|
reset_config_dir_writable_cache()
|
|
yield
|
|
reset_config_dir_writable_cache()
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _clear_torrent_fetch_cache():
|
|
"""Keep the shared torrent fetch cache from leaking between tests."""
|
|
from shelfmark.download.clients.torrent_utils import clear_torrent_fetch_cache
|
|
|
|
clear_torrent_fetch_cache()
|
|
yield
|
|
clear_torrent_fetch_cache()
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _public_destination_dns(monkeypatch):
|
|
"""Resolve every host to a public address for the destination guard.
|
|
|
|
The guard looks hosts up before a request. Tests use made-up and real hostnames and
|
|
must not depend on the DNS of the machine running them (a filtering resolver answers
|
|
``::`` for some mirrors), so the guard's resolver is stubbed. Tests of the guard
|
|
itself replace it again.
|
|
"""
|
|
from shelfmark.download import network
|
|
|
|
monkeypatch.setattr(
|
|
network,
|
|
"_system_getaddrinfo",
|
|
lambda *_args, **_kwargs: [(2, 1, 6, "", ("93.184.216.34", 0))],
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def sample_prowlarr_result():
|
|
"""Sample Prowlarr API search result."""
|
|
return {
|
|
"guid": "abc123-guid",
|
|
"title": "The Great Book by Author Name.epub",
|
|
"indexer": "MyIndexer",
|
|
"protocol": "torrent",
|
|
"size": 5242880, # 5 MB
|
|
"downloadUrl": "magnet:?xt=urn:btih:abc123",
|
|
"infoUrl": "https://example.com/book/123",
|
|
"seeders": 10,
|
|
"leechers": 2,
|
|
"publishDate": "2024-01-15T12:00:00Z",
|
|
"categories": [{"id": 7020, "name": "Books/EBook"}],
|
|
"indexerId": 1,
|
|
}
|
|
|
|
|
|
@pytest.fixture
|
|
def sample_nzb_result():
|
|
"""Sample Prowlarr API NZB result."""
|
|
return {
|
|
"guid": "nzb456-guid",
|
|
"title": "Another Book [PDF] by Writer",
|
|
"indexer": "NZBIndexer",
|
|
"protocol": "usenet",
|
|
"size": 10485760, # 10 MB
|
|
"downloadUrl": "https://example.com/download.nzb",
|
|
"infoUrl": "https://example.com/nzb/456",
|
|
"grabs": 50,
|
|
"publishDate": "2024-02-20T10:30:00Z",
|
|
"categories": [{"id": 7020, "name": "Books/EBook"}],
|
|
"indexerId": 2,
|
|
}
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_config(monkeypatch):
|
|
"""Fixture to mock config values."""
|
|
config_values = {}
|
|
|
|
def mock_get(key, default=""):
|
|
return config_values.get(key, default)
|
|
|
|
def set_config(key, value):
|
|
config_values[key] = value
|
|
|
|
# Create a mock config module
|
|
class MockConfig:
|
|
get = staticmethod(mock_get)
|
|
set = staticmethod(set_config)
|
|
_values = config_values
|
|
|
|
return MockConfig
|