mirror of
https://github.com/calibrain/shelfmark.git
synced 2026-11-15 15:08:11 +00:00
splitsec2
a430341a93
I went through what Shelfmark does per search and per cover and measured the pieces that showed up. Four of them are cheap to fix, so this is one PR. Shelfmark runs on a single gevent worker, so any of these blocks every other request while it runs, and I'm assuming a lot of people run it on small hardware (a Pi, say). Cover cache. Every cache hit rewrote the whole `cache_index.json` just to bump the entry's last-access time. On my box that is 3.7 ms of CPU per hit at 500 entries and about 100 ms at 15,000 (2.2 MB written each time), and a results page asks for around 40 covers. The access time now stays in memory and the index is written at most every 30 seconds, or whenever something else saves it. A crash inside that window loses some access times, which only changes eviction order. Settings reload. `config.refresh()` rebuilt the cache by reading and parsing a tab's JSON file once per field, so about 250 parses per refresh, 100 ms of CPU, and the metadata search, providers and config routes each trigger it. It now reads each file once per rebuild (several tabs share `settings.json`, so the memo is keyed on the file path). Library match. When the library check is on, each search result is compared against every library entry, and the title was split into words again for every entry. The words are now computed once per result. I measured 33 ms per result at 5,000 library entries and 106 ms at 20,000, which is a second or more per page of 40 results. The matching itself is unchanged. Cover URL probe. `is_covers_cache_enabled()` creates and deletes a file in the config directory to check it is writable, and it runs for every cover, queue task and history row. The answer is remembered for 60 seconds. The probe is 0.045 ms on its own, but it is called a few hundred times per request in the worst case. Each change has a test that fails without it (call counts on the index saves, file opens during a refresh, tokenisation calls, and probe calls). Ruff, basedpyright on the changed files and the non-e2e suite pass. The timings are from a dev machine with an empty config, so the real numbers depend on cache and library size.
86 lines
2.6 KiB
Python
86 lines
2.6 KiB
Python
"""Tests for the shared token/ISBN/surname matching helpers."""
|
|
|
|
import pytest
|
|
|
|
from shelfmark.core import text_match
|
|
|
|
_TWENTY_WORDS = " ".join(f"word{i:02d}" for i in range(20))
|
|
|
|
|
|
def test_tokens_lowercases_and_splits_on_non_alphanumerics():
|
|
assert text_match.tokens("Dungeon Crawler Carl: Book 1!") == [
|
|
"dungeon",
|
|
"crawler",
|
|
"carl",
|
|
"book",
|
|
"1",
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize("value", [None, "", " --- "])
|
|
def test_tokens_empty_input(value):
|
|
assert text_match.tokens(value) == []
|
|
|
|
|
|
def test_significant_tokens_drops_stopwords_and_single_characters():
|
|
assert text_match.significant_tokens("The Way of Kings, Part A 2") == ["way", "kings", "part"]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("author", "expected"),
|
|
[
|
|
("Sanderson, Brandon", "sanderson"),
|
|
("Brandon Sanderson", "sanderson"),
|
|
("Dinniman, Matt J.", "dinniman"),
|
|
("", None),
|
|
(None, None),
|
|
("A", None),
|
|
],
|
|
)
|
|
def test_author_surname(author, expected):
|
|
assert text_match.author_surname(author) == expected
|
|
|
|
|
|
def test_title_tokens_match_default_threshold_edges():
|
|
words = _TWENTY_WORDS.split()
|
|
|
|
assert text_match.title_tokens_match(_TWENTY_WORDS, set(words[:17])) is True # 17/20 == 0.85
|
|
assert text_match.title_tokens_match(_TWENTY_WORDS, set(words[:16])) is False
|
|
|
|
|
|
def test_title_tokens_match_explicit_threshold_is_inclusive():
|
|
assert text_match.title_tokens_match("alpha beta", {"alpha"}, threshold=0.5) is True
|
|
assert text_match.title_tokens_match("alpha beta", {"alpha"}, threshold=0.51) is False
|
|
|
|
|
|
def test_title_tokens_match_ignores_stopwords_in_title():
|
|
assert text_match.title_tokens_match("The Name of the Wind", {"name", "wind"}) is True
|
|
|
|
|
|
@pytest.mark.parametrize("title", [None, "", "the of a"])
|
|
def test_title_tokens_match_without_significant_tokens_is_false(title):
|
|
assert text_match.title_tokens_match(title, {"the", "of"}) is False
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("value", "expected"),
|
|
[
|
|
("978-0-59-382024-7", "9780593820247"),
|
|
("0-306-40615-x", "030640615X"),
|
|
(" 0306406152 ", "0306406152"),
|
|
(9780593820247, "9780593820247"),
|
|
(None, ""),
|
|
("", ""),
|
|
],
|
|
)
|
|
def test_normalize_isbn(value, expected):
|
|
assert text_match.normalize_isbn(value) == expected
|
|
|
|
|
|
def test_significant_tokens_match_agrees_with_title_tokens_match():
|
|
haystack = {"alpha", "beta", "gamma"}
|
|
for title in ("Alpha Beta", "alpha delta", "the of", ""):
|
|
assert text_match.significant_tokens_match(
|
|
text_match.significant_tokens(title), haystack
|
|
) is text_match.title_tokens_match(title, haystack)
|