diff --git a/data/book-languages.json b/data/book-languages.json index a4b42c3f..a1b1382b 100644 --- a/data/book-languages.json +++ b/data/book-languages.json @@ -1,72 +1,79 @@ [ - { "language": "English", "code": "en" }, - { "language": "Chinese", "code": "zh" }, - { "language": "Russian", "code": "ru" }, - { "language": "Spanish", "code": "es" }, - { "language": "French", "code": "fr" }, - { "language": "German", "code": "de" }, - { "language": "Italian", "code": "it" }, - { "language": "Portuguese", "code": "pt" }, - { "language": "Polish", "code": "pl" }, - { "language": "Bulgarian", "code": "bg" }, - { "language": "Dutch", "code": "nl" }, - { "language": "Japanese", "code": "ja" }, - { "language": "Arabic", "code": "ar" }, - { "language": "Hebrew", "code": "he" }, - { "language": "Turkish", "code": "tr" }, - { "language": "Hungarian", "code": "hu" }, - { "language": "Latin", "code": "la" }, - { "language": "Czech", "code": "cs" }, - { "language": "Korean", "code": "ko" }, - { "language": "Ukrainian", "code": "uk" }, - { "language": "Indonesian", "code": "id" }, - { "language": "Romanian", "code": "ro" }, - { "language": "Swedish", "code": "sv" }, - { "language": "Greek", "code": "el" }, - { "language": "Lithuanian", "code": "lt" }, - { "language": "Bangla", "code": "bn" }, - { "language": "Traditional Chinese", "code": "zh‑Hant" }, - { "language": "Afrikaans", "code": "af" }, - { "language": "Catalan", "code": "ca" }, - { "language": "Danish", "code": "da" }, - { "language": "Thai", "code": "th" }, - { "language": "Hindi", "code": "hi" }, - { "language": "Irish", "code": "ga" }, - { "language": "Latvian", "code": "lv" }, + { "language": "English", "code": "en", "aliases": ["eng"] }, + { "language": "Chinese", "code": "zh", "aliases": ["chi", "zho"] }, + { "language": "Russian", "code": "ru", "aliases": ["rus"] }, + { "language": "Spanish", "code": "es", "aliases": ["spa"] }, + { "language": "French", "code": "fr", "aliases": ["fra", "fre"] }, + { "language": "German", "code": "de", "aliases": ["deu", "ger"] }, + { "language": "Italian", "code": "it", "aliases": ["ita"] }, + { "language": "Portuguese", "code": "pt", "aliases": ["por"] }, + { "language": "Polish", "code": "pl", "aliases": ["pol"] }, + { "language": "Bulgarian", "code": "bg", "aliases": ["bul"] }, + { "language": "Dutch", "code": "nl", "aliases": ["dut", "nld"] }, + { "language": "Japanese", "code": "ja", "aliases": ["jap", "jpn"] }, + { "language": "Arabic", "code": "ar", "aliases": ["ara"] }, + { "language": "Hebrew", "code": "he", "aliases": ["heb"] }, + { "language": "Turkish", "code": "tr", "aliases": ["tur"] }, + { "language": "Hungarian", "code": "hu", "aliases": ["hun"] }, + { "language": "Latin", "code": "la", "aliases": ["lat"] }, + { "language": "Czech", "code": "cs", "aliases": ["ces", "cze"] }, + { "language": "Korean", "code": "ko", "aliases": ["kor"] }, + { "language": "Ukrainian", "code": "uk", "aliases": ["ukr"] }, + { "language": "Indonesian", "code": "id", "aliases": ["ind"] }, + { "language": "Romanian", "code": "ro", "aliases": ["rom", "ron"] }, + { "language": "Swedish", "code": "sv", "aliases": ["swe"] }, + { "language": "Greek", "code": "el", "aliases": ["ell", "gre"] }, + { "language": "Lithuanian", "code": "lt", "aliases": ["lit"] }, + { "language": "Bangla", "code": "bn", "aliases": ["ben", "bengali"] }, + { "language": "Traditional Chinese", "code": "zh-Hant", "aliases": ["zh‑Hant"] }, + { "language": "Afrikaans", "code": "af", "aliases": ["afr"] }, + { "language": "Catalan", "code": "ca", "aliases": ["cat"] }, + { "language": "Danish", "code": "da", "aliases": ["dan"] }, + { "language": "Thai", "code": "th", "aliases": ["tha"] }, + { "language": "Hindi", "code": "hi", "aliases": ["hin"] }, + { "language": "Irish", "code": "ga", "aliases": ["gle"] }, + { "language": "Latvian", "code": "lv", "aliases": ["lav"] }, { "language": "Tibetan", "code": "bo" }, - { "language": "Kannada", "code": "kn" }, - { "language": "Serbian", "code": "sr" }, - { "language": "Persian", "code": "fa" }, - { "language": "Croatian", "code": "hr" }, + { "language": "Kannada", "code": "kn", "aliases": ["kan"] }, + { "language": "Serbian", "code": "sr", "aliases": ["srp"] }, + { "language": "Persian", "code": "fa", "aliases": ["farsi", "fas", "per"] }, + { "language": "Croatian", "code": "hr", "aliases": ["hrv"] }, { "language": "Slovak", "code": "sk" }, - { "language": "Javanese", "code": "jv" }, - { "language": "Vietnamese", "code": "vi" }, - { "language": "Urdu", "code": "ur" }, - { "language": "Finnish", "code": "fi" }, - { "language": "Norwegian", "code": "no" }, + { "language": "Javanese", "code": "jv", "aliases": ["jav"] }, + { "language": "Vietnamese", "code": "vi", "aliases": ["vie"] }, + { "language": "Urdu", "code": "ur", "aliases": ["urd"] }, + { "language": "Finnish", "code": "fi", "aliases": ["fin"] }, + { "language": "Norwegian", "code": "no", "aliases": ["nor"] }, { "language": "Kinyarwanda", "code": "rw" }, - { "language": "Tamil", "code": "ta" }, + { "language": "Tamil", "code": "ta", "aliases": ["tam"] }, { "language": "Belarusian", "code": "be" }, { "language": "Kazakh", "code": "kk" }, { "language": "Mongolian", "code": "mn" }, { "language": "Georgian", "code": "ka" }, - { "language": "Slovenian", "code": "sl" }, + { "language": "Slovenian", "code": "sl", "aliases": ["slv"] }, { "language": "Esperanto", "code": "eo" }, { "language": "Galician", "code": "gl" }, - { "language": "Marathi", "code": "mr" }, - { "language": "Filipino", "code": "fil" }, - { "language": "Gujarati", "code": "gu" }, - { "language": "Malayalam", "code": "ml" }, + { "language": "Marathi", "code": "mr", "aliases": ["mar"] }, + { "language": "Filipino", "code": "fil", "aliases": ["tagalog", "tgl"] }, + { "language": "Gujarati", "code": "gu", "aliases": ["guj"] }, + { "language": "Malayalam", "code": "ml", "aliases": ["mal"] }, { "language": "Kyrgyz", "code": "ky" }, { "language": "Azerbaijani", "code": "az" }, { "language": "Quechua", "code": "qu" }, { "language": "Swahili", "code": "sw" }, { "language": "Bashkir", "code": "ba" }, - { "language": "Punjabi", "code": "pa" }, - { "language": "Malay", "code": "ms" }, - { "language": "Telugu", "code": "te" }, + { "language": "Punjabi", "code": "pa", "aliases": ["pan"] }, + { "language": "Malay", "code": "ms", "aliases": ["may", "msa"] }, + { "language": "Telugu", "code": "te", "aliases": ["tel"] }, { "language": "Albanian", "code": "sq" }, { "language": "Uyghur", "code": "ug" }, { "language": "Armenian", "code": "hy" }, - { "language": "Shan", "code": "shn" } + { "language": "Shan", "code": "shn" }, + { "language": "Bosnian", "code": "bs", "aliases": ["bos"] }, + { "language": "Burmese", "code": "my", "aliases": ["bur", "mya"] }, + { "language": "Estonian", "code": "et", "aliases": ["est"] }, + { "language": "Icelandic", "code": "is", "aliases": ["ice", "isl"] }, + { "language": "Manx", "code": "gv", "aliases": ["glv"] }, + { "language": "Scottish Gaelic", "code": "gd", "aliases": ["gla"] }, + { "language": "Sanskrit", "code": "sa", "aliases": ["san"] } ] diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 7e845b52..828308aa 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -432,8 +432,8 @@ The release source tab to open by default in the release modal for audiobooks. U | `BOOKS_OUTPUT_MODE` | Choose where completed book files are sent. | string (choice) | `folder` | | `INGEST_DIR` | Directory where downloaded files are saved. Use {User} for per-user folders (e.g. /books/{User}). | string | `/books` | | `FILE_ORGANIZATION` | Choose how downloaded book files are named and organized. | string (choice) | `rename` | -| `TEMPLATE_RENAME` | Variables: {Author}, {Title}, {Year}, {User}, {OriginalName} (source filename without extension). Universal adds: {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. Rename templates are filename-only (no '/' or '\'); use Organize for folders. Applies to single-file downloads. | string | `{Author} - {Title} ({Year})` | -| `TEMPLATE_ORGANIZE` | Use / to create folders. Variables: {Author}, {Title}, {Year}, {User}, {OriginalName} (source filename without extension). Universal adds: {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. | string | `{Author}/{Title} ({Year})` | +| `TEMPLATE_RENAME` | Variables: {Author}, {Title}, {Year}, {Language}, {User}, {OriginalName} (source filename without extension). Universal adds: {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. Rename templates are filename-only (no '/' or '\'); use Organize for folders. Applies to single-file downloads. | string | `{Author} - {Title} ({Year})` | +| `TEMPLATE_ORGANIZE` | Use / to create folders. Variables: {Author}, {Title}, {Year}, {Language}, {User}, {OriginalName} (source filename without extension). Universal adds: {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. | string | `{Author}/{Title} ({Year})` | | `HARDLINK_TORRENTS` | Create hardlinks instead of copying. Preserves seeding but archives won't be extracted. Don't use if destination is a library ingest folder. | boolean | `false` | | `BOOKLORE_HOST` | Base URL of your Grimmory instance | string | _none_ | | `BOOKLORE_USERNAME` | Grimmory account username | string | _none_ | @@ -454,8 +454,8 @@ The release source tab to open by default in the release modal for audiobooks. U | `EMAIL_ALLOW_UNVERIFIED_TLS` | Disable TLS certificate verification (not recommended). | boolean | `false` | | `DESTINATION_AUDIOBOOK` | Directory where downloaded audiobook files are saved. Leave empty to use the Books destination. | string | _none_ | | `FILE_ORGANIZATION_AUDIOBOOK` | Choose how downloaded audiobook files are named and organized. | string (choice) | `rename` | -| `TEMPLATE_AUDIOBOOK_RENAME` | Variables: {Author}, {Title}, {Year}, {User}, {OriginalName} (source filename without extension), {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}, {PartNumber}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. Rename templates are filename-only (no '/' or '\'); use Organize for folders. Applies to single-file downloads. | string | `{Author} - {Title}` | -| `TEMPLATE_AUDIOBOOK_ORGANIZE` | Use / to create folders. Variables: {Author}, {Title}, {Year}, {User}, {OriginalName} (source filename without extension), {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}, {PartNumber}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. | string | `{Author}/{Title}/{Title}` | +| `TEMPLATE_AUDIOBOOK_RENAME` | Variables: {Author}, {Title}, {Year}, {Language}, {User}, {OriginalName} (source filename without extension), {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}, {PartNumber}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. Rename templates are filename-only (no '/' or '\'); use Organize for folders. Applies to single-file downloads. | string | `{Author} - {Title}` | +| `TEMPLATE_AUDIOBOOK_ORGANIZE` | Use / to create folders. Variables: {Author}, {Title}, {Year}, {Language}, {User}, {OriginalName} (source filename without extension), {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}, {PartNumber}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. | string | `{Author}/{Title}/{Title}` | | `HARDLINK_TORRENTS_AUDIOBOOK` | Create hardlinks instead of copying. Preserves seeding but archives won't be extracted. Don't use if destination is a library ingest folder. | boolean | `true` | | `AUTO_OPEN_DOWNLOADS_SIDEBAR` | Automatically open the downloads sidebar when a new download is queued. | boolean | `false` | | `DOWNLOAD_TO_BROWSER_CONTENT_TYPES` | Automatically download completed files to your browser for the selected content types. | string (comma-separated) | _empty list_ | @@ -499,7 +499,7 @@ Choose how downloaded book files are named and organized. **Naming Template** -Variables: {Author}, {Title}, {Year}, {User}, {OriginalName} (source filename without extension). Universal adds: {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. Rename templates are filename-only (no '/' or '\'); use Organize for folders. Applies to single-file downloads. +Variables: {Author}, {Title}, {Year}, {Language}, {User}, {OriginalName} (source filename without extension). Universal adds: {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. Rename templates are filename-only (no '/' or '\'); use Organize for folders. Applies to single-file downloads. - **Type:** string - **Default:** `{Author} - {Title} ({Year})` @@ -508,7 +508,7 @@ Variables: {Author}, {Title}, {Year}, {User}, {OriginalName} (source filename wi **Path Template** -Use / to create folders. Variables: {Author}, {Title}, {Year}, {User}, {OriginalName} (source filename without extension). Universal adds: {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. +Use / to create folders. Variables: {Author}, {Title}, {Year}, {Language}, {User}, {OriginalName} (source filename without extension). Universal adds: {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. - **Type:** string - **Default:** `{Author}/{Title} ({Year})` @@ -709,7 +709,7 @@ Choose how downloaded audiobook files are named and organized. **Naming Template** -Variables: {Author}, {Title}, {Year}, {User}, {OriginalName} (source filename without extension), {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}, {PartNumber}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. Rename templates are filename-only (no '/' or '\'); use Organize for folders. Applies to single-file downloads. +Variables: {Author}, {Title}, {Year}, {Language}, {User}, {OriginalName} (source filename without extension), {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}, {PartNumber}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. Rename templates are filename-only (no '/' or '\'); use Organize for folders. Applies to single-file downloads. - **Type:** string - **Default:** `{Author} - {Title}` @@ -718,7 +718,7 @@ Variables: {Author}, {Title}, {Year}, {User}, {OriginalName} (source filename wi **Path Template** -Use / to create folders. Variables: {Author}, {Title}, {Year}, {User}, {OriginalName} (source filename without extension), {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}, {PartNumber}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. +Use / to create folders. Variables: {Author}, {Title}, {Year}, {Language}, {User}, {OriginalName} (source filename without extension), {Series}, {SeriesPosition}, {Subtitle}, {PrimaryTitle}, {PartNumber}. Use arbitrary prefix/suffix: {Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. - **Type:** string - **Default:** `{Author}/{Title}/{Title}` diff --git a/shelfmark/config/settings.py b/shelfmark/config/settings.py index e0f3727b..cf163db2 100644 --- a/shelfmark/config/settings.py +++ b/shelfmark/config/settings.py @@ -1,6 +1,5 @@ """Core settings registration and derived configuration values.""" -import json from pathlib import Path from typing import Any @@ -15,6 +14,7 @@ from shelfmark.config.download_settings_handlers import ( check_books_destination, ) from shelfmark.config.email_settings import check_email_connection +from shelfmark.core.languages import supported_book_languages from shelfmark.core.logger import setup_logger from shelfmark.core.settings_registry import ( ActionButton, @@ -143,11 +143,8 @@ for key in ["CONFIG_DIR", "LOG_DIR", "TMP_DIR", "INGEST_DIR", "DEBUG", "DOCKERMO if hasattr(env, key): logger.debug(" %s: %s", key, getattr(env, key)) -# Load supported book languages from data file -# Path is relative to the package root, not this file -_DATA_DIR = Path(__file__).resolve().parent.parent.parent / "data" -with (_DATA_DIR / "book-languages.json").open() as file: - _SUPPORTED_BOOK_LANGUAGE = json.load(file) +# Selectable book languages, without the resolution aliases clients do not need. +_SUPPORTED_BOOK_LANGUAGE = supported_book_languages() # Directory settings BASE_DIR = Path(__file__).resolve().parent.parent.parent @@ -1012,7 +1009,7 @@ def download_settings() -> list[SettingsField]: key="TEMPLATE_RENAME", label="Naming Template", description=( - "Variables: {Author}, {Title}, {Year}, {User}, {OriginalName} " + "Variables: {Author}, {Title}, {Year}, {Language}, {User}, {OriginalName} " "(source filename without extension). Universal adds: {Series}, " "{SeriesPosition}, {Subtitle}, {PrimaryTitle}. Use arbitrary prefix/suffix: " "{Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. " @@ -1031,7 +1028,7 @@ def download_settings() -> list[SettingsField]: key="TEMPLATE_ORGANIZE", label="Path Template", description=( - "Use / to create folders. Variables: {Author}, {Title}, {Year}, {User}, " + "Use / to create folders. Variables: {Author}, {Title}, {Year}, {Language}, {User}, " "{OriginalName} (source filename without extension). Universal adds: {Series}, " "{SeriesPosition}, {Subtitle}, {PrimaryTitle}. Use arbitrary prefix/suffix: " "{Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty." @@ -1294,7 +1291,7 @@ def download_settings() -> list[SettingsField]: key="TEMPLATE_AUDIOBOOK_RENAME", label="Naming Template", description=( - "Variables: {Author}, {Title}, {Year}, {User}, {OriginalName} " + "Variables: {Author}, {Title}, {Year}, {Language}, {User}, {OriginalName} " "(source filename without extension), {Series}, {SeriesPosition}, {Subtitle}, " "{PrimaryTitle}, {PartNumber}. Use arbitrary prefix/suffix: " "{Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty. " @@ -1311,7 +1308,7 @@ def download_settings() -> list[SettingsField]: key="TEMPLATE_AUDIOBOOK_ORGANIZE", label="Path Template", description=( - "Use / to create folders. Variables: {Author}, {Title}, {Year}, {User}, " + "Use / to create folders. Variables: {Author}, {Title}, {Year}, {Language}, {User}, " "{OriginalName} (source filename without extension), {Series}, {SeriesPosition}, " "{Subtitle}, {PrimaryTitle}, {PartNumber}. Use arbitrary prefix/suffix: " "{Vol. SeriesPosition - } outputs 'Vol. 2 - ' when set, nothing when empty." diff --git a/shelfmark/core/languages.py b/shelfmark/core/languages.py new file mode 100644 index 00000000..ed1ed7aa --- /dev/null +++ b/shelfmark/core/languages.py @@ -0,0 +1,138 @@ +"""Canonical language resolution shared by every release source. + +Release sources report a language in whatever shape their upstream uses: a +two-letter code, an ISO 639-2 three-letter code in either the bibliographic or +terminological form, or an English name. They all need the same ISO 639-1 code +out the other side, so the aliases live in one place (``data/book-languages.json``) +and adding a language means editing one file. +""" + +import json +import threading +import unicodedata +from pathlib import Path + +from shelfmark.core.logger import setup_logger + +logger = setup_logger(__name__) + +LANGUAGE_DATA_PATH = Path(__file__).resolve().parents[1].parent / "data" / "book-languages.json" + +# Values a source uses to mean "we could not tell". +LANGUAGE_PLACEHOLDERS = frozenset({"", "-", "--", "unknown", "unk", "n/a", "na", "none", "null"}) + +_ALIAS_TO_CODE: dict[str, str] | None = None +_CODE_TO_NAME: dict[str, str] | None = None +_LOCK = threading.Lock() + + +# Separators that stand in for the hyphen in a subtag. The dashes turn up in +# codes copied from web pages -- "zh‑Hant" used U+2011, which renders close +# enough to both a hyphen and an underscore to go unnoticed -- and the +# underscore is the spelling Direct Download accepted before this module existed. +_SUBTAG_SEPARATORS = dict.fromkeys(map(ord, "‐‑‒–—―−﹘﹣－_"), "-") + + +def _fold(value: str) -> str: + """Casefold, strip accents, and normalize subtag separators, so 'Español' + and 'espanol', or 'zh-Hant', 'zh‑Hant' and 'zh_Hant', all match.""" + decomposed = unicodedata.normalize("NFKD", value).translate(_SUBTAG_SEPARATORS) + stripped = "".join(ch for ch in decomposed if not unicodedata.combining(ch)) + return " ".join(stripped.split()).casefold() + + +def _load() -> tuple[dict[str, str], dict[str, str]]: + global _ALIAS_TO_CODE, _CODE_TO_NAME + + if _ALIAS_TO_CODE is not None and _CODE_TO_NAME is not None: + return _ALIAS_TO_CODE, _CODE_TO_NAME + + with _LOCK: + if _ALIAS_TO_CODE is not None and _CODE_TO_NAME is not None: + return _ALIAS_TO_CODE, _CODE_TO_NAME + + alias_to_code: dict[str, str] = {} + code_to_name: dict[str, str] = {} + + try: + raw = json.loads(LANGUAGE_DATA_PATH.read_text(encoding="utf-8")) + except OSError, ValueError: + logger.exception("Failed to load language data from %s", LANGUAGE_DATA_PATH) + raw = [] + + if not isinstance(raw, list): + logger.warning("Language data at %s is not a list", LANGUAGE_DATA_PATH) + raw = [] + + for item in raw: + if not isinstance(item, dict): + continue + code = str(item.get("code") or "").strip() + name = str(item.get("language") or "").strip() + if not code: + continue + + code_to_name.setdefault(code, name or code) + + for candidate in (code, name, *(item.get("aliases") or [])): + folded = _fold(str(candidate)) + if folded and folded not in LANGUAGE_PLACEHOLDERS: + alias_to_code.setdefault(folded, code) + + _ALIAS_TO_CODE = alias_to_code + _CODE_TO_NAME = code_to_name + return alias_to_code, code_to_name + + +def normalize_language(value: object) -> str | None: + """Resolve any known spelling of a language to its ISO 639-1 code. + + Accepts a two-letter code, an ISO 639-2 three-letter code in either the + bibliographic or terminological form, or an English name. Returns None for + anything unrecognised or for the placeholders a source uses to say it does + not know, so callers can treat "no language" uniformly. + """ + if value is None: + return None + + folded = _fold(str(value)) + if not folded or folded in LANGUAGE_PLACEHOLDERS: + return None + + alias_to_code, _ = _load() + return alias_to_code.get(folded) + + +def language_name(code: str | None) -> str | None: + """Return the English name for a language code, or None if unknown.""" + if not code: + return None + + _, code_to_name = _load() + return code_to_name.get(str(code).strip()) + + +def language_alias_map() -> dict[str, str]: + """Every known alias mapped to its code, for callers doing their own matching. + + Direct Download scans free-text paths and needs the whole alias set up front + to look for, rather than resolving one candidate at a time. + """ + alias_to_code, _ = _load() + return dict(alias_to_code) + + +def supported_book_languages() -> list[dict[str, str]]: + """The selectable languages, as ``{"language": ..., "code": ...}``. + + Aliases are an implementation detail of resolution, so they are left out of + what the settings dropdown and the API hand to clients. + """ + _, code_to_name = _load() + return [{"language": name, "code": code} for code, name in code_to_name.items()] + + +def known_language_codes() -> frozenset[str]: + """Every ISO 639-1 code the bundled language data defines.""" + _, code_to_name = _load() + return frozenset(code_to_name) diff --git a/shelfmark/core/models.py b/shelfmark/core/models.py index d26515e3..d27ebd64 100644 --- a/shelfmark/core/models.py +++ b/shelfmark/core/models.py @@ -119,6 +119,7 @@ class DownloadTask: series_name: str | None = None series_position: float | None = None # Float for novellas (e.g., 1.5) subtitle: str | None = None # Book subtitle for naming templates + language: str | None = None # Release language code for the {Language} template variable # Hardlinking support original_download_path: str | None = None # Path in download client (for hardlinking) diff --git a/shelfmark/core/naming.py b/shelfmark/core/naming.py index fa908de9..4f3a73e9 100644 --- a/shelfmark/core/naming.py +++ b/shelfmark/core/naming.py @@ -4,6 +4,7 @@ import re from pathlib import Path from typing import TYPE_CHECKING +from shelfmark.core.languages import LANGUAGE_PLACEHOLDERS, normalize_language from shelfmark.core.logger import setup_logger if TYPE_CHECKING: @@ -19,6 +20,7 @@ KNOWN_TOKENS = [ "primarytitle", "originalname", "partnumber", + "language", "subtitle", "author", "series", @@ -66,6 +68,33 @@ def format_series_position(position: str | float | None) -> str: return str(position) +def normalize_language_code(language: str | None) -> str: + """Resolve a release language to the single spelling used in a path. + + Sources report the same language in different shapes: "en", "eng", "English". + All of them have to collapse to one code, or the editions they identify end + up in separate folders, which is the collision this token exists to prevent. + Placeholder values render empty so `{ (Language)}` disappears entirely + rather than labelling a folder "(unknown)". + + A language the bundled data does not know is kept, casefolded, rather than + dropped: it still separates editions, and it cannot collide with a resolved + code precisely because nothing resolves it. + """ + if not language: + return "" + + resolved = normalize_language(language) + if resolved is not None: + return resolved + + normalized = " ".join(str(language).split()).strip().casefold() + if normalized in LANGUAGE_PLACEHOLDERS: + return "" + + return normalized + + def derive_primary_title(title: str | None, subtitle: str | None) -> str: """Return the title without an explicit subtitle suffix when possible.""" title_value = " ".join(str(title or "").split()).strip() diff --git a/shelfmark/download/orchestrator.py b/shelfmark/download/orchestrator.py index 2cf0cd3d..b436d311 100644 --- a/shelfmark/download/orchestrator.py +++ b/shelfmark/download/orchestrator.py @@ -251,6 +251,7 @@ def queue_release( series_name = release_data.get("series_name") or extra.get("series_name") series_position = release_data.get("series_position") or extra.get("series_position") subtitle = release_data.get("subtitle") or extra.get("subtitle") + language = release_data.get("language") or extra.get("language") books_output_mode = ( str(config.get("BOOKS_OUTPUT_MODE", "folder", user_id=user_id) or "folder") @@ -285,6 +286,7 @@ def queue_release( series_name=series_name, series_position=series_position, subtitle=subtitle, + language=language, search_mode=search_mode, output_mode=output_mode, output_args=output_args, @@ -420,6 +422,7 @@ def serialize_task_for_retry(task: DownloadTask) -> dict[str, Any]: "series_name": getattr(task, "series_name", None), "series_position": getattr(task, "series_position", None), "subtitle": getattr(task, "subtitle", None), + "language": getattr(task, "language", None), "search_mode": search_mode, "output_mode": getattr(task, "output_mode", None), "output_args": dict(raw_output_args) if isinstance(raw_output_args, dict) else {}, @@ -477,6 +480,7 @@ def _restore_task_from_retry_payload(payload: object) -> DownloadTask | None: series_name=normalize_optional_text(payload.get("series_name")), series_position=_optional_number(payload.get("series_position")), subtitle=normalize_optional_text(payload.get("subtitle")), + language=normalize_optional_text(payload.get("language")), search_mode=search_mode, output_mode=normalize_optional_text(payload.get("output_mode")), output_args=dict(output_args) if isinstance(output_args, dict) else {}, diff --git a/shelfmark/download/postprocess/custom_script.py b/shelfmark/download/postprocess/custom_script.py index 21ac6fd8..73ce003c 100644 --- a/shelfmark/download/postprocess/custom_script.py +++ b/shelfmark/download/postprocess/custom_script.py @@ -229,6 +229,7 @@ def _build_custom_script_payload( "series_name": context.task.series_name, "series_position": context.task.series_position, "subtitle": context.task.subtitle, + "language": context.task.language, "original_download_path": context.task.original_download_path, }, "output": { diff --git a/shelfmark/download/postprocess/transfer.py b/shelfmark/download/postprocess/transfer.py index a0f5b7f5..ba0a8320 100644 --- a/shelfmark/download/postprocess/transfer.py +++ b/shelfmark/download/postprocess/transfer.py @@ -12,6 +12,7 @@ from shelfmark.core.naming import ( assign_part_numbers, build_library_path, derive_primary_title, + normalize_language_code, parse_naming_template, sanitize_filename, ) @@ -63,6 +64,7 @@ def build_metadata_dict(task: DownloadTask) -> dict: "Year": task.year, "Series": task.series_name, "SeriesPosition": task.series_position, + "Language": normalize_language_code(task.language), "User": task.username, } diff --git a/shelfmark/metadata_providers/README.md b/shelfmark/metadata_providers/README.md index 5a789441..8124d67f 100644 --- a/shelfmark/metadata_providers/README.md +++ b/shelfmark/metadata_providers/README.md @@ -24,12 +24,12 @@ Dataclass representing a book from a metadata provider: ```python @dataclass class BookMetadata: - provider: str # Internal provider name (e.g., "hardcover") - provider_id: str # ID in that provider's system + provider: str # Internal provider name (e.g., "hardcover") + provider_id: str # ID in that provider's system title: str # Optional fields - provider_display_name: str # Human-readable name (e.g., "Hardcover") + provider_display_name: str # Human-readable name (e.g., "Hardcover") authors: List[str] isbn_10: str isbn_13: str @@ -39,7 +39,7 @@ class BookMetadata: publish_year: int language: str genres: List[str] - source_url: str # Link to book on provider's site + source_url: str # Link to book on provider's site display_fields: List[DisplayField] # Provider-specific display data ``` @@ -50,9 +50,9 @@ Provider-specific metadata for UI cards (ratings, page counts, reader counts, et ```python @dataclass class DisplayField: - label: str # e.g., "Rating", "Pages", "Readers" - value: str # e.g., "4.5", "496", "8,041" - icon: str # Icon name: "star", "book", "users", "editions" + label: str # e.g., "Rating", "Pages", "Readers" + value: str # e.g., "4.5", "496", "8,041" + icon: str # Icon name: "star", "book", "users", "editions" ``` ### MetadataSearchOptions @@ -64,7 +64,7 @@ Unified search options that work across all providers: class MetadataSearchOptions: query: str search_type: SearchType = SearchType.GENERAL # GENERAL, TITLE, AUTHOR, ISBN - language: str = None # ISO 639-1 code (e.g., "en") + language: str = None # ISO 639-1 code (e.g., "en") sort: SortOrder = SortOrder.RELEVANCE limit: int = 40 page: int = 1 @@ -88,10 +88,10 @@ All providers must implement this interface: ```python class MetadataProvider(ABC): - name: str # Internal identifier - display_name: str # Human-readable name - requires_auth: bool # True if API key required - supported_sorts: List[SortOrder] # Supported sort options + name: str # Internal identifier + display_name: str # Human-readable name + requires_auth: bool # True if API key required + supported_sorts: List[SortOrder] # Supported sort options @abstractmethod def search(self, options: MetadataSearchOptions) -> List[BookMetadata]: @@ -121,9 +121,9 @@ class MetadataProvider(ABC): ```python from shelfmark.metadata_providers import register_provider - @register_provider("my_provider") -class MyProvider(MetadataProvider): ... +class MyProvider(MetadataProvider): + ... ``` ### Getting Providers @@ -281,13 +281,11 @@ from shelfmark.config.env import ( METADATA_CACHE_BOOK_TTL, ) - @cacheable(ttl=METADATA_CACHE_SEARCH_TTL, key_prefix="myprovider:search") def _search_cached(self, cache_key: str, options: MetadataSearchOptions): # Cached search implementation pass - @cacheable(ttl=METADATA_CACHE_BOOK_TTL, key_prefix="myprovider:book") def get_book(self, book_id: str): # Cached book lookup @@ -304,7 +302,6 @@ from shelfmark.metadata_providers.openlibrary import RateLimiter # 90 requests per 60 seconds rate_limiter = RateLimiter(max_requests=90, window_seconds=60) - def make_request(self): rate_limiter.wait_if_needed() # Blocks if rate limited # ... make request diff --git a/shelfmark/release_sources/audiobookbay/source.py b/shelfmark/release_sources/audiobookbay/source.py index 66700bf4..c672c34a 100644 --- a/shelfmark/release_sources/audiobookbay/source.py +++ b/shelfmark/release_sources/audiobookbay/source.py @@ -9,6 +9,7 @@ if TYPE_CHECKING: from shelfmark.metadata_providers import BookMetadata from shelfmark.core.config import config +from shelfmark.core.languages import normalize_language from shelfmark.core.logger import setup_logger from shelfmark.release_sources import ( ColumnAlign, @@ -43,41 +44,6 @@ def _coerce_positive_int(value: object, default: int) -> int: # Map language names to ISO 639-1 codes (matching frontend color maps) -LANGUAGE_MAP = { - "english": "en", - "spanish": "es", - "french": "fr", - "german": "de", - "italian": "it", - "portuguese": "pt", - "russian": "ru", - "japanese": "ja", - "chinese": "zh", - "dutch": "nl", - "swedish": "sv", - "norwegian": "no", - "danish": "da", - "finnish": "fi", - "polish": "pl", - "czech": "cs", - "hungarian": "hu", - "korean": "ko", - "arabic": "ar", - "hebrew": "he", - "turkish": "tr", - "greek": "el", - "hindi": "hi", - "thai": "th", - "vietnamese": "vi", - "indonesian": "id", - "ukrainian": "uk", - "romanian": "ro", - "bulgarian": "bg", - "catalan": "ca", - "croatian": "hr", - "slovenian": "sl", - "serbian": "sr", -} def _split_title_and_author(raw_title: str) -> tuple[str, str | None]: @@ -119,8 +85,9 @@ def _map_language(language: str) -> str | None: if not language: return None - lang_lower = language.lower().strip() - return LANGUAGE_MAP.get(lang_lower, lang_lower) + # Fall back to the raw value so an unrecognised language is still shown + # rather than silently dropped from the release row. + return normalize_language(language) or language.lower().strip() def _parse_bitrate_to_kbps(bitrate: str | None) -> int | None: diff --git a/shelfmark/release_sources/direct_download.py b/shelfmark/release_sources/direct_download.py index bdc91983..488ff403 100644 --- a/shelfmark/release_sources/direct_download.py +++ b/shelfmark/release_sources/direct_download.py @@ -18,6 +18,7 @@ from bs4.element import NavigableString from shelfmark.config.env import DEBUG_SKIP_SOURCES, TMP_DIR from shelfmark.core.config import config +from shelfmark.core.languages import language_alias_map from shelfmark.core.logger import setup_logger from shelfmark.core.models import DownloadTask, SearchFilters, build_filename from shelfmark.core.utils import CONTENT_TYPES, get_aa_content_type_dir @@ -267,7 +268,7 @@ def _fold_text(value: str) -> str: def _language_alias_to_code() -> dict[str, str]: - """Build alias→code map from bundled language metadata (lazy, cached).""" + """Alias to code map, delegating to the shared language data.""" global _LANGUAGE_ALIAS_TO_CODE cached = _LANGUAGE_ALIAS_TO_CODE if cached is not None: @@ -278,35 +279,7 @@ def _language_alias_to_code() -> dict[str, str]: if cached is not None: return cached - mapping: dict[str, str] = {} - data_path = Path(__file__).resolve().parents[2] / "data" / "book-languages.json" - - try: - raw = json.loads(data_path.read_text(encoding="utf-8")) - except OSError, ValueError, TypeError: - _LANGUAGE_ALIAS_TO_CODE = {} - return _LANGUAGE_ALIAS_TO_CODE - - if not isinstance(raw, list): - _LANGUAGE_ALIAS_TO_CODE = {} - return _LANGUAGE_ALIAS_TO_CODE - - for item in raw: - if not isinstance(item, dict): - continue - code = _normalize_language_token(str(item.get("code", ""))) - name = _normalize_language_token(str(item.get("language", ""))) - if not code: - continue - mapping.setdefault(code, code) - mapping.setdefault(code.replace("-", "_"), code) - mapping.setdefault(code.split("-")[0], code) - mapping.setdefault(_fold_text(code), code) - if name: - mapping.setdefault(name, code) - mapping.setdefault(_fold_text(name), code) - - _LANGUAGE_ALIAS_TO_CODE = mapping + _LANGUAGE_ALIAS_TO_CODE = language_alias_map() return _LANGUAGE_ALIAS_TO_CODE diff --git a/shelfmark/release_sources/prowlarr/source.py b/shelfmark/release_sources/prowlarr/source.py index 904c05e7..0205d351 100644 --- a/shelfmark/release_sources/prowlarr/source.py +++ b/shelfmark/release_sources/prowlarr/source.py @@ -12,6 +12,7 @@ if TYPE_CHECKING: from shelfmark.metadata_providers import BookMetadata from shelfmark.core.config import config +from shelfmark.core.languages import normalize_language from shelfmark.core.logger import setup_logger from shelfmark.core.request_helpers import normalize_optional_text from shelfmark.core.search_plan import ReleaseSearchVariant @@ -226,50 +227,6 @@ AUDIOBOOK_FORMATS = ["m4b", "mp3", "m4a", "flac", "ogg", "wma", "aac", "wav", "o # Combined list for format detection (audiobook formats first for priority) ALL_BOOK_FORMATS = AUDIOBOOK_FORMATS + EBOOK_FORMATS -# Map 3-char MAM language codes to 2-char ISO codes used by frontend color maps -MAM_LANGUAGE_MAP = { - "eng": "en", - "ita": "it", - "spa": "es", - "fra": "fr", - "fre": "fr", - "ger": "de", - "deu": "de", - "por": "pt", - "rus": "ru", - "jpn": "ja", - "jap": "ja", - "chi": "zh", - "zho": "zh", - "dut": "nl", - "nld": "nl", - "swe": "sv", - "nor": "no", - "dan": "da", - "fin": "fi", - "pol": "pl", - "cze": "cs", - "ces": "cs", - "hun": "hu", - "kor": "ko", - "ara": "ar", - "heb": "he", - "tur": "tr", - "gre": "el", - "ell": "el", - "hin": "hi", - "tha": "th", - "vie": "vi", - "ind": "id", - "ukr": "uk", - "rom": "ro", - "ron": "ro", - "bul": "bg", - "cat": "ca", - "hrv": "hr", - "slv": "sl", - "srp": "sr", -} # Backend safeguard: cap total Prowlarr search time per request. PROWLARR_SEARCH_TIMEOUT_SECONDS = 120.0 @@ -317,8 +274,9 @@ def _extract_mam_language(raw_title: str) -> str | None: for token in tokens: lang_code = token.lower() - if lang_code in MAM_LANGUAGE_MAP: - return MAM_LANGUAGE_MAP[lang_code] + resolved = normalize_language(lang_code) + if resolved is not None: + return resolved return None diff --git a/src/frontend/src/App.tsx b/src/frontend/src/App.tsx index a1990eec..2558edf1 100644 --- a/src/frontend/src/App.tsx +++ b/src/frontend/src/App.tsx @@ -1098,6 +1098,9 @@ function App() { series_name: book.series_name, series_position: book.series_position, subtitle: book.subtitle, + // From the release, never the book: book.language is the provider's + // canonical edition, which would mislabel a translated release. + language: release.language ?? undefined, }; }, [], diff --git a/src/frontend/src/services/api.ts b/src/frontend/src/services/api.ts index fcba6413..9d19b64b 100644 --- a/src/frontend/src/services/api.ts +++ b/src/frontend/src/services/api.ts @@ -500,6 +500,7 @@ export type DownloadReleasePayload = { series_name?: string; series_position?: number; subtitle?: string; + language?: string; // Release language code, for the {Language} naming variable search_author?: string; search_mode?: 'direct' | 'universal'; }; diff --git a/src/frontend/src/tests/namingTemplatePreview.test.ts b/src/frontend/src/tests/namingTemplatePreview.test.ts index 7116a3e1..e17eae04 100644 --- a/src/frontend/src/tests/namingTemplatePreview.test.ts +++ b/src/frontend/src/tests/namingTemplatePreview.test.ts @@ -48,4 +48,47 @@ describe('namingTemplatePreview', () => { expect(preview.unknownTokens).toEqual(['NotAThing']); expect(preview.value).toBe('Arthur Conan Doyle'); }); + + it('offers Language as a core variable for both content types', () => { + const language = NAMING_TEMPLATE_TOKENS.find((token) => token.token === 'Language'); + + expect(language?.group).toBe('Core'); + expect(language?.audiobookOnly).toBeFalsy(); + }); + + it('separates translated editions into their own folder', () => { + const template = '{Author}/{Title}{ (Language)}'; + + const swedish = renderNamingTemplate( + template, + { + ...SAMPLE_NAMING_METADATA, + Author: 'Andy Weir', + Title: 'Project Hail Mary', + Language: 'sv', + }, + { allowPathSeparators: true }, + ); + const english = renderNamingTemplate( + template, + { ...SAMPLE_NAMING_METADATA, Author: 'Andy Weir', Title: 'Project Hail Mary', Language: '' }, + { allowPathSeparators: true }, + ); + + expect(swedish.value).toBe('Andy Weir/Project Hail Mary (sv)'); + expect(english.value).toBe('Andy Weir/Project Hail Mary'); + expect(swedish.value).not.toBe(english.value); + }); + + it('keeps the picker and the known-token list in lockstep', () => { + // KNOWN_TOKENS is a hand-maintained duplicate of the Python list. A token + // added to the picker but not to it would render as an unknown variable. + for (const token of NAMING_TEMPLATE_TOKENS) { + const preview = renderNamingTemplate(`{${token.token}}`, SAMPLE_NAMING_METADATA, { + allowPathSeparators: true, + }); + + expect(preview.unknownTokens, `${token.token} is missing from KNOWN_TOKENS`).toEqual([]); + } + }); }); diff --git a/src/frontend/src/tests/requestPayload.test.ts b/src/frontend/src/tests/requestPayload.test.ts index 181cb86d..f4e4bbf8 100644 --- a/src/frontend/src/tests/requestPayload.test.ts +++ b/src/frontend/src/tests/requestPayload.test.ts @@ -138,4 +138,29 @@ describe('requestPayload utilities', () => { ); expect(getRequestSuccessMessage(payloadUntitled)).toBe('Request submitted: Untitled'); }); + + it('takes the language from the release, not the book', () => { + // book.language is the metadata provider's canonical edition. Using it would + // label a Swedish release "en" and put both editions back in one folder. + const data = buildReleaseDataFromMetadataRelease( + { ...baseBook, language: 'en' }, + { ...baseRelease, language: 'sv' }, + 'ebook', + ); + + expect(data.language).toBe('sv'); + }); + + it('leaves language undefined when the release has none', () => { + const data = buildReleaseDataFromMetadataRelease(baseBook, baseRelease, 'ebook'); + + expect(data.language).toBeUndefined(); + }); + + it('uses the book language when browsing a source directly', () => { + // In direct mode the book record IS the release record. + const data = buildReleaseDataFromDirectBook({ ...baseBook, language: 'de' }); + + expect(data.language).toBe('de'); + }); }); diff --git a/src/frontend/src/utils/namingTemplatePreview.ts b/src/frontend/src/utils/namingTemplatePreview.ts index d1dc9700..0e09c57f 100644 --- a/src/frontend/src/utils/namingTemplatePreview.ts +++ b/src/frontend/src/utils/namingTemplatePreview.ts @@ -48,6 +48,13 @@ export const NAMING_TEMPLATE_TOKENS: NamingTemplateToken[] = [ value: '1902', group: 'Core', }, + { + token: 'Language', + label: 'Language', + description: 'Release language code, so translations do not share a folder', + value: 'en', + group: 'Core', + }, { token: 'User', label: 'User', @@ -98,6 +105,7 @@ const KNOWN_TOKENS = [ 'primarytitle', 'originalname', 'partnumber', + 'language', 'subtitle', 'author', 'series', diff --git a/src/frontend/src/utils/requestPayload.ts b/src/frontend/src/utils/requestPayload.ts index 60423475..6cb1715d 100644 --- a/src/frontend/src/utils/requestPayload.ts +++ b/src/frontend/src/utils/requestPayload.ts @@ -90,6 +90,7 @@ export const buildReleaseDataFromMetadataRelease = ( series_position: book.series_position, series_count: book.series_count, subtitle: book.subtitle, + language: release.language, ...(isSourceBackedReleaseContext ? { search_mode: 'direct' as const } : {}), }; }; @@ -106,6 +107,8 @@ export const buildReleaseDataFromDirectBook = (book: Book) => { size: book.size, preview: book.preview, content_type: 'ebook' as const, + // Browsing a source directly means the book record IS the release record. + language: book.language, search_mode: 'direct' as const, }; }; diff --git a/tests/config/test_generate_env_docs.py b/tests/config/test_generate_env_docs.py index 98e35256..214f3dd9 100644 --- a/tests/config/test_generate_env_docs.py +++ b/tests/config/test_generate_env_docs.py @@ -53,7 +53,7 @@ def test_generated_env_docs_include_custom_component_value_fields() -> None: assert ( "| `TEMPLATE_AUDIOBOOK_ORGANIZE` | Use / to create folders. Variables: " - "{Author}, {Title}, {Year}, {User}, {OriginalName} " + "{Author}, {Title}, {Year}, {Language}, {User}, {OriginalName} " "(source filename without extension), {Series}, {SeriesPosition}, {Subtitle}, " "{PrimaryTitle}, {PartNumber}. Use arbitrary prefix/suffix:" ) in docs diff --git a/tests/core/fixtures/language_alias_baseline.json b/tests/core/fixtures/language_alias_baseline.json new file mode 100644 index 00000000..89fee6a8 --- /dev/null +++ b/tests/core/fixtures/language_alias_baseline.json @@ -0,0 +1,280 @@ +{ + "_comment": "Frozen snapshot of the per-source language handling as it was before consolidation into shelfmark.core.languages. prowlarr_three_letter and audiobookbay_names were explicit maps; direct_download_derived is what its loader built from the data file, including the underscore spelling of a hyphenated code. Every alias here must still resolve to the same code. The one deliberate change: Traditional Chinese was canonically 'zh‑Hant' with a U+2011 non-breaking hyphen and is now the ASCII 'zh-Hant', so expectations for it name the new code while the old spelling remains a resolvable alias.", + "audiobookbay_names": { + "afrikaans": "af", + "arabic": "ar", + "bangla": "bn", + "bengali": "bn", + "bosnian": "bs", + "bulgarian": "bg", + "burmese": "my", + "catalan": "ca", + "chinese": "zh", + "croatian": "hr", + "czech": "cs", + "danish": "da", + "dutch": "nl", + "english": "en", + "estonian": "et", + "farsi": "fa", + "filipino": "fil", + "finnish": "fi", + "french": "fr", + "german": "de", + "greek": "el", + "gujarati": "gu", + "hebrew": "he", + "hindi": "hi", + "hungarian": "hu", + "icelandic": "is", + "indonesian": "id", + "irish": "ga", + "italian": "it", + "japanese": "ja", + "javanese": "jv", + "kannada": "kn", + "korean": "ko", + "latin": "la", + "latvian": "lv", + "lithuanian": "lt", + "malay": "ms", + "malayalam": "ml", + "manx": "gv", + "marathi": "mr", + "norwegian": "no", + "persian": "fa", + "polish": "pl", + "portuguese": "pt", + "punjabi": "pa", + "romanian": "ro", + "russian": "ru", + "sanskrit": "sa", + "scottish gaelic": "gd", + "serbian": "sr", + "slovenian": "sl", + "spanish": "es", + "swedish": "sv", + "tagalog": "fil", + "tamil": "ta", + "telugu": "te", + "thai": "th", + "turkish": "tr", + "ukrainian": "uk", + "urdu": "ur", + "vietnamese": "vi" + }, + "direct_download_derived": { + "af": "af", + "afrikaans": "af", + "albanian": "sq", + "ar": "ar", + "arabic": "ar", + "armenian": "hy", + "az": "az", + "azerbaijani": "az", + "ba": "ba", + "bangla": "bn", + "bashkir": "ba", + "be": "be", + "belarusian": "be", + "bg": "bg", + "bn": "bn", + "bo": "bo", + "bulgarian": "bg", + "ca": "ca", + "catalan": "ca", + "chinese": "zh", + "croatian": "hr", + "cs": "cs", + "czech": "cs", + "da": "da", + "danish": "da", + "de": "de", + "dutch": "nl", + "el": "el", + "en": "en", + "english": "en", + "eo": "eo", + "es": "es", + "esperanto": "eo", + "fa": "fa", + "fi": "fi", + "fil": "fil", + "filipino": "fil", + "finnish": "fi", + "fr": "fr", + "french": "fr", + "ga": "ga", + "galician": "gl", + "georgian": "ka", + "german": "de", + "gl": "gl", + "greek": "el", + "gu": "gu", + "gujarati": "gu", + "he": "he", + "hebrew": "he", + "hi": "hi", + "hindi": "hi", + "hr": "hr", + "hu": "hu", + "hungarian": "hu", + "hy": "hy", + "id": "id", + "indonesian": "id", + "irish": "ga", + "it": "it", + "italian": "it", + "ja": "ja", + "japanese": "ja", + "javanese": "jv", + "jv": "jv", + "ka": "ka", + "kannada": "kn", + "kazakh": "kk", + "kinyarwanda": "rw", + "kk": "kk", + "kn": "kn", + "ko": "ko", + "korean": "ko", + "ky": "ky", + "kyrgyz": "ky", + "la": "la", + "latin": "la", + "latvian": "lv", + "lithuanian": "lt", + "lt": "lt", + "lv": "lv", + "malay": "ms", + "malayalam": "ml", + "marathi": "mr", + "ml": "ml", + "mn": "mn", + "mongolian": "mn", + "mr": "mr", + "ms": "ms", + "nl": "nl", + "no": "no", + "norwegian": "no", + "pa": "pa", + "persian": "fa", + "pl": "pl", + "polish": "pl", + "portuguese": "pt", + "pt": "pt", + "punjabi": "pa", + "qu": "qu", + "quechua": "qu", + "ro": "ro", + "romanian": "ro", + "ru": "ru", + "russian": "ru", + "rw": "rw", + "serbian": "sr", + "shan": "shn", + "shn": "shn", + "sk": "sk", + "sl": "sl", + "slovak": "sk", + "slovenian": "sl", + "spanish": "es", + "sq": "sq", + "sr": "sr", + "sv": "sv", + "sw": "sw", + "swahili": "sw", + "swedish": "sv", + "ta": "ta", + "tamil": "ta", + "te": "te", + "telugu": "te", + "th": "th", + "thai": "th", + "tibetan": "bo", + "tr": "tr", + "traditional chinese": "zh-Hant", + "turkish": "tr", + "ug": "ug", + "uk": "uk", + "ukrainian": "uk", + "ur": "ur", + "urdu": "ur", + "uyghur": "ug", + "vi": "vi", + "vietnamese": "vi", + "zh": "zh", + "zh‑hant": "zh-Hant" + }, + "prowlarr_three_letter": { + "afr": "af", + "ara": "ar", + "ben": "bn", + "bos": "bs", + "bul": "bg", + "bur": "my", + "cat": "ca", + "ces": "cs", + "chi": "zh", + "cze": "cs", + "dan": "da", + "deu": "de", + "dut": "nl", + "ell": "el", + "eng": "en", + "est": "et", + "fas": "fa", + "fin": "fi", + "fra": "fr", + "fre": "fr", + "ger": "de", + "gla": "gd", + "gle": "ga", + "glv": "gv", + "gre": "el", + "guj": "gu", + "heb": "he", + "hin": "hi", + "hrv": "hr", + "hun": "hu", + "ice": "is", + "ind": "id", + "isl": "is", + "ita": "it", + "jap": "ja", + "jav": "jv", + "jpn": "ja", + "kan": "kn", + "kor": "ko", + "lat": "la", + "lav": "lv", + "lit": "lt", + "mal": "ml", + "mar": "mr", + "may": "ms", + "msa": "ms", + "mya": "my", + "nld": "nl", + "nor": "no", + "pan": "pa", + "per": "fa", + "pol": "pl", + "por": "pt", + "rom": "ro", + "ron": "ro", + "rus": "ru", + "san": "sa", + "slv": "sl", + "spa": "es", + "srp": "sr", + "swe": "sv", + "tam": "ta", + "tel": "te", + "tgl": "fil", + "tha": "th", + "tur": "tr", + "ukr": "uk", + "urd": "ur", + "vie": "vi", + "zho": "zh" + } +} diff --git a/tests/core/test_config_api.py b/tests/core/test_config_api.py index 4a1a8fb6..bd994d42 100644 --- a/tests/core/test_config_api.py +++ b/tests/core/test_config_api.py @@ -131,3 +131,46 @@ def test_frontend_dist_resolves_from_repo_root(main_module): assert main_module.PROJECT_ROOT == expected_project_root assert main_module.FRONTEND_DIST == expected_project_root / "frontend-dist" + + +def test_config_endpoint_serves_languages_without_resolution_aliases(main_module, client): + """book_languages is a client contract, not a dump of the language data file. + + data/book-languages.json also carries the aliases used to resolve a source's + spelling of a language to a code. Those are server-side only: the frontend + Language type is {code, language}, and shipping the aliases inflated every + config response by around 40%. + """ + _set_session(client, user_id="reader-1", db_user_id=1, is_admin=False) + + with ( + patch("shelfmark.config.env._is_config_dir_writable", return_value=True), + patch("shelfmark.core.onboarding.is_onboarding_complete", return_value=True), + ): + resp = client.get("/api/config") + + assert resp.status_code == 200 + languages = resp.get_json()["book_languages"] + + assert languages, "no languages served" + offending = [entry for entry in languages if set(entry) != {"code", "language"}] + assert offending == [], f"unexpected keys leaked to clients: {offending[:3]}" + + +def test_language_data_file_is_only_read_by_the_shared_module(main_module): + """Reading data/book-languages.json anywhere else reintroduces the drift the + shared module exists to prevent, and bypasses the alias handling.""" + del main_module + + repo_root = Path(__file__).resolve().parents[2] + allowed = {Path("shelfmark/core/languages.py")} + + offenders = [] + for path in (repo_root / "shelfmark").rglob("*.py"): + relative = path.relative_to(repo_root) + if relative in allowed: + continue + if "book-languages" in path.read_text(encoding="utf-8"): + offenders.append(str(relative)) + + assert offenders == [], f"should use shelfmark.core.languages instead: {offenders}" diff --git a/tests/core/test_language_template_variable.py b/tests/core/test_language_template_variable.py new file mode 100644 index 00000000..04422754 --- /dev/null +++ b/tests/core/test_language_template_variable.py @@ -0,0 +1,196 @@ +"""Tests for the {Language} template variable. + +Different-language editions of one book resolve to the same title, so without a +language token they render to the same path and land in one folder. Audiobookshelf +treats a folder as exactly one library item, so the two editions become a single +book with both files as tracks (calibrain/shelfmark#1138). +""" + +import pytest + +from shelfmark.core.models import DownloadTask +from shelfmark.core.naming import ( + KNOWN_TOKENS, + normalize_language_code, + parse_naming_template, +) +from shelfmark.download.orchestrator import ( + _restore_task_from_retry_payload, + serialize_task_for_retry, +) +from shelfmark.download.postprocess.transfer import build_metadata_dict + + +class TestLanguageInKnownTokens: + def test_language_in_known_tokens(self): + assert "language" in KNOWN_TOKENS + + def test_language_token_parsed(self): + assert parse_naming_template("{Language}", {"Language": "sv"}) == "sv" + + def test_language_token_case_insensitive(self): + assert parse_naming_template("{language}", {"Language": "sv"}) == "sv" + + +class TestKnownTokensOrdering: + """find_placeholder() does a substring find over KNOWN_TOKENS in list order. + + Nothing else guards this contract, so a future token added in the wrong + position would silently shadow an existing one. + """ + + def test_tokens_are_ordered_longest_first(self): + lengths = [len(token) for token in KNOWN_TOKENS] + assert lengths == sorted(lengths, reverse=True) + + def test_no_token_is_shadowed_by_an_earlier_substring(self): + for shorter_index, shorter in enumerate(KNOWN_TOKENS): + for longer_index, longer in enumerate(KNOWN_TOKENS): + if shorter is longer or shorter not in longer: + continue + assert shorter_index > longer_index, ( + f"{shorter!r} precedes {longer!r} and would shadow it" + ) + + +class TestLanguageTemplateSubstitution: + """The acceptance cases from the issue.""" + + TEMPLATE = "{Author}/{Title}{ (Language)}/{Author} - {Title}" + BASE = {"Author": "Andy Weir", "Title": "Project Hail Mary"} + + def test_translated_edition_gets_its_own_folder(self): + result = parse_naming_template( + self.TEMPLATE, {**self.BASE, "Language": "sv"}, allow_path_separators=True + ) + assert result == "Andy Weir/Project Hail Mary (sv)/Andy Weir - Project Hail Mary" + + def test_untagged_edition_is_unchanged(self): + result = parse_naming_template( + self.TEMPLATE, {**self.BASE, "Language": None}, allow_path_separators=True + ) + assert result == "Andy Weir/Project Hail Mary/Andy Weir - Project Hail Mary" + + def test_the_two_editions_do_not_collide(self): + english = parse_naming_template( + self.TEMPLATE, {**self.BASE, "Language": None}, allow_path_separators=True + ) + swedish = parse_naming_template( + self.TEMPLATE, {**self.BASE, "Language": "sv"}, allow_path_separators=True + ) + assert english != swedish + + def test_language_as_a_leading_folder(self): + result = parse_naming_template( + "{Language/}{Author}/{Title}", + {**self.BASE, "Language": "sv"}, + allow_path_separators=True, + ) + assert result == "sv/Andy Weir/Project Hail Mary" + + def test_language_in_a_filename_template(self): + result = parse_naming_template( + "{Author} - {Title}{ (Language)}", {**self.BASE, "Language": "sv"} + ) + assert result == "Andy Weir - Project Hail Mary (sv)" + + def test_language_is_sanitized(self): + result = parse_naming_template("{Title}{ (Language)}", {"Title": "Book", "Language": "s/v"}) + assert "/" not in result + + +class TestNormalizeLanguageCode: + def test_lowercases(self): + assert normalize_language_code("EN") == "en" + assert normalize_language_code("Sv") == "sv" + + def test_strips_whitespace(self): + assert normalize_language_code(" sv ") == "sv" + + def test_placeholders_render_empty(self): + for placeholder in ("unknown", "unk", "n/a", "na", "-", "--", "none", "null", ""): + assert normalize_language_code(placeholder) == "", placeholder + + def test_placeholders_are_matched_case_insensitively(self): + assert normalize_language_code("Unknown") == "" + + def test_none_renders_empty(self): + assert normalize_language_code(None) == "" + + +class TestBuildMetadataWithLanguage: + def test_language_reaches_the_template_metadata(self): + task = DownloadTask(task_id="t", source="prowlarr", title="Book", language="sv") + assert build_metadata_dict(task)["Language"] == "sv" + + def test_language_is_normalized_on_the_way_out(self): + task = DownloadTask(task_id="t", source="prowlarr", title="Book", language="SV") + assert build_metadata_dict(task)["Language"] == "sv" + + def test_placeholder_language_does_not_reach_the_path(self): + # Anna's Archive reports the literal string "unknown" when it cannot tell. + task = DownloadTask(task_id="t", source="direct", title="Book", language="unknown") + assert build_metadata_dict(task)["Language"] == "" + + def test_missing_language_renders_empty(self): + task = DownloadTask(task_id="t", source="prowlarr", title="Book") + assert build_metadata_dict(task)["Language"] == "" + + +class TestLanguageSurvivesRetry: + """DownloadTask is not rebuilt from dataclasses.fields(), so each of the + three orchestrator sites has to carry the field explicitly.""" + + def test_roundtrip_preserves_language(self): + task = DownloadTask(task_id="t", source="prowlarr", title="Book", language="sv") + restored = _restore_task_from_retry_payload(serialize_task_for_retry(task)) + assert restored is not None + assert restored.language == "sv" + + def test_legacy_payload_without_language_restores_cleanly(self): + task = DownloadTask(task_id="t", source="prowlarr", title="Book", language="sv") + payload = serialize_task_for_retry(task) + del payload["language"] + + restored = _restore_task_from_retry_payload(payload) + + assert restored is not None + assert restored.language is None + + +class TestEverySpellingCollapsesToOneFolder: + """Sources report the same language differently; if the token rendered each + spelling verbatim they would land in separate folders, which is the exact + collision this token exists to prevent (reported on PR #1142).""" + + TEMPLATE = "{Author}/{Title}{ (Language)}/{Title}" + + def _folder(self, language): + task = DownloadTask( + task_id="t", source="prowlarr", title="Dune", author="Frank Herbert", language=language + ) + return parse_naming_template( + self.TEMPLATE, build_metadata_dict(task), allow_path_separators=True + ) + + @pytest.mark.parametrize( + "spellings", + [ + ("en", "eng", "English", "english", "ENG", " Eng "), + ("sv", "swe", "Swedish"), + ("de", "ger", "deu", "German"), + ("ml", "mal", "Malayalam"), + ("fa", "per", "fas", "Farsi", "Persian"), + ], + ) + def test_all_spellings_of_a_language_share_one_folder(self, spellings): + rendered = {self._folder(spelling) for spelling in spellings} + assert len(rendered) == 1, f"{spellings} produced {sorted(rendered)}" + + def test_a_language_we_cannot_resolve_is_kept_rather_than_dropped(self): + # It still separates editions, and cannot collide with a resolved code + # precisely because nothing resolves to it. + assert "klingon" in self._folder("Klingon") + + def test_different_languages_still_get_different_folders(self): + assert self._folder("English") != self._folder("Swedish") diff --git a/tests/core/test_languages.py b/tests/core/test_languages.py new file mode 100644 index 00000000..60ac6804 --- /dev/null +++ b/tests/core/test_languages.py @@ -0,0 +1,256 @@ +"""Tests for the shared language resolution used by every release source.""" + +import json +from pathlib import Path + +import pytest + +from shelfmark.core.languages import ( + LANGUAGE_DATA_PATH, + known_language_codes, + language_alias_map, + language_name, + normalize_language, + supported_book_languages, +) + +BASELINE = json.loads( + (Path(__file__).parent / "fixtures" / "language_alias_baseline.json").read_text( + encoding="utf-8" + ) +) + + +class TestBaselineEquivalence: + """Every alias the per-source maps used to handle must still resolve the same. + + These maps lived in prowlarr/source.py and audiobookbay/source.py before they + were consolidated here. The fixture is a frozen snapshot taken before the + move, so a regression shows up as a concrete alias rather than a vague + behaviour change. + """ + + @pytest.mark.parametrize( + ("alias", "expected"), sorted(BASELINE["prowlarr_three_letter"].items()) + ) + def test_prowlarr_three_letter_aliases_unchanged(self, alias, expected): + assert normalize_language(alias) == expected + + @pytest.mark.parametrize(("alias", "expected"), sorted(BASELINE["audiobookbay_names"].items())) + def test_audiobookbay_names_unchanged(self, alias, expected): + assert normalize_language(alias) == expected + + @pytest.mark.parametrize( + ("alias", "expected"), sorted(BASELINE["direct_download_derived"].items()) + ) + def test_direct_download_derived_aliases_unchanged(self, alias, expected): + # Direct Download built its aliases from the data file rather than a + # literal map, so consolidating silently dropped the spellings it + # derived -- notably the underscore form of a hyphenated code. + assert normalize_language(alias) == expected + + +class TestNormalizeLanguage: + def test_accepts_two_letter_codes(self): + assert normalize_language("en") == "en" + assert normalize_language("sv") == "sv" + + def test_accepts_three_letter_codes_in_both_iso_639_2_forms(self): + # Bibliographic and terminological forms differ for these. + assert normalize_language("ger") == normalize_language("deu") == "de" + assert normalize_language("fre") == normalize_language("fra") == "fr" + assert normalize_language("per") == normalize_language("fas") == "fa" + assert normalize_language("ice") == normalize_language("isl") == "is" + assert normalize_language("may") == normalize_language("msa") == "ms" + + def test_accepts_english_names(self): + assert normalize_language("Swedish") == "sv" + assert normalize_language("Scottish Gaelic") == "gd" + + def test_is_case_and_whitespace_insensitive(self): + assert normalize_language(" ENG ") == "en" + assert normalize_language("sWeDiSh") == "sv" + + def test_returns_none_for_placeholders(self): + for placeholder in ("unknown", "unk", "n/a", "na", "-", "--", "none", "null", "", " "): + assert normalize_language(placeholder) is None, placeholder + + def test_returns_none_for_unknown_values(self): + assert normalize_language("xyz") is None + assert normalize_language("Klingon") is None + + def test_returns_none_for_none(self): + assert normalize_language(None) is None + + +class TestLanguageData: + def test_every_alias_resolves_to_a_known_code(self): + codes = known_language_codes() + assert set(language_alias_map().values()) <= codes + + def test_codes_are_ascii(self): + # "zh-Hant" once used a U+2011 non-breaking hyphen, which silently + # defeats any comparison against the normal spelling. + entries = json.loads(LANGUAGE_DATA_PATH.read_text(encoding="utf-8")) + assert [e["code"] for e in entries if not e["code"].isascii()] == [] + + def test_codes_are_unique(self): + entries = json.loads(LANGUAGE_DATA_PATH.read_text(encoding="utf-8")) + codes = [e["code"] for e in entries] + assert len(codes) == len(set(codes)) + + def test_language_name_round_trips(self): + assert language_name("sv") == "Swedish" + assert language_name("ml") == "Malayalam" + assert language_name("zzz") is None + assert language_name(None) is None + + +class TestMyAnonamouseCoverage: + """MyAnonamouse offers 62 languages and Prowlarr passes its code through + untransformed, so every one has to resolve here or the language is lost.""" + + # Observed in live MyAnonamouse data via Prowlarr. + OBSERVED = {"ENG": "en", "SWE": "sv", "MAL": "ml"} + + @pytest.mark.parametrize(("tag", "expected"), sorted(OBSERVED.items())) + def test_observed_tags_resolve(self, tag, expected): + assert normalize_language(tag) == expected + + def test_every_offered_language_resolves(self): + # Names as MyAnonamouse's own searchLanguages selector lists them. + offered = [ + "English", + "Afrikaans", + "Arabic", + "Bengali", + "Bosnian", + "Bulgarian", + "Burmese", + "Catalan", + "Chinese", + "Croatian", + "Czech", + "Danish", + "Dutch", + "Estonian", + "Farsi", + "Finnish", + "French", + "German", + "Greek", + "Gujarati", + "Hebrew", + "Hindi", + "Hungarian", + "Icelandic", + "Indonesian", + "Irish", + "Italian", + "Japanese", + "Javanese", + "Kannada", + "Korean", + "Lithuanian", + "Latin", + "Latvian", + "Malay", + "Malayalam", + "Manx", + "Marathi", + "Norwegian", + "Polish", + "Portuguese", + "Punjabi", + "Romanian", + "Russian", + "Scottish Gaelic", + "Sanskrit", + "Serbian", + "Slovenian", + "Spanish", + "Swedish", + "Tagalog", + "Tamil", + "Telugu", + "Thai", + "Turkish", + "Ukrainian", + "Urdu", + "Vietnamese", + ] + unresolved = [name for name in offered if normalize_language(name) is None] + assert unresolved == [] + + +class TestSupportedBookLanguages: + """What the settings dropdown and /api/config expose to clients.""" + + def test_exposes_only_the_fields_clients_declare(self): + # The frontend Language type is {code, language}. Aliases are an + # implementation detail and would bloat every /api/config response. + entries = supported_book_languages() + assert entries + assert all(set(e) == {"code", "language"} for e in entries) + + def test_covers_every_known_code(self): + assert {e["code"] for e in supported_book_languages()} == set(known_language_codes()) + + +class TestLegacyTraditionalChineseCode: + """Traditional Chinese was stored with a U+2011 non-breaking hyphen. + + The canonical code is now the ASCII spelling, but anything persisted + earlier still carries U+2011, so both have to resolve to the same language + or those users lose their selection (reported on PR #1142). + """ + + LEGACY = "zh\u2011Hant" + CANONICAL = "zh-Hant" + + def test_the_legacy_spelling_still_resolves(self): + assert normalize_language(self.LEGACY) == self.CANONICAL + + def test_both_spellings_are_the_same_language(self): + assert normalize_language(self.LEGACY) == normalize_language(self.CANONICAL) + + def test_the_legacy_spelling_really_does_use_a_different_character(self): + # Guards the test itself: if this ever became a plain hyphen the two + # cases above would pass for the wrong reason. + assert self.LEGACY != self.CANONICAL + assert not self.LEGACY.isascii() + + @pytest.mark.parametrize("dash", ["-", "\u2010", "\u2011", "\u2012", "\u2013", "\u2014"]) + def test_any_dash_variant_resolves(self, dash): + assert normalize_language(f"zh{dash}Hant") == self.CANONICAL + + +class TestCodesDoNotShadowEachOther: + """A code must never resolve to a different language than itself. + + 'zh' and 'zh-Hant' are distinct entries; registering the base of a + hyphenated code as an alias made 'zh' resolve correctly only because + Chinese happens to appear first in the data file. + """ + + def test_chinese_does_not_resolve_to_traditional_chinese(self): + assert normalize_language("zh") == "zh" + assert normalize_language("zh-Hant") == "zh-Hant" + + def test_every_code_resolves_to_itself(self): + for code in known_language_codes(): + assert normalize_language(code) == code, f"{code} resolved elsewhere" + + +class TestSubtagSeparators: + """The U+2011 in the old Traditional Chinese code renders close enough to + both a hyphen and an underscore that either is a plausible thing to type.""" + + @pytest.mark.parametrize("separator", ["-", "_", "‐", "‑", "–", "—"]) + def test_any_separator_spelling_resolves(self, separator): + assert normalize_language(f"zh{separator}Hant") == "zh-Hant" + + def test_separators_do_not_merge_unrelated_codes(self): + # Folding a separator must not make one language answer to another. + assert normalize_language("zh") == "zh" + assert normalize_language("en_GB") is None diff --git a/tests/download/test_orchestrator_user_output_mode.py b/tests/download/test_orchestrator_user_output_mode.py index c759b3ac..6ee1974d 100644 --- a/tests/download/test_orchestrator_user_output_mode.py +++ b/tests/download/test_orchestrator_user_output_mode.py @@ -387,3 +387,58 @@ def test_queue_release_returns_error_for_operational_queue_failure(monkeypatch): assert success is False assert error == "Error queueing release: queue offline" + + +def _queue_and_capture(monkeypatch, release_data): + import shelfmark.download.orchestrator as orchestrator + + captured: dict[str, object] = {} + + monkeypatch.setattr(orchestrator.config, "get", lambda key, default=None, user_id=None: default) + monkeypatch.setattr( + orchestrator.book_queue, "add", lambda task: captured.setdefault("task", task) or True + ) + monkeypatch.setattr(orchestrator, "ws_manager", None) + + success, error = orchestrator.queue_release(release_data, user_id=1, username="alice") + + assert success is True, error + return captured["task"] + + +def test_queue_release_carries_top_level_language(monkeypatch): + task = _queue_and_capture( + monkeypatch, + { + "source": "prowlarr", + "source_id": "release-sv", + "title": "Project Hail Mary", + "language": "sv", + }, + ) + + assert task.language == "sv" + + +def test_queue_release_falls_back_to_language_in_extra(monkeypatch): + # direct_download sets language inside extra as well as top level. + task = _queue_and_capture( + monkeypatch, + { + "source": "direct_download", + "source_id": "release-de", + "title": "Project Hail Mary", + "extra": {"language": "de"}, + }, + ) + + assert task.language == "de" + + +def test_queue_release_without_language_leaves_it_unset(monkeypatch): + task = _queue_and_capture( + monkeypatch, + {"source": "prowlarr", "source_id": "release-none", "title": "Project Hail Mary"}, + ) + + assert task.language is None diff --git a/tests/prowlarr/test_source.py b/tests/prowlarr/test_source.py index d3e6a0d8..cadef3c2 100644 --- a/tests/prowlarr/test_source.py +++ b/tests/prowlarr/test_source.py @@ -15,6 +15,7 @@ from shelfmark.release_sources.prowlarr.source import ( _collapse_duplicate_indexer_results, _detect_content_type_from_categories, _extract_format, + _extract_mam_language, _fetch_indexer_seed_settings, _last_known_seed_settings, _parse_size, @@ -729,6 +730,52 @@ class TestFetchIndexerSeedSettingsFallback: assert _fetch_indexer_seed_settings(FailingClient(), None) == {} +class TestMamLanguageCoverage: + """MyAnonamouse offers 62 languages; an unmapped code is dropped entirely, + which would leave {Language} empty and different-language editions colliding.""" + + def test_unmapped_language_is_dropped_not_passed_through(self): + # Documents why coverage matters: there is no raw fallback. + assert _extract_mam_language("Some Book [XYZ / EPUB]") is None + + def test_maps_the_common_three_letter_codes(self): + cases = { + "ENG": "en", + "SWE": "sv", + "GER": "de", + "DEU": "de", + "FRE": "fr", + "FRA": "fr", + "CZE": "cs", + "CES": "cs", + } + for tag, expected in cases.items(): + assert _extract_mam_language(f"Book [{tag} / EPUB]") == expected, tag + + def test_maps_languages_added_for_mam_parity(self): + cases = { + "LAT": "la", + "PER": "fa", + "FAS": "fa", + "TAM": "ta", + "URD": "ur", + "EST": "et", + "ICE": "is", + "ISL": "is", + "GLE": "ga", + "TGL": "fil", + "BEN": "bn", + "BOS": "bs", + "SAN": "sa", + "GLA": "gd", + "GLV": "gv", + "MAY": "ms", + "MSA": "ms", + "BUR": "my", + "MYA": "my", + } + for tag, expected in cases.items(): + assert _extract_mam_language(f"Book [{tag} / M4B]") == expected, tag class _MultiIndexerClient: """Torznab client where each indexer entry returns its own result set.