From 39bdeedecd094c7af4c0ce38a55fe268d3b9c672 Mon Sep 17 00:00:00 2001 From: ThePhaseless Date: Wed, 7 Oct 2026 23:49:41 +0200 Subject: [PATCH 1/4] fix: catch invisible_playwright's own error classes invisible_playwright drives Firefox through a vendored copy of Playwright (invisible_playwright._pw), and its Error/TimeoutError are not subclasses of playwright.async_api's classes or of the builtin TimeoutError. Every except and suppress in Byparr named the playwright.async_api classes, so in production none of them matched: - a Turnstile widget read that outlasted its 1s timeout escaped the suppress() in the solve loop and turned the whole request into a 500 (the Locator.bounding_box traceback in #419), even with budget left - an unreachable host was a 500 instead of a 502 - a goto/domcontentloaded timeout was a 500 instead of a 408 - a page that never reaches networkidle was a 500 instead of returning the loaded page, and /load returned empty content for it Nothing in Byparr runs the real playwright package at runtime any more; it is only used for type hints. So the handlers now import the classes invisible_playwright re-exports from its public async_api, and nothing else. That module exports them since 0.15.0, hence the raised floor. The unit-test fakes raised the playwright.async_api classes too, which is why the suite stayed green. They now raise the vendored classes, and a new test covers a widget read timing out mid-solve. Refs #419 Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 2 +- src/challenge.py | 4 ++-- src/endpoints.py | 4 ++-- src/owui.py | 2 +- tests/main_test.py | 28 +++++++++++++++++++++++++--- tests/owui_test.py | 2 +- uv.lock | 2 +- 7 files changed, 33 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e27a1f6..867fbd7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ description = "API for getting cookies for Cloudflare challenges" readme = "README.md" dependencies = [ "fastapi[standard]==0.142.*", - "invisible-playwright>=0.6.1", + "invisible-playwright>=0.15.0", "playwright==1.63.*", "playwright-captcha==0.1.*", "pydantic==2.*", diff --git a/src/challenge.py b/src/challenge.py index 3222cc3..bb691a4 100644 --- a/src/challenge.py +++ b/src/challenge.py @@ -2,9 +2,9 @@ import time from asyncio import sleep from contextlib import suppress -from playwright.async_api import Error as PlaywrightError +from invisible_playwright.async_api import Error as PlaywrightError +from invisible_playwright.async_api import TimeoutError as PlaywrightTimeoutError from playwright.async_api import FloatRect, Page -from playwright.async_api import TimeoutError as PlaywrightTimeoutError from playwright_captcha.solvers.click.cloudflare.utils.detection import ( CF_INTERSTITIAL_INDICATORS_SELECTORS, detect_cloudflare_challenge, diff --git a/src/endpoints.py b/src/endpoints.py index bd43dd6..26d634e 100644 --- a/src/endpoints.py +++ b/src/endpoints.py @@ -5,8 +5,8 @@ from typing import Annotated from fastapi import APIRouter, Depends, HTTPException from fastapi.responses import RedirectResponse -from playwright.async_api import Error as PlaywrightError -from playwright.async_api import TimeoutError as PlaywrightTimeoutError +from invisible_playwright.async_api import Error as PlaywrightError +from invisible_playwright.async_api import TimeoutError as PlaywrightTimeoutError from src.challenge import challenge_present, solve_challenge from src.content import build_response_content diff --git a/src/owui.py b/src/owui.py index f824e13..3001189 100644 --- a/src/owui.py +++ b/src/owui.py @@ -7,8 +7,8 @@ from typing import Annotated import trafilatura from fastapi import APIRouter, Depends, Header, HTTPException +from invisible_playwright.async_api import TimeoutError as PlaywrightTimeoutError from playwright.async_api import Page -from playwright.async_api import TimeoutError as PlaywrightTimeoutError from pydantic import BaseModel from src.consts import OWUI_API_KEY diff --git a/tests/main_test.py b/tests/main_test.py index 06d0ac6..de91a41 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -6,8 +6,8 @@ from unittest.mock import AsyncMock, MagicMock import httpx2 import pytest from fastapi import HTTPException -from playwright.async_api import Error as PlaywrightError -from playwright.async_api import TimeoutError as PlaywrightTimeoutError +from invisible_playwright.async_api import Error as PlaywrightError +from invisible_playwright.async_api import TimeoutError as PlaywrightTimeoutError from starlette.testclient import TestClient from main import app @@ -148,6 +148,7 @@ def fake_dep( challenged: bool = False, marker_counts: list[int] | None = None, widget_box: dict[str, float] | None = None, + widget_error: Exception | None = None, user_agent: str | None = "UnitTestBrowser/1.0", ) -> BrowserDepClass: """Build a browser dependency pair backed by mocks.""" @@ -171,7 +172,9 @@ def fake_dep( def locator(selector: str) -> MagicMock: handle = MagicMock() handle.count = AsyncMock(side_effect=lambda: count_for(selector)) - handle.first.bounding_box = AsyncMock(return_value=widget_box) + handle.first.bounding_box = AsyncMock( + return_value=widget_box, side_effect=widget_error + ) handle.first.input_value = AsyncMock(return_value="") return handle @@ -298,6 +301,25 @@ async def test_challenge_that_clears_on_its_own_is_never_clicked(): dep.page.mouse.down.assert_not_awaited() +@pytest.mark.asyncio +async def test_slow_widget_read_does_not_abort_the_solve(): + """One widget read outlasting its short timeout is retried, never a 500 (#419).""" + dep = fake_dep( + challenged=True, + marker_counts=[1, 1, 1, 0], + widget_error=PlaywrightTimeoutError( + "Locator.bounding_box: Timeout 1000ms exceeded." + ), + ) + + response = await read_item( + LinkRequest(url="https://example.test/login", max_timeout=5), dep + ) + + assert response.status == "ok" + dep.page.mouse.down.assert_not_awaited() + + @pytest.mark.asyncio async def test_challenge_that_never_clears_returns_408(): """A challenge still up when the budget runs out is a timeout, not a 500.""" diff --git a/tests/owui_test.py b/tests/owui_test.py index ffa66a2..9331f07 100644 --- a/tests/owui_test.py +++ b/tests/owui_test.py @@ -2,7 +2,7 @@ from http import HTTPStatus from unittest.mock import AsyncMock, MagicMock, patch import pytest -from playwright.async_api import TimeoutError as PlaywrightTimeoutError +from invisible_playwright.async_api import TimeoutError as PlaywrightTimeoutError from starlette.testclient import TestClient from main import app diff --git a/uv.lock b/uv.lock index b2f8899..6aa2402 100644 --- a/uv.lock +++ b/uv.lock @@ -105,7 +105,7 @@ test = [ [package.metadata] requires-dist = [ { name = "fastapi", extras = ["standard"], specifier = "==0.142.*" }, - { name = "invisible-playwright", specifier = ">=0.6.1" }, + { name = "invisible-playwright", specifier = ">=0.15.0" }, { name = "playwright", specifier = "==1.63.*" }, { name = "playwright-captcha", specifier = "==0.1.*" }, { name = "pydantic", specifier = "==2.*" }, From 70d6ced2cb7ce846178e9f91ae2b9838e322383f Mon Sep 17 00:00:00 2001 From: ThePhaseless Date: Thu, 8 Oct 2026 00:03:59 +0200 Subject: [PATCH 2/4] fix: resolve the browser timezone once per process instead of per request Every request builds a new InvisiblePlaywright, and its __aenter__ calls prepare_session_geo(): with the default timezone="" and no proxy that discovers the egress IP through api.ipify.org, icanhazip.com and checkip.amazonaws.com, then calls ensure_geoip_mmdb(), which HEADs GitHub for a newer GeoIP release, three times per session. On a network where those hosts are slow or blocked every request paid for it before the browser even started. Measured in the app image: ipify black-holed adds ~5s per request, all three echo hosts black-holed ~12.5s, GitHub black-holed ~75s. Setting BROWSER_LOCALE alone did not help, because the timezone was still "auto". Without a proxy the egress is the host's own, so the answer does not change between requests. Resolve it on the first request, off the event loop and under a lock so concurrent first requests share one lookup, then hand the zone and the locale's primary tag to InvisiblePlaywright explicitly; an explicit zone and tag make prepare_session_geo touch no network. The resulting browser is the same as with "auto": passing the primary tag back reproduces the decided language list for every Firefox build the locale table can pick, and without a proxy the egress IP, coordinates and srflx are not used by InvisiblePlaywright anyway. When the lookup fails upstream falls back to the host timezone by passing "", but passing "" again would repeat the lookup on every request, so the host zone is named explicitly (TZ, else the /etc/localtime link, else UTC) and the lookup is retried after ten minutes. Sessions behind a proxy (PROXY_SERVER or X-Proxy-Server) still resolve per request through the proxy: each proxy has its own egress, and upstream deliberately fails the launch when it cannot be resolved. BROWSER_TIMEZONE mirrors BROWSER_LOCALE and wins also behind a proxy; with both set the lookup is skipped entirely. Fixes #416 Co-Authored-By: Claude Opus 5.5 --- README.md | 9 ++++ src/consts.py | 2 + src/utils.py | 60 ++++++++++++++++++++- tests/utils_test.py | 126 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 196 insertions(+), 1 deletion(-) create mode 100644 tests/utils_test.py diff --git a/README.md b/README.md index f374e7d..3305473 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ | `PROXY_PASSWORD` | None | Password for proxy authentication. | | `OWUI_API_KEY` | None | Bearer token for `/load` endpoint authentication. Must match `EXTERNAL_WEB_LOADER_API_KEY` in Open WebUI. | | `BROWSER_LOCALE` | None | Override the browser's language with a [BCP-47](https://www.rfc-editor.org/rfc/bcp/bcp47.txt) tag, e.g. `en-US`, `de-DE`, `fr-FR`. When unset, the locale is derived from the egress country. | +| `BROWSER_TIMEZONE` | None | Override the browser's timezone with an [IANA](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) name, e.g. `Europe/Berlin`, `Asia/Shanghai`. When unset, the timezone is derived from the egress IP. | #### Browser language @@ -25,6 +26,14 @@ Set `BROWSER_LOCALE` to a [BCP-47](https://www.rfc-editor.org/rfc/bcp/bcp47.txt) Valid tags are maintained in the [IANA Language Subtag Registry](https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry). For a friendlier list, see [List of ISO 639-1 codes](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) (language) combined with an [ISO 3166-1 alpha-2](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2) region code for the full tag, e.g. `pt-BR`. +#### Browser timezone + +Set `BROWSER_TIMEZONE` to an [IANA timezone name](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) like `Europe/Berlin` or `Asia/Shanghai` to fix the browser's timezone. An explicit value is used as is, also behind a proxy. + +When either `BROWSER_TIMEZONE` or `BROWSER_LOCALE` is unset, Byparr looks up its public IP through `api.ipify.org`, `icanhazip.com` or `checkip.amazonaws.com` and maps it to a timezone and language with a GeoIP database downloaded from GitHub. Without a proxy this happens once, on the first request, and the result is reused until Byparr restarts (restart it if its public IP moves to another country). If that lookup fails, the host timezone (`TZ`, otherwise `/etc/localtime`, otherwise UTC) and `en-US` are used, and the lookup is retried after 10 minutes. With a proxy (`PROXY_SERVER` or the `X-Proxy-Server` header) it runs for every request, through the proxy. + +If those hosts are slow or blocked on your network, set both `BROWSER_TIMEZONE` and `BROWSER_LOCALE`: the lookup is then skipped entirely and no request is made before the browser starts. + ## Proxy Recommendation Recently I've partnered with a _new in town_ proxy service - ProxyBase - to offer affordable proxy services that seems to work seamlessly with Byparr! Using my affiliate code `byparr` (case sensitive!) when signing up will not only get you access to their cost-effective (**$0.69/GB with occasional promotions** _at the time of writing_) proxy network but will also help support the continued development of this project. ProxyBase's proxies can significantly improve your success rate when bypassing anti-bot challenges. [Check out ProxyBase](https://client.proxybase.org/signup?ref=byparr) and enhance your Byparr experience! diff --git a/src/consts.py b/src/consts.py index d2f2a34..d59a7bb 100644 --- a/src/consts.py +++ b/src/consts.py @@ -20,6 +20,7 @@ class Settings(BaseSettings): return_only_cookies: bool = False owui_api_key: str | None = None browser_locale: str | None = None + browser_timezone: str | None = None settings = Settings() @@ -39,3 +40,4 @@ RETURN_ONLY_COOKIES = settings.return_only_cookies OWUI_API_KEY = settings.owui_api_key BROWSER_LOCALE = settings.browser_locale +BROWSER_TIMEZONE = settings.browser_timezone diff --git a/src/utils.py b/src/utils.py index 5d7a94e..1272075 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,15 +1,20 @@ +import asyncio import logging +import math +import os import time from collections.abc import AsyncGenerator from typing import Annotated, NamedTuple, cast from fastapi import Header +from invisible_core import prepare_session_geo from invisible_playwright.async_api import InvisiblePlaywright from playwright.async_api import Browser, BrowserContext, Page from pydantic import BaseModel, Field from src.consts import ( BROWSER_LOCALE, + BROWSER_TIMEZONE, LOG_LEVEL, PROXY_PASSWORD, PROXY_SERVER, @@ -52,6 +57,54 @@ class BrowserDepClass(NamedTuple): context: BrowserContext +GEO_RETRY_SECONDS = 600 + + +class BrowserGeo(NamedTuple): + timezone: str + locale: str + expires_at: float = math.inf + + +_geo_lock = asyncio.Lock() +_geo: BrowserGeo | None = None + + +def host_timezone() -> str: + """IANA zone of the host, the one the browser falls back to.""" + if tz := os.environ.get("TZ", "").removeprefix(":"): + return tz + _, found, zone = os.path.realpath("/etc/localtime").partition("/zoneinfo/") + return zone if found else "UTC" + + +def resolve_browser_geo() -> BrowserGeo: + """Resolve the timezone and locale of a session without a proxy.""" + geo = prepare_session_geo(BROWSER_TIMEZONE or "", None, BROWSER_LOCALE or "auto") + locale = BROWSER_LOCALE or (geo.locale.primary if geo.locale else "en-US") + if geo.timezone: + logger.info("Browser timezone %s, locale %s", geo.timezone, locale) + return BrowserGeo(geo.timezone, locale) + timezone = host_timezone() + logger.warning( + "Could not resolve the timezone from the egress IP, using %s and locale %s " + "for %d seconds. Set BROWSER_TIMEZONE and BROWSER_LOCALE to skip the lookup.", + timezone, + locale, + GEO_RETRY_SECONDS, + ) + return BrowserGeo(timezone, locale, time.monotonic() + GEO_RETRY_SECONDS) + + +async def get_browser_geo() -> BrowserGeo: + """Per-process cache of resolve_browser_geo().""" + global _geo # noqa: PLW0603 + async with _geo_lock: + if _geo is None or time.monotonic() >= _geo.expires_at: + _geo = await asyncio.to_thread(resolve_browser_geo) + return _geo + + async def get_browser( x_proxy_server: Annotated[ str | None, @@ -79,6 +132,8 @@ async def get_browser( header_password = x_proxy_password proxy_config = None + timezone = BROWSER_TIMEZONE or "" + locale = BROWSER_LOCALE or "auto" if header_server: proxy_config = { @@ -92,12 +147,15 @@ async def get_browser( "username": PROXY_USERNAME, "password": PROXY_PASSWORD, } + else: + timezone, locale, _ = await get_browser_geo() async with InvisiblePlaywright( headless=True, proxy=proxy_config, humanize=True, - locale=BROWSER_LOCALE or "auto", + locale=locale, + timezone=timezone, extra_prefs={ "devtools.jsonview.enabled": False, "browser.tabs.remote.useCrossOriginOpenerPolicy": False, diff --git a/tests/utils_test.py b/tests/utils_test.py new file mode 100644 index 0000000..4d88ae3 --- /dev/null +++ b/tests/utils_test.py @@ -0,0 +1,126 @@ +import asyncio +import math +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from invisible_core import SessionGeo, SessionLocale + +from src import utils + +BERLIN = SessionGeo( + "Europe/Berlin", + "203.0.113.7", + locale=SessionLocale(languages=("de", "en-US", "en"), region="DE"), +) +FAILED = SessionGeo( + "", None, locale=SessionLocale(languages=("en-US", "en"), region="US") +) + + +@pytest.fixture(autouse=True) +def fresh_geo_cache(monkeypatch: pytest.MonkeyPatch): + """Every test starts with nothing resolved.""" + monkeypatch.setattr(utils, "_geo", None) + monkeypatch.setattr(utils, "_geo_lock", asyncio.Lock()) + + +def test_host_timezone_prefers_tz_env(monkeypatch: pytest.MonkeyPatch): + """TZ wins over /etc/localtime, with or without the leading colon.""" + monkeypatch.setenv("TZ", ":Asia/Shanghai") + assert utils.host_timezone() == "Asia/Shanghai" + + +def test_host_timezone_reads_localtime_symlink(monkeypatch: pytest.MonkeyPatch): + """Without TZ the zone comes from where /etc/localtime points.""" + monkeypatch.delenv("TZ", raising=False) + with patch( + "src.utils.os.path.realpath", return_value="/usr/share/zoneinfo/Europe/Warsaw" + ): + assert utils.host_timezone() == "Europe/Warsaw" + + +def test_host_timezone_defaults_to_utc(monkeypatch: pytest.MonkeyPatch): + """No TZ and no zoneinfo link is what the Docker image has: UTC.""" + monkeypatch.delenv("TZ", raising=False) + with patch("src.utils.os.path.realpath", return_value="/etc/localtime"): + assert utils.host_timezone() == "UTC" + + +@pytest.mark.asyncio +async def test_egress_lookup_runs_once_per_process(): + """Concurrent first requests and every later one share a single lookup.""" + with patch("src.utils.prepare_session_geo", return_value=BERLIN) as lookup: + results = await asyncio.gather(*(utils.get_browser_geo() for _ in range(3))) + results.append(await utils.get_browser_geo()) + + lookup.assert_called_once_with("", None, "auto") + assert set(results) == {utils.BrowserGeo("Europe/Berlin", "de", math.inf)} + + +@pytest.mark.asyncio +async def test_failed_lookup_uses_host_timezone_then_retries( + monkeypatch: pytest.MonkeyPatch, +): + """A failed lookup is not repaid per request, but is retried later.""" + monkeypatch.setenv("TZ", "Asia/Shanghai") + with patch("src.utils.prepare_session_geo", return_value=FAILED) as lookup: + geo = await utils.get_browser_geo() + assert (geo.timezone, geo.locale) == ("Asia/Shanghai", "en-US") + await utils.get_browser_geo() + assert lookup.call_count == 1 + + monkeypatch.setattr(utils, "_geo", geo._replace(expires_at=0)) + await utils.get_browser_geo() + assert lookup.call_count == 2 # noqa: PLR2004 + + +@pytest.mark.asyncio +async def test_explicit_timezone_and_locale_touch_no_network(): + """BROWSER_TIMEZONE with BROWSER_LOCALE needs neither egress IP nor geoip.""" + with ( + patch("src.utils.BROWSER_TIMEZONE", "Asia/Shanghai"), + patch("src.utils.BROWSER_LOCALE", "zh-CN"), + patch("invisible_core._geo.discover_egress_ip") as discover, + patch("invisible_core.download.ensure_geoip_mmdb") as geoip, + ): + geo = await utils.get_browser_geo() + + assert (geo.timezone, geo.locale) == ("Asia/Shanghai", "zh-CN") + discover.assert_not_called() + geoip.assert_not_called() + + +def fake_invisible_playwright() -> MagicMock: + """Stand-in for InvisiblePlaywright that records how it was built.""" + browser = MagicMock() + browser.new_context = AsyncMock(return_value=AsyncMock()) + session = MagicMock() + session.__aenter__ = AsyncMock(return_value=browser) + session.__aexit__ = AsyncMock(return_value=False) + return MagicMock(return_value=session) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("proxy_header", "expected"), + [ + (None, {"timezone": "Europe/Berlin", "locale": "de"}), + ("socks5://proxy.test:1080", {"timezone": "", "locale": "auto"}), + ], +) +async def test_proxy_sessions_resolve_their_own_egress( + proxy_header: str | None, expected: dict[str, str] +): + """The cache only covers direct sessions; a proxy's egress is its own.""" + launcher = fake_invisible_playwright() + with ( + patch("src.utils.PROXY_SERVER", None), + patch("src.utils.InvisiblePlaywright", launcher), + patch("src.utils.prepare_session_geo", return_value=BERLIN) as lookup, + ): + async for _ in utils.get_browser(x_proxy_server=proxy_header): + pass + + kwargs = launcher.call_args.kwargs + assert {"timezone": kwargs["timezone"], "locale": kwargs["locale"]} == expected + assert lookup.called is (proxy_header is None) From 06b67c498e1986d7cba52df8b5ac611fea771b59 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 19:37:48 +0000 Subject: [PATCH 3/4] fix(deps): update dependency fastapi to ==0.143.* (#430) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- uv.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e27a1f6..8ea9adf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ version = "0.1.0" description = "API for getting cookies for Cloudflare challenges" readme = "README.md" dependencies = [ - "fastapi[standard]==0.142.*", + "fastapi[standard]==0.143.*", "invisible-playwright>=0.6.1", "playwright==1.63.*", "playwright-captcha==0.1.*", diff --git a/uv.lock b/uv.lock index b2f8899..543c5a5 100644 --- a/uv.lock +++ b/uv.lock @@ -104,7 +104,7 @@ test = [ [package.metadata] requires-dist = [ - { name = "fastapi", extras = ["standard"], specifier = "==0.142.*" }, + { name = "fastapi", extras = ["standard"], specifier = "==0.143.*" }, { name = "invisible-playwright", specifier = ">=0.6.1" }, { name = "playwright", specifier = "==1.63.*" }, { name = "playwright-captcha", specifier = "==0.1.*" }, @@ -310,7 +310,7 @@ wheels = [ [[package]] name = "fastapi" -version = "0.142.2" +version = "0.143.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-doc" }, @@ -320,9 +320,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/56/4f/f7c30a73127e0a8bbffe788369b8359e530b01ae06e2757936fa35bc5e6d/fastapi-0.142.2.tar.gz", hash = "sha256:06366626f2e70576367714d9ab2fe8472e6c8456dba69b399f9f797ab5e92570", size = 466721, upload-time = "2026-09-30T09:51:11.277Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/d7/6a8753ab6c1d432dc53703c3e1b92974a94531b7d047c32bbaae461ea844/fastapi-0.143.0.tar.gz", hash = "sha256:1acffe48206a80917cf7dac21992b5c44b25384e8902bf745c1fd9dabcf6c51f", size = 468391, upload-time = "2026-10-08T12:29:46.54Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/b6/78aaf9141fb46742928c113f3cf6ef2259d538cb02b604b7656c1dc9883c/fastapi-0.142.2-py3-none-any.whl", hash = "sha256:bd5f4d81f1e93a88bcd77caf4dfe3c2dbffc3805407a0007e9a114c18b3a670b", size = 144409, upload-time = "2026-09-30T09:51:09.8Z" }, + { url = "https://files.pythonhosted.org/packages/bd/f4/27e386913417ad32aae42bba48b0c0cce40e9ff2fba1a871ca2702c37324/fastapi-0.143.0-py3-none-any.whl", hash = "sha256:3e9395fd35276425b61b516a31fdd7c77fe2af83e41b4da22e30696fb1304c5d", size = 144665, upload-time = "2026-10-08T12:29:44.853Z" }, ] [package.optional-dependencies] From 804c776a806a8ba1adaabf50ac3924569c33ed3f Mon Sep 17 00:00:00 2001 From: ThePhaseless Date: Thu, 8 Oct 2026 23:52:44 +0200 Subject: [PATCH 4/4] refactor: take the host timezone from tzlocal The fallback zone used after a failed egress lookup came from hand-rolled TZ and /etc/localtime parsing. That parsing passed values such as TZ=UTC0, TZ=CET-1CEST,M3.5.0,M10.5.0/3 or TZ=:/usr/share/zoneinfo/... to the browser unchanged, and Playwright rejects them as an invalid timezone ID. tzlocal.get_localzone_name() handles those forms and returns only names it could load, or None. A None result still falls back to UTC. tzlocal already came in through trafilatura (htmldate -> dateparser). It is now a direct dependency because Byparr imports it. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- pyproject.toml | 1 + src/utils.py | 12 ++---------- tests/utils_test.py | 40 ++++++++++++++++------------------------ uv.lock | 2 ++ 5 files changed, 22 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 3305473..c8b43f1 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Valid tags are maintained in the [IANA Language Subtag Registry](https://www.ian Set `BROWSER_TIMEZONE` to an [IANA timezone name](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) like `Europe/Berlin` or `Asia/Shanghai` to fix the browser's timezone. An explicit value is used as is, also behind a proxy. -When either `BROWSER_TIMEZONE` or `BROWSER_LOCALE` is unset, Byparr looks up its public IP through `api.ipify.org`, `icanhazip.com` or `checkip.amazonaws.com` and maps it to a timezone and language with a GeoIP database downloaded from GitHub. Without a proxy this happens once, on the first request, and the result is reused until Byparr restarts (restart it if its public IP moves to another country). If that lookup fails, the host timezone (`TZ`, otherwise `/etc/localtime`, otherwise UTC) and `en-US` are used, and the lookup is retried after 10 minutes. With a proxy (`PROXY_SERVER` or the `X-Proxy-Server` header) it runs for every request, through the proxy. +When either `BROWSER_TIMEZONE` or `BROWSER_LOCALE` is unset, Byparr looks up its public IP through `api.ipify.org`, `icanhazip.com` or `checkip.amazonaws.com` and maps it to a timezone and language with a GeoIP database downloaded from GitHub. Without a proxy this happens once, on the first request, and the result is reused until Byparr restarts (restart it if its public IP moves to another country). If that lookup fails, the host timezone (`TZ`, otherwise the system's, otherwise UTC) and `en-US` are used, and the lookup is retried after 10 minutes. With a proxy (`PROXY_SERVER` or the `X-Proxy-Server` header) it runs for every request, through the proxy. If those hosts are slow or blocked on your network, set both `BROWSER_TIMEZONE` and `BROWSER_LOCALE`: the lookup is then skipped entirely and no request is made before the browser starts. diff --git a/pyproject.toml b/pyproject.toml index e27a1f6..45d148b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ dependencies = [ "pydantic==2.*", "pydantic-settings==2.*", "trafilatura==2.3.*", + "tzlocal==5.*", ] urls = { repository = "https://github.com/ThePhaseless/Byparr" } [dependency-groups] diff --git a/src/utils.py b/src/utils.py index 1272075..29be2de 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,7 +1,6 @@ import asyncio import logging import math -import os import time from collections.abc import AsyncGenerator from typing import Annotated, NamedTuple, cast @@ -11,6 +10,7 @@ from invisible_core import prepare_session_geo from invisible_playwright.async_api import InvisiblePlaywright from playwright.async_api import Browser, BrowserContext, Page from pydantic import BaseModel, Field +from tzlocal import get_localzone_name from src.consts import ( BROWSER_LOCALE, @@ -70,14 +70,6 @@ _geo_lock = asyncio.Lock() _geo: BrowserGeo | None = None -def host_timezone() -> str: - """IANA zone of the host, the one the browser falls back to.""" - if tz := os.environ.get("TZ", "").removeprefix(":"): - return tz - _, found, zone = os.path.realpath("/etc/localtime").partition("/zoneinfo/") - return zone if found else "UTC" - - def resolve_browser_geo() -> BrowserGeo: """Resolve the timezone and locale of a session without a proxy.""" geo = prepare_session_geo(BROWSER_TIMEZONE or "", None, BROWSER_LOCALE or "auto") @@ -85,7 +77,7 @@ def resolve_browser_geo() -> BrowserGeo: if geo.timezone: logger.info("Browser timezone %s, locale %s", geo.timezone, locale) return BrowserGeo(geo.timezone, locale) - timezone = host_timezone() + timezone = get_localzone_name() or "UTC" logger.warning( "Could not resolve the timezone from the egress IP, using %s and locale %s " "for %d seconds. Set BROWSER_TIMEZONE and BROWSER_LOCALE to skip the lookup.", diff --git a/tests/utils_test.py b/tests/utils_test.py index 4d88ae3..6dd594e 100644 --- a/tests/utils_test.py +++ b/tests/utils_test.py @@ -24,28 +24,6 @@ def fresh_geo_cache(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(utils, "_geo_lock", asyncio.Lock()) -def test_host_timezone_prefers_tz_env(monkeypatch: pytest.MonkeyPatch): - """TZ wins over /etc/localtime, with or without the leading colon.""" - monkeypatch.setenv("TZ", ":Asia/Shanghai") - assert utils.host_timezone() == "Asia/Shanghai" - - -def test_host_timezone_reads_localtime_symlink(monkeypatch: pytest.MonkeyPatch): - """Without TZ the zone comes from where /etc/localtime points.""" - monkeypatch.delenv("TZ", raising=False) - with patch( - "src.utils.os.path.realpath", return_value="/usr/share/zoneinfo/Europe/Warsaw" - ): - assert utils.host_timezone() == "Europe/Warsaw" - - -def test_host_timezone_defaults_to_utc(monkeypatch: pytest.MonkeyPatch): - """No TZ and no zoneinfo link is what the Docker image has: UTC.""" - monkeypatch.delenv("TZ", raising=False) - with patch("src.utils.os.path.realpath", return_value="/etc/localtime"): - assert utils.host_timezone() == "UTC" - - @pytest.mark.asyncio async def test_egress_lookup_runs_once_per_process(): """Concurrent first requests and every later one share a single lookup.""" @@ -62,8 +40,10 @@ async def test_failed_lookup_uses_host_timezone_then_retries( monkeypatch: pytest.MonkeyPatch, ): """A failed lookup is not repaid per request, but is retried later.""" - monkeypatch.setenv("TZ", "Asia/Shanghai") - with patch("src.utils.prepare_session_geo", return_value=FAILED) as lookup: + with ( + patch("src.utils.get_localzone_name", return_value="Asia/Shanghai"), + patch("src.utils.prepare_session_geo", return_value=FAILED) as lookup, + ): geo = await utils.get_browser_geo() assert (geo.timezone, geo.locale) == ("Asia/Shanghai", "en-US") await utils.get_browser_geo() @@ -74,6 +54,18 @@ async def test_failed_lookup_uses_host_timezone_then_retries( assert lookup.call_count == 2 # noqa: PLR2004 +@pytest.mark.asyncio +async def test_failed_lookup_without_a_host_zone_uses_utc(): + """No zone from the host either is what the Docker image has: UTC.""" + with ( + patch("src.utils.get_localzone_name", return_value=None), + patch("src.utils.prepare_session_geo", return_value=FAILED), + ): + geo = await utils.get_browser_geo() + + assert geo.timezone == "UTC" + + @pytest.mark.asyncio async def test_explicit_timezone_and_locale_touch_no_network(): """BROWSER_TIMEZONE with BROWSER_LOCALE needs neither egress IP nor geoip.""" diff --git a/uv.lock b/uv.lock index b2f8899..e25bab0 100644 --- a/uv.lock +++ b/uv.lock @@ -86,6 +86,7 @@ dependencies = [ { name = "pydantic" }, { name = "pydantic-settings" }, { name = "trafilatura" }, + { name = "tzlocal" }, ] [package.dev-dependencies] @@ -111,6 +112,7 @@ requires-dist = [ { name = "pydantic", specifier = "==2.*" }, { name = "pydantic-settings", specifier = "==2.*" }, { name = "trafilatura", specifier = "==2.3.*" }, + { name = "tzlocal", specifier = "==5.*" }, ] [package.metadata.requires-dev]