Merge remote-tracking branch 'origin/main' into fix/stale-snapshot-js-redirect

# Conflicts:
#	tests/main_test.py

README.md

@@ -18,6 +18,7 @@
 | `PROXY_PASSWORD`     | None      | Password for proxy authentication.                                                                                                                                |
 | `OWUI_API_KEY`       | None      | Bearer token for `/load` endpoint authentication. Must match `EXTERNAL_WEB_LOADER_API_KEY` in Open WebUI.                                                         |
 | `BROWSER_LOCALE`     | None      | Override the browser's language with a [BCP-47](https://www.rfc-editor.org/rfc/bcp/bcp47.txt) tag, e.g. `en-US`, `de-DE`, `fr-FR`. When unset, the locale is derived from the egress country. |
+| `BROWSER_TIMEZONE`   | None      | Override the browser's timezone with an [IANA](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) name, e.g. `Europe/Berlin`, `Asia/Shanghai`. When unset, the timezone is derived from the egress IP. |
 
 #### Browser language
 
@@ -25,6 +26,14 @@ Set `BROWSER_LOCALE` to a [BCP-47](https://www.rfc-editor.org/rfc/bcp/bcp47.txt)
 
 Valid tags are maintained in the [IANA Language Subtag Registry](https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry). For a friendlier list, see [List of ISO 639-1 codes](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) (language) combined with an [ISO 3166-1 alpha-2](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2) region code for the full tag, e.g. `pt-BR`.
 
+#### Browser timezone
+
+Set `BROWSER_TIMEZONE` to an [IANA timezone name](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) like `Europe/Berlin` or `Asia/Shanghai` to fix the browser's timezone. An explicit value is used as is, also behind a proxy.
+
+When either `BROWSER_TIMEZONE` or `BROWSER_LOCALE` is unset, Byparr looks up its public IP through `api.ipify.org`, `icanhazip.com` or `checkip.amazonaws.com` and maps it to a timezone and language with a GeoIP database downloaded from GitHub. Without a proxy this happens once, on the first request, and the result is reused until Byparr restarts (restart it if its public IP moves to another country). If that lookup fails, the host timezone (`TZ`, otherwise the system's, otherwise UTC) and `en-US` are used, and the lookup is retried after 10 minutes. With a proxy (`PROXY_SERVER` or the `X-Proxy-Server` header) it runs for every request, through the proxy.
+
+If those hosts are slow or blocked on your network, set both `BROWSER_TIMEZONE` and `BROWSER_LOCALE`: the lookup is then skipped entirely and no request is made before the browser starts.
+
 ## Proxy Recommendation
 
 Recently I've partnered with a _new in town_ proxy service - ProxyBase - to offer affordable proxy services that seems to work seamlessly with Byparr! Using my affiliate code `byparr` (case sensitive!) when signing up will not only get you access to their cost-effective (**$0.69/GB with occasional promotions** _at the time of writing_) proxy network but will also help support the continued development of this project. ProxyBase's proxies can significantly improve your success rate when bypassing anti-bot challenges. [Check out ProxyBase](https://client.proxybase.org/signup?ref=byparr) and enhance your Byparr experience!

pyproject.toml

@@ -8,13 +8,14 @@ version = "0.1.0"
 description = "API for getting cookies for Cloudflare challenges"
 readme = "README.md"
 dependencies = [
-    "fastapi[standard]==0.142.*",
-    "invisible-playwright>=0.6.1",
+    "fastapi[standard]==0.143.*",
+    "invisible-playwright>=0.15.0",
     "playwright==1.63.*",
     "playwright-captcha==0.1.*",
     "pydantic==2.*",
     "pydantic-settings==2.*",
     "trafilatura==2.3.*",
+    "tzlocal==5.*",
 ]
 urls = { repository = "https://github.com/ThePhaseless/Byparr" }
 [dependency-groups]

src/challenge.py

@@ -2,9 +2,9 @@ import time
 from asyncio import sleep
 from contextlib import suppress
 
-from playwright.async_api import Error as PlaywrightError
+from invisible_playwright.async_api import Error as PlaywrightError
+from invisible_playwright.async_api import TimeoutError as PlaywrightTimeoutError
 from playwright.async_api import FloatRect, Page
-from playwright.async_api import TimeoutError as PlaywrightTimeoutError
 from playwright_captcha.solvers.click.cloudflare.utils.detection import (
     CF_INTERSTITIAL_INDICATORS_SELECTORS,
     detect_cloudflare_challenge,

src/consts.py

@@ -20,6 +20,7 @@ class Settings(BaseSettings):
     return_only_cookies: bool = False
     owui_api_key: str | None = None
     browser_locale: str | None = None
+    browser_timezone: str | None = None
 
 
 settings = Settings()
@@ -39,3 +40,4 @@ RETURN_ONLY_COOKIES = settings.return_only_cookies
 
 OWUI_API_KEY = settings.owui_api_key
 BROWSER_LOCALE = settings.browser_locale
+BROWSER_TIMEZONE = settings.browser_timezone

src/endpoints.py

@@ -5,8 +5,8 @@ from typing import Annotated
 
 from fastapi import APIRouter, Depends, HTTPException
 from fastapi.responses import RedirectResponse
-from playwright.async_api import Error as PlaywrightError
-from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+from invisible_playwright.async_api import Error as PlaywrightError
+from invisible_playwright.async_api import TimeoutError as PlaywrightTimeoutError
 
 from src.challenge import challenge_present, solve_challenge
 from src.content import build_response_content

src/owui.py

@@ -7,8 +7,8 @@ from typing import Annotated
 
 import trafilatura
 from fastapi import APIRouter, Depends, Header, HTTPException
+from invisible_playwright.async_api import TimeoutError as PlaywrightTimeoutError
 from playwright.async_api import Page
-from playwright.async_api import TimeoutError as PlaywrightTimeoutError
 from pydantic import BaseModel
 
 from src.consts import OWUI_API_KEY

src/utils.py

@@ -1,15 +1,20 @@
+import asyncio
 import logging
+import math
 import time
 from collections.abc import AsyncGenerator
 from typing import Annotated, NamedTuple, cast
 
 from fastapi import Header
+from invisible_core import prepare_session_geo
 from invisible_playwright.async_api import InvisiblePlaywright
 from playwright.async_api import Browser, BrowserContext, Page
 from pydantic import BaseModel, Field
+from tzlocal import get_localzone_name
 
 from src.consts import (
     BROWSER_LOCALE,
+    BROWSER_TIMEZONE,
     LOG_LEVEL,
     PROXY_PASSWORD,
     PROXY_SERVER,
@@ -52,6 +57,46 @@ class BrowserDepClass(NamedTuple):
     context: BrowserContext
 
 
+GEO_RETRY_SECONDS = 600
+
+
+class BrowserGeo(NamedTuple):
+    timezone: str
+    locale: str
+    expires_at: float = math.inf
+
+
+_geo_lock = asyncio.Lock()
+_geo: BrowserGeo | None = None
+
+
+def resolve_browser_geo() -> BrowserGeo:
+    """Resolve the timezone and locale of a session without a proxy."""
+    geo = prepare_session_geo(BROWSER_TIMEZONE or "", None, BROWSER_LOCALE or "auto")
+    locale = BROWSER_LOCALE or (geo.locale.primary if geo.locale else "en-US")
+    if geo.timezone:
+        logger.info("Browser timezone %s, locale %s", geo.timezone, locale)
+        return BrowserGeo(geo.timezone, locale)
+    timezone = get_localzone_name() or "UTC"
+    logger.warning(
+        "Could not resolve the timezone from the egress IP, using %s and locale %s "
+        "for %d seconds. Set BROWSER_TIMEZONE and BROWSER_LOCALE to skip the lookup.",
+        timezone,
+        locale,
+        GEO_RETRY_SECONDS,
+    )
+    return BrowserGeo(timezone, locale, time.monotonic() + GEO_RETRY_SECONDS)
+
+
+async def get_browser_geo() -> BrowserGeo:
+    """Per-process cache of resolve_browser_geo()."""
+    global _geo  # noqa: PLW0603
+    async with _geo_lock:
+        if _geo is None or time.monotonic() >= _geo.expires_at:
+            _geo = await asyncio.to_thread(resolve_browser_geo)
+        return _geo
+
+
 async def get_browser(
     x_proxy_server: Annotated[
         str | None,
@@ -79,6 +124,8 @@ async def get_browser(
     header_password = x_proxy_password
 
     proxy_config = None
+    timezone = BROWSER_TIMEZONE or ""
+    locale = BROWSER_LOCALE or "auto"
 
     if header_server:
         proxy_config = {
@@ -92,12 +139,15 @@ async def get_browser(
             "username": PROXY_USERNAME,
             "password": PROXY_PASSWORD,
         }
+    else:
+        timezone, locale, _ = await get_browser_geo()
 
     async with InvisiblePlaywright(
         headless=True,
         proxy=proxy_config,
         humanize=True,
-        locale=BROWSER_LOCALE or "auto",
+        locale=locale,
+        timezone=timezone,
         extra_prefs={
             "devtools.jsonview.enabled": False,
             "browser.tabs.remote.useCrossOriginOpenerPolicy": False,

tests/main_test.py

@@ -6,9 +6,8 @@ from unittest.mock import AsyncMock, MagicMock
 import httpx2
 import pytest
 from fastapi import HTTPException
-from invisible_playwright.async_api import Error as InvisiblePlaywrightError
-from playwright.async_api import Error as PlaywrightError
-from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+from invisible_playwright.async_api import Error as PlaywrightError
+from invisible_playwright.async_api import TimeoutError as PlaywrightTimeoutError
 from starlette.testclient import TestClient
 
 from main import app
@@ -149,6 +148,7 @@ def fake_dep(
     challenged: bool = False,
     marker_counts: list[int] | None = None,
     widget_box: dict[str, float] | None = None,
+    widget_error: Exception | None = None,
     user_agent: str | None = "UnitTestBrowser/1.0",
 ) -> BrowserDepClass:
     """Build a browser dependency pair backed by mocks."""
@@ -172,7 +172,9 @@ def fake_dep(
     def locator(selector: str) -> MagicMock:
         handle = MagicMock()
         handle.count = AsyncMock(side_effect=lambda: count_for(selector))
-        handle.first.bounding_box = AsyncMock(return_value=widget_box)
+        handle.first.bounding_box = AsyncMock(
+            return_value=widget_box, side_effect=widget_error
+        )
         handle.first.input_value = AsyncMock(return_value="")
         return handle
 
@@ -228,7 +230,7 @@ async def test_html_read_racing_a_navigation_is_retried():
     """A read that loses its document to a navigation reads the new one instead."""
     dep = fake_dep()
     dep.page.content.side_effect = [
-        InvisiblePlaywrightError("Failed to find execution context with id = id-5"),
+        PlaywrightError("Failed to find execution context with id = id-5"),
         "<html><title>Article</title></html>",
     ]
 
@@ -332,6 +334,25 @@ async def test_challenge_that_clears_on_its_own_is_never_clicked():
     dep.page.mouse.down.assert_not_awaited()
 
 
+@pytest.mark.asyncio
+async def test_slow_widget_read_does_not_abort_the_solve():
+    """One widget read outlasting its short timeout is retried, never a 500 (#419)."""
+    dep = fake_dep(
+        challenged=True,
+        marker_counts=[1, 1, 1, 0],
+        widget_error=PlaywrightTimeoutError(
+            "Locator.bounding_box: Timeout 1000ms exceeded."
+        ),
+    )
+
+    response = await read_item(
+        LinkRequest(url="https://example.test/login", max_timeout=5), dep
+    )
+
+    assert response.status == "ok"
+    dep.page.mouse.down.assert_not_awaited()
+
+
 @pytest.mark.asyncio
 async def test_challenge_that_never_clears_returns_408():
     """A challenge still up when the budget runs out is a timeout, not a 500."""

tests/owui_test.py

@@ -2,7 +2,7 @@ from http import HTTPStatus
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
-from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+from invisible_playwright.async_api import TimeoutError as PlaywrightTimeoutError
 from starlette.testclient import TestClient
 
 from main import app

tests/utils_test.py

@@ -0,0 +1,118 @@
+import asyncio
+import math
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from invisible_core import SessionGeo, SessionLocale
+
+from src import utils
+
+BERLIN = SessionGeo(
+    "Europe/Berlin",
+    "203.0.113.7",
+    locale=SessionLocale(languages=("de", "en-US", "en"), region="DE"),
+)
+FAILED = SessionGeo(
+    "", None, locale=SessionLocale(languages=("en-US", "en"), region="US")
+)
+
+
+@pytest.fixture(autouse=True)
+def fresh_geo_cache(monkeypatch: pytest.MonkeyPatch):
+    """Every test starts with nothing resolved."""
+    monkeypatch.setattr(utils, "_geo", None)
+    monkeypatch.setattr(utils, "_geo_lock", asyncio.Lock())
+
+
+@pytest.mark.asyncio
+async def test_egress_lookup_runs_once_per_process():
+    """Concurrent first requests and every later one share a single lookup."""
+    with patch("src.utils.prepare_session_geo", return_value=BERLIN) as lookup:
+        results = await asyncio.gather(*(utils.get_browser_geo() for _ in range(3)))
+        results.append(await utils.get_browser_geo())
+
+    lookup.assert_called_once_with("", None, "auto")
+    assert set(results) == {utils.BrowserGeo("Europe/Berlin", "de", math.inf)}
+
+
+@pytest.mark.asyncio
+async def test_failed_lookup_uses_host_timezone_then_retries(
+    monkeypatch: pytest.MonkeyPatch,
+):
+    """A failed lookup is not repaid per request, but is retried later."""
+    with (
+        patch("src.utils.get_localzone_name", return_value="Asia/Shanghai"),
+        patch("src.utils.prepare_session_geo", return_value=FAILED) as lookup,
+    ):
+        geo = await utils.get_browser_geo()
+        assert (geo.timezone, geo.locale) == ("Asia/Shanghai", "en-US")
+        await utils.get_browser_geo()
+        assert lookup.call_count == 1
+
+        monkeypatch.setattr(utils, "_geo", geo._replace(expires_at=0))
+        await utils.get_browser_geo()
+        assert lookup.call_count == 2  # noqa: PLR2004
+
+
+@pytest.mark.asyncio
+async def test_failed_lookup_without_a_host_zone_uses_utc():
+    """No zone from the host either is what the Docker image has: UTC."""
+    with (
+        patch("src.utils.get_localzone_name", return_value=None),
+        patch("src.utils.prepare_session_geo", return_value=FAILED),
+    ):
+        geo = await utils.get_browser_geo()
+
+    assert geo.timezone == "UTC"
+
+
+@pytest.mark.asyncio
+async def test_explicit_timezone_and_locale_touch_no_network():
+    """BROWSER_TIMEZONE with BROWSER_LOCALE needs neither egress IP nor geoip."""
+    with (
+        patch("src.utils.BROWSER_TIMEZONE", "Asia/Shanghai"),
+        patch("src.utils.BROWSER_LOCALE", "zh-CN"),
+        patch("invisible_core._geo.discover_egress_ip") as discover,
+        patch("invisible_core.download.ensure_geoip_mmdb") as geoip,
+    ):
+        geo = await utils.get_browser_geo()
+
+    assert (geo.timezone, geo.locale) == ("Asia/Shanghai", "zh-CN")
+    discover.assert_not_called()
+    geoip.assert_not_called()
+
+
+def fake_invisible_playwright() -> MagicMock:
+    """Stand-in for InvisiblePlaywright that records how it was built."""
+    browser = MagicMock()
+    browser.new_context = AsyncMock(return_value=AsyncMock())
+    session = MagicMock()
+    session.__aenter__ = AsyncMock(return_value=browser)
+    session.__aexit__ = AsyncMock(return_value=False)
+    return MagicMock(return_value=session)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("proxy_header", "expected"),
+    [
+        (None, {"timezone": "Europe/Berlin", "locale": "de"}),
+        ("socks5://proxy.test:1080", {"timezone": "", "locale": "auto"}),
+    ],
+)
+async def test_proxy_sessions_resolve_their_own_egress(
+    proxy_header: str | None, expected: dict[str, str]
+):
+    """The cache only covers direct sessions; a proxy's egress is its own."""
+    launcher = fake_invisible_playwright()
+    with (
+        patch("src.utils.PROXY_SERVER", None),
+        patch("src.utils.InvisiblePlaywright", launcher),
+        patch("src.utils.prepare_session_geo", return_value=BERLIN) as lookup,
+    ):
+        async for _ in utils.get_browser(x_proxy_server=proxy_header):
+            pass
+
+    kwargs = launcher.call_args.kwargs
+    assert {"timezone": kwargs["timezone"], "locale": kwargs["locale"]} == expected
+    assert lookup.called is (proxy_header is None)

uv.lock

@@ -86,6 +86,7 @@ dependencies = [
     { name = "pydantic" },
     { name = "pydantic-settings" },
     { name = "trafilatura" },
+    { name = "tzlocal" },
 ]
 
 [package.dev-dependencies]
@@ -104,13 +105,14 @@ test = [
 
 [package.metadata]
 requires-dist = [
-    { name = "fastapi", extras = ["standard"], specifier = "==0.142.*" },
-    { name = "invisible-playwright", specifier = ">=0.6.1" },
+    { name = "fastapi", extras = ["standard"], specifier = "==0.143.*" },
+    { name = "invisible-playwright", specifier = ">=0.15.0" },
     { name = "playwright", specifier = "==1.63.*" },
     { name = "playwright-captcha", specifier = "==0.1.*" },
     { name = "pydantic", specifier = "==2.*" },
     { name = "pydantic-settings", specifier = "==2.*" },
     { name = "trafilatura", specifier = "==2.3.*" },
+    { name = "tzlocal", specifier = "==5.*" },
 ]
 
 [package.metadata.requires-dev]
@@ -310,7 +312,7 @@ wheels = [
 
 [[package]]
 name = "fastapi"
-version = "0.142.2"
+version = "0.143.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "annotated-doc" },
@@ -320,9 +322,9 @@ dependencies = [
     { name = "typing-extensions" },
     { name = "typing-inspection" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/56/4f/f7c30a73127e0a8bbffe788369b8359e530b01ae06e2757936fa35bc5e6d/fastapi-0.142.2.tar.gz", hash = "sha256:06366626f2e70576367714d9ab2fe8472e6c8456dba69b399f9f797ab5e92570", size = 466721, upload-time = "2026-09-30T09:51:11.277Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/0b/d7/6a8753ab6c1d432dc53703c3e1b92974a94531b7d047c32bbaae461ea844/fastapi-0.143.0.tar.gz", hash = "sha256:1acffe48206a80917cf7dac21992b5c44b25384e8902bf745c1fd9dabcf6c51f", size = 468391, upload-time = "2026-10-08T12:29:46.54Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/a0/b6/78aaf9141fb46742928c113f3cf6ef2259d538cb02b604b7656c1dc9883c/fastapi-0.142.2-py3-none-any.whl", hash = "sha256:bd5f4d81f1e93a88bcd77caf4dfe3c2dbffc3805407a0007e9a114c18b3a670b", size = 144409, upload-time = "2026-09-30T09:51:09.8Z" },
+    { url = "https://files.pythonhosted.org/packages/bd/f4/27e386913417ad32aae42bba48b0c0cce40e9ff2fba1a871ca2702c37324/fastapi-0.143.0-py3-none-any.whl", hash = "sha256:3e9395fd35276425b61b516a31fdd7c77fe2af83e41b4da22e30696fb1304c5d", size = 144665, upload-time = "2026-10-08T12:29:44.853Z" },
 ]
 
 [package.optional-dependencies]