mirror of
https://github.com/ThePhaseless/Byparr.git
synced 2026-11-10 11:08:20 +00:00
refactor: take the host timezone from tzlocal
The fallback zone used after a failed egress lookup came from hand-rolled TZ and /etc/localtime parsing. That parsing passed values such as TZ=UTC0, TZ=CET-1CEST,M3.5.0,M10.5.0/3 or TZ=:/usr/share/zoneinfo/... to the browser unchanged, and Playwright rejects them as an invalid timezone ID. tzlocal.get_localzone_name() handles those forms and returns only names it could load, or None. A None result still falls back to UTC. tzlocal already came in through trafilatura (htmldate -> dateparser). It is now a direct dependency because Byparr imports it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
70d6ced2cb
commit
804c776a80
5 files changed
+22
-35
No files matched your search
@@ -30,7 +30,7 @@ Valid tags are maintained in the [IANA Language Subtag Registry](https://www.ian
|
||||
|
||||
Set `BROWSER_TIMEZONE` to an [IANA timezone name](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) like `Europe/Berlin` or `Asia/Shanghai` to fix the browser's timezone. An explicit value is used as is, also behind a proxy.
|
||||
|
||||
When either `BROWSER_TIMEZONE` or `BROWSER_LOCALE` is unset, Byparr looks up its public IP through `api.ipify.org`, `icanhazip.com` or `checkip.amazonaws.com` and maps it to a timezone and language with a GeoIP database downloaded from GitHub. Without a proxy this happens once, on the first request, and the result is reused until Byparr restarts (restart it if its public IP moves to another country). If that lookup fails, the host timezone (`TZ`, otherwise `/etc/localtime`, otherwise UTC) and `en-US` are used, and the lookup is retried after 10 minutes. With a proxy (`PROXY_SERVER` or the `X-Proxy-Server` header) it runs for every request, through the proxy.
|
||||
When either `BROWSER_TIMEZONE` or `BROWSER_LOCALE` is unset, Byparr looks up its public IP through `api.ipify.org`, `icanhazip.com` or `checkip.amazonaws.com` and maps it to a timezone and language with a GeoIP database downloaded from GitHub. Without a proxy this happens once, on the first request, and the result is reused until Byparr restarts (restart it if its public IP moves to another country). If that lookup fails, the host timezone (`TZ`, otherwise the system's, otherwise UTC) and `en-US` are used, and the lookup is retried after 10 minutes. With a proxy (`PROXY_SERVER` or the `X-Proxy-Server` header) it runs for every request, through the proxy.
|
||||
|
||||
If those hosts are slow or blocked on your network, set both `BROWSER_TIMEZONE` and `BROWSER_LOCALE`: the lookup is then skipped entirely and no request is made before the browser starts.
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ dependencies = [
|
||||
"pydantic==2.*",
|
||||
"pydantic-settings==2.*",
|
||||
"trafilatura==2.3.*",
|
||||
"tzlocal==5.*",
|
||||
]
|
||||
urls = { repository = "https://github.com/ThePhaseless/Byparr" }
|
||||
[dependency-groups]
|
||||
|
||||
+2
-10
@@ -1,7 +1,6 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import time
|
||||
from collections.abc import AsyncGenerator
|
||||
from typing import Annotated, NamedTuple, cast
|
||||
@@ -11,6 +10,7 @@ from invisible_core import prepare_session_geo
|
||||
from invisible_playwright.async_api import InvisiblePlaywright
|
||||
from playwright.async_api import Browser, BrowserContext, Page
|
||||
from pydantic import BaseModel, Field
|
||||
from tzlocal import get_localzone_name
|
||||
|
||||
from src.consts import (
|
||||
BROWSER_LOCALE,
|
||||
@@ -70,14 +70,6 @@ _geo_lock = asyncio.Lock()
|
||||
_geo: BrowserGeo | None = None
|
||||
|
||||
|
||||
def host_timezone() -> str:
|
||||
"""IANA zone of the host, the one the browser falls back to."""
|
||||
if tz := os.environ.get("TZ", "").removeprefix(":"):
|
||||
return tz
|
||||
_, found, zone = os.path.realpath("/etc/localtime").partition("/zoneinfo/")
|
||||
return zone if found else "UTC"
|
||||
|
||||
|
||||
def resolve_browser_geo() -> BrowserGeo:
|
||||
"""Resolve the timezone and locale of a session without a proxy."""
|
||||
geo = prepare_session_geo(BROWSER_TIMEZONE or "", None, BROWSER_LOCALE or "auto")
|
||||
@@ -85,7 +77,7 @@ def resolve_browser_geo() -> BrowserGeo:
|
||||
if geo.timezone:
|
||||
logger.info("Browser timezone %s, locale %s", geo.timezone, locale)
|
||||
return BrowserGeo(geo.timezone, locale)
|
||||
timezone = host_timezone()
|
||||
timezone = get_localzone_name() or "UTC"
|
||||
logger.warning(
|
||||
"Could not resolve the timezone from the egress IP, using %s and locale %s "
|
||||
"for %d seconds. Set BROWSER_TIMEZONE and BROWSER_LOCALE to skip the lookup.",
|
||||
|
||||
+16
-24
@@ -24,28 +24,6 @@ def fresh_geo_cache(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(utils, "_geo_lock", asyncio.Lock())
|
||||
|
||||
|
||||
def test_host_timezone_prefers_tz_env(monkeypatch: pytest.MonkeyPatch):
|
||||
"""TZ wins over /etc/localtime, with or without the leading colon."""
|
||||
monkeypatch.setenv("TZ", ":Asia/Shanghai")
|
||||
assert utils.host_timezone() == "Asia/Shanghai"
|
||||
|
||||
|
||||
def test_host_timezone_reads_localtime_symlink(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Without TZ the zone comes from where /etc/localtime points."""
|
||||
monkeypatch.delenv("TZ", raising=False)
|
||||
with patch(
|
||||
"src.utils.os.path.realpath", return_value="/usr/share/zoneinfo/Europe/Warsaw"
|
||||
):
|
||||
assert utils.host_timezone() == "Europe/Warsaw"
|
||||
|
||||
|
||||
def test_host_timezone_defaults_to_utc(monkeypatch: pytest.MonkeyPatch):
|
||||
"""No TZ and no zoneinfo link is what the Docker image has: UTC."""
|
||||
monkeypatch.delenv("TZ", raising=False)
|
||||
with patch("src.utils.os.path.realpath", return_value="/etc/localtime"):
|
||||
assert utils.host_timezone() == "UTC"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_egress_lookup_runs_once_per_process():
|
||||
"""Concurrent first requests and every later one share a single lookup."""
|
||||
@@ -62,8 +40,10 @@ async def test_failed_lookup_uses_host_timezone_then_retries(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""A failed lookup is not repaid per request, but is retried later."""
|
||||
monkeypatch.setenv("TZ", "Asia/Shanghai")
|
||||
with patch("src.utils.prepare_session_geo", return_value=FAILED) as lookup:
|
||||
with (
|
||||
patch("src.utils.get_localzone_name", return_value="Asia/Shanghai"),
|
||||
patch("src.utils.prepare_session_geo", return_value=FAILED) as lookup,
|
||||
):
|
||||
geo = await utils.get_browser_geo()
|
||||
assert (geo.timezone, geo.locale) == ("Asia/Shanghai", "en-US")
|
||||
await utils.get_browser_geo()
|
||||
@@ -74,6 +54,18 @@ async def test_failed_lookup_uses_host_timezone_then_retries(
|
||||
assert lookup.call_count == 2 # noqa: PLR2004
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_lookup_without_a_host_zone_uses_utc():
|
||||
"""No zone from the host either is what the Docker image has: UTC."""
|
||||
with (
|
||||
patch("src.utils.get_localzone_name", return_value=None),
|
||||
patch("src.utils.prepare_session_geo", return_value=FAILED),
|
||||
):
|
||||
geo = await utils.get_browser_geo()
|
||||
|
||||
assert geo.timezone == "UTC"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_explicit_timezone_and_locale_touch_no_network():
|
||||
"""BROWSER_TIMEZONE with BROWSER_LOCALE needs neither egress IP nor geoip."""
|
||||
|
||||
@@ -86,6 +86,7 @@ dependencies = [
|
||||
{ name = "pydantic" },
|
||||
{ name = "pydantic-settings" },
|
||||
{ name = "trafilatura" },
|
||||
{ name = "tzlocal" },
|
||||
]
|
||||
|
||||
[package.dev-dependencies]
|
||||
@@ -111,6 +112,7 @@ requires-dist = [
|
||||
{ name = "pydantic", specifier = "==2.*" },
|
||||
{ name = "pydantic-settings", specifier = "==2.*" },
|
||||
{ name = "trafilatura", specifier = "==2.3.*" },
|
||||
{ name = "tzlocal", specifier = "==5.*" },
|
||||
]
|
||||
|
||||
[package.metadata.requires-dev]
|
||||
|
||||
Reference in new issue
Block a user