mirror of
https://github.com/ThePhaseless/Byparr.git
synced 2026-09-24 14:20:08 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f821290bac | ||
|
|
5de9266d7e | ||
|
|
89dcf5e16c | ||
|
|
3c45ef9691 | ||
|
|
bd4c62de38 | ||
|
|
92043725f5 | ||
|
|
f8d087bab8 | ||
|
|
c38a6f4e85 | ||
|
|
aa7bfee7bb | ||
|
|
336773d7da | ||
|
|
8cb5770b84 | ||
|
|
ae28c7098f | ||
|
|
1c4b377613 | ||
|
|
8ef4c62249 | ||
|
|
baad431605 | ||
|
|
7e1a5d4329 | ||
|
|
221f27acca | ||
|
|
bdd59d7e60 | ||
|
|
1801eaa40c | ||
|
|
bc529e5915 | ||
|
|
1c9093f218 | ||
|
|
0c44ce1a4d | ||
|
|
07309c8d8e | ||
|
|
25194c3bca |
@@ -9,7 +9,7 @@ on:
|
||||
schedule:
|
||||
- cron: "25 0 * * *"
|
||||
push:
|
||||
branches: ["*"]
|
||||
branches: ["main"]
|
||||
# Publish semver tags as releases.
|
||||
tags: ["v*.*.*"]
|
||||
paths:
|
||||
@@ -65,10 +65,12 @@ jobs:
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64
|
||||
cache-from: type=gha,scope=x64
|
||||
cache-from: type=gha,scope=amd64
|
||||
pull: true
|
||||
cache-to: type=gha,mode=max,scope=x64
|
||||
cache-to: type=gha,mode=max,scope=amd64
|
||||
target: test
|
||||
build-args: |
|
||||
GITHUB_BUILD=true
|
||||
|
||||
build:
|
||||
needs: test
|
||||
@@ -135,8 +137,8 @@ jobs:
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
platforms: ${{ matrix.platform }}
|
||||
cache-from: type=gha,scope=${{ matrix.platform }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.platform }}
|
||||
cache-from: type=gha,scope=${{ steps.vars.outputs.SURFIX }}
|
||||
cache-to: type=gha,mode=max,scope=${{ steps.vars.outputs.SURFIX }}
|
||||
build-args: |
|
||||
GITHUB_BUILD=true
|
||||
VERSION=${{ github.ref_type == 'tag' && github.ref_name || github.sha }}
|
||||
@@ -184,6 +186,7 @@ jobs:
|
||||
|
||||
# Create manifest lists and push
|
||||
- name: Create and push manifest lists
|
||||
id: manifests
|
||||
run: |
|
||||
TAGS="${{ steps.meta.outputs.tags }}"
|
||||
args=""
|
||||
@@ -213,11 +216,19 @@ jobs:
|
||||
${image}:${{github.sha}}-arm64
|
||||
fi
|
||||
|
||||
# Sign the manifest
|
||||
- name: Sign the manifests
|
||||
# All tags created above alias a single manifest list; capture its digest
|
||||
# so the signature is bound to the image bytes, not a mutable tag.
|
||||
# Tags from metadata-action are full references (image:tag), one per line,
|
||||
# so take the first line rather than splitting on spaces.
|
||||
FIRST_TAG=$(printf '%s' "$TAGS" | head -n1)
|
||||
DIGEST=$(docker buildx imagetools inspect --format '{{.Manifest.Digest}}' "$FIRST_TAG")
|
||||
echo "DIGEST=$DIGEST" >> $GITHUB_OUTPUT
|
||||
|
||||
# Sign the manifest list by digest — every consumer tag aliases this digest
|
||||
- name: Sign the manifest list by digest
|
||||
env:
|
||||
TAGS: ${{ steps.meta.outputs.tags }}
|
||||
IMAGE: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
DIGEST: ${{ steps.manifests.outputs.DIGEST }}
|
||||
run: |
|
||||
for TAG in $TAGS; do
|
||||
cosign sign --yes $TAG
|
||||
done
|
||||
image=${IMAGE,,}
|
||||
cosign sign --yes ${image}@${DIGEST}
|
||||
|
||||
+7
-8
@@ -3,19 +3,16 @@
|
||||
# cannot install firefox deps for (no libgtk-3 -> camoufox fails to launch).
|
||||
FROM ubuntu:24.04 AS base
|
||||
|
||||
ARG GITHUB_BUILD=false \
|
||||
VERSION
|
||||
ARG GITHUB_BUILD=false
|
||||
|
||||
ENV GITHUB_BUILD=${GITHUB_BUILD}\
|
||||
VERSION=${VERSION}\
|
||||
DEBIAN_FRONTEND=noninteractive \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
# prevents python creating .pyc files
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
UV_LINK_MODE=copy \
|
||||
PORT=8191 \
|
||||
XDG_CACHE_HOME=/cache \
|
||||
HOME=/tmp
|
||||
HOME=/home/byparr
|
||||
|
||||
RUN apt-get update &&\
|
||||
apt-get install -y --no-install-recommends curl ca-certificates git tini &&\
|
||||
@@ -47,9 +44,9 @@ RUN mkdir -p /cache &&\
|
||||
|
||||
COPY . .
|
||||
|
||||
# Make app and cache world-readable; cache must be writable for runtime browser/profile data
|
||||
RUN chmod -R o+rX /app /cache &&\
|
||||
chmod -R o+w /cache
|
||||
RUN mkdir -p /home/byparr &&\
|
||||
chmod -R o+rX /app &&\
|
||||
chmod -R a+rwX /cache /home/byparr
|
||||
|
||||
FROM app AS test
|
||||
RUN \
|
||||
@@ -57,6 +54,8 @@ RUN \
|
||||
uv run pytest --retries 3
|
||||
|
||||
FROM app
|
||||
ARG VERSION
|
||||
ENV VERSION=${VERSION}
|
||||
USER 1000
|
||||
EXPOSE $PORT
|
||||
HEALTHCHECK --interval=15m --timeout=30s --start-period=5s --retries=3 CMD curl "http://127.0.0.1:${PORT}/health"
|
||||
|
||||
@@ -17,6 +17,13 @@
|
||||
| `PROXY_USERNAME` | None | Username for proxy authentication. |
|
||||
| `PROXY_PASSWORD` | None | Password for proxy authentication. |
|
||||
| `OWUI_API_KEY` | None | Bearer token for `/load` endpoint authentication. Must match `EXTERNAL_WEB_LOADER_API_KEY` in Open WebUI. |
|
||||
| `BROWSER_LOCALE` | None | Override the browser's language with a [BCP-47](https://www.rfc-editor.org/rfc/bcp/bcp47.txt) tag, e.g. `en-US`, `de-DE`, `fr-FR`. When unset, the locale is derived from the egress country. |
|
||||
|
||||
#### Browser language
|
||||
|
||||
Set `BROWSER_LOCALE` to a [BCP-47](https://www.rfc-editor.org/rfc/bcp/bcp47.txt) language tag like `en-US`, `de-DE`, `fr-FR`, `pl-PL`, or `zh-CN` to fix the browser's language and `Accept-Language` header. When unset, Byparr derives the locale from the egress country (e.g. a French proxy → `fr-FR`), keeping the browser language consistent with the exit IP.
|
||||
|
||||
Valid tags are maintained in the [IANA Language Subtag Registry](https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry). For a friendlier list, see [List of ISO 639-1 codes](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) (language) combined with an [ISO 3166-1 alpha-2](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2) region code for the full tag, e.g. `pt-BR`.
|
||||
|
||||
## Proxy Recommendation
|
||||
|
||||
|
||||
+3
-2
@@ -20,7 +20,7 @@ urls = { repository = "https://github.com/ThePhaseless/Byparr" }
|
||||
[dependency-groups]
|
||||
|
||||
test = [
|
||||
"httpx2==2.9.*",
|
||||
"httpx2==2.10.*",
|
||||
"pytest==9.1.*",
|
||||
"pytest-asyncio==1.4.*",
|
||||
"pytest-retry==1.7.*",
|
||||
@@ -54,7 +54,8 @@ ignore = [
|
||||
"G004",
|
||||
"ANN001",
|
||||
"ANN204",
|
||||
"ANN206",
|
||||
"CPY001",
|
||||
"BLE001",
|
||||
]
|
||||
select = ["ALL"]
|
||||
extend-safe-fixes = ["D415"]
|
||||
|
||||
+2
-11
@@ -3,8 +3,6 @@ import sys
|
||||
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
from playwright_captcha import CaptchaType
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(env_file=".env", extra="ignore")
|
||||
@@ -24,6 +22,7 @@ class Settings(BaseSettings):
|
||||
block_media: bool = False
|
||||
return_only_cookies: bool = False
|
||||
owui_api_key: str | None = None
|
||||
browser_locale: str | None = None
|
||||
|
||||
|
||||
settings = Settings()
|
||||
@@ -44,12 +43,4 @@ BLOCK_MEDIA = settings.block_media
|
||||
RETURN_ONLY_COOKIES = settings.return_only_cookies
|
||||
|
||||
OWUI_API_KEY = settings.owui_api_key
|
||||
|
||||
CHALLENGE_TITLES_MAP: dict[CaptchaType, list[str]] = {
|
||||
# Cloudflare
|
||||
CaptchaType.CLOUDFLARE_INTERSTITIAL: ["Just a moment..."],
|
||||
}
|
||||
|
||||
CHALLENGE_TITLES = [
|
||||
title for titles in CHALLENGE_TITLES_MAP.values() for title in titles
|
||||
]
|
||||
BROWSER_LOCALE = settings.browser_locale
|
||||
|
||||
+149
-72
@@ -9,8 +9,10 @@ from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi.responses import RedirectResponse
|
||||
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
|
||||
from playwright_captcha import CaptchaType
|
||||
from playwright_captcha.solvers.click.cloudflare.utils.detection import (
|
||||
detect_cloudflare_challenge,
|
||||
)
|
||||
|
||||
from src.consts import CHALLENGE_TITLES
|
||||
from src.models import (
|
||||
HealthcheckResponse,
|
||||
LinkRequest,
|
||||
@@ -26,8 +28,16 @@ router = APIRouter()
|
||||
|
||||
BrowserDep = Annotated[BrowserDepClass, Depends(get_browser)]
|
||||
|
||||
CSP_HEADERS = frozenset(
|
||||
{"content-security-policy", "content-security-policy-report-only"}
|
||||
# Headers to strip from the fulfilled response: CSP is removed for navigation
|
||||
# freedom, and content-encoding/content-length are stale once we request an
|
||||
# uncompressed body via accept-encoding: identity below.
|
||||
DROP_HEADERS = frozenset(
|
||||
{
|
||||
"content-security-policy",
|
||||
"content-security-policy-report-only",
|
||||
"content-encoding",
|
||||
"content-length",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -59,12 +69,53 @@ async def health_check(sb: BrowserDep):
|
||||
async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse:
|
||||
"""Handle POST requests."""
|
||||
start_time = int(time.time() * 1000)
|
||||
|
||||
timer = TimeoutTimer(duration=request.max_timeout)
|
||||
|
||||
request.url = request.url.replace('"', "").strip()
|
||||
|
||||
final_url = await setup_routes(request, dep)
|
||||
|
||||
try:
|
||||
challenge_detected, page_html, page_request, status = (
|
||||
await _navigate_and_solve(dep, request, timer)
|
||||
)
|
||||
except (TimeoutError, PlaywrightTimeoutError) as e:
|
||||
logger.error("Timed out while loading the page or solving the challenge")
|
||||
raise HTTPException(
|
||||
status_code=408,
|
||||
detail="Timed out while loading the page or solving the challenge",
|
||||
) from e
|
||||
|
||||
cookies = await dep.context.cookies()
|
||||
content_type, response_content = await build_response_content(
|
||||
dep, request, page_request,
|
||||
challenge_detected=challenge_detected,
|
||||
page_html=page_html,
|
||||
)
|
||||
|
||||
return LinkResponse(
|
||||
message="Success",
|
||||
solution=Solution(
|
||||
user_agent=await dep.page.evaluate("navigator.userAgent"),
|
||||
url=final_url if final_url is not None else dep.page.url,
|
||||
status=status,
|
||||
cookies=cookies,
|
||||
headers=page_request.headers if page_request else {},
|
||||
response=response_content,
|
||||
content_type=content_type,
|
||||
),
|
||||
start_timestamp=start_time,
|
||||
)
|
||||
|
||||
|
||||
async def setup_routes(request: LinkRequest, dep: BrowserDep) -> str | None:
|
||||
"""
|
||||
Install request routes for media blocking and CSP stripping.
|
||||
|
||||
Returns the final URL captured during navigation; callers read it after
|
||||
the page settles.
|
||||
"""
|
||||
if request.block_media:
|
||||
|
||||
async def block_media_route(route) -> None:
|
||||
if route.request.resource_type in ("image", "media", "font"):
|
||||
await route.abort()
|
||||
@@ -80,7 +131,13 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse:
|
||||
if route.request.resource_type != "document":
|
||||
await route.continue_()
|
||||
return
|
||||
response = await route.fetch()
|
||||
# Request an uncompressed body via accept-encoding: identity. When
|
||||
# route.fulfill re-serves the fetched response (by uid), it forwards
|
||||
# the original compressed bytes; stripping content-encoding below
|
||||
# would leave the browser reading compressed bytes as plain text.
|
||||
response = await route.fetch(
|
||||
headers={**route.request.headers, "accept-encoding": "identity"}
|
||||
)
|
||||
if route.request.frame == dep.page.main_frame:
|
||||
final_url = response.url
|
||||
await route.fulfill(
|
||||
@@ -88,84 +145,104 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse:
|
||||
headers={
|
||||
key: value
|
||||
for key, value in response.headers.items()
|
||||
if key.lower() not in CSP_HEADERS
|
||||
if key.lower() not in DROP_HEADERS
|
||||
},
|
||||
)
|
||||
|
||||
await dep.page.route("**/*", strip_csp_route)
|
||||
return final_url
|
||||
|
||||
|
||||
async def _navigate_and_solve(
|
||||
dep: BrowserDep,
|
||||
request: LinkRequest,
|
||||
timer: TimeoutTimer,
|
||||
) -> tuple[bool, str | None, object, HTTPStatus]:
|
||||
"""Navigate to the URL, then solve a challenge or wait for network idle."""
|
||||
page_html: str | None = None
|
||||
page_request = await dep.page.goto(
|
||||
request.url, timeout=timer.remaining() * 1000
|
||||
)
|
||||
status = page_request.status if page_request else HTTPStatus.OK
|
||||
await dep.page.wait_for_load_state(
|
||||
state="domcontentloaded", timeout=timer.remaining() * 1000
|
||||
)
|
||||
|
||||
challenge_active = (
|
||||
await detect_cloudflare_challenge(dep.page, "interstitial")
|
||||
or await detect_cloudflare_challenge(dep.page, "turnstile")
|
||||
)
|
||||
if not challenge_active:
|
||||
page_html = await dep.page.content()
|
||||
await _wait_for_networkidle(dep, timer)
|
||||
return False, page_html, page_request, status
|
||||
|
||||
await _solve_challenge(dep, timer)
|
||||
status = HTTPStatus.OK
|
||||
return True, page_html, page_request, status
|
||||
|
||||
|
||||
async def _solve_challenge(dep: BrowserDep, timer: TimeoutTimer) -> None:
|
||||
"""Attempt to solve a detected Cloudflare interstitial challenge."""
|
||||
logger.info("Challenge detected, attempting to solve...")
|
||||
await wait_for(
|
||||
dep.solver.solve_captcha( # pyright: ignore[reportUnknownMemberType,reportUnknownArgumentType]
|
||||
captcha_container=dep.page,
|
||||
captcha_type=CaptchaType.CLOUDFLARE_INTERSTITIAL,
|
||||
wait_checkbox_attempts=1,
|
||||
wait_checkbox_delay=0.5,
|
||||
),
|
||||
timeout=timer.remaining(),
|
||||
)
|
||||
logger.debug("Challenge solved successfully.")
|
||||
|
||||
|
||||
async def _wait_for_networkidle(dep: BrowserDep, timer: TimeoutTimer) -> None:
|
||||
"""Wait for network idle, tolerating post-DOM-load stalls."""
|
||||
try:
|
||||
page_request = await dep.page.goto(
|
||||
request.url, timeout=timer.remaining() * 1000
|
||||
)
|
||||
status = page_request.status if page_request else HTTPStatus.OK
|
||||
await dep.page.wait_for_load_state(
|
||||
state="domcontentloaded", timeout=timer.remaining() * 1000
|
||||
"networkidle", timeout=timer.remaining() * 1000
|
||||
)
|
||||
except PlaywrightTimeoutError:
|
||||
logger.info(
|
||||
"networkidle timed out after domcontentloaded; "
|
||||
"continuing with loaded page"
|
||||
)
|
||||
|
||||
if await dep.page.title() in CHALLENGE_TITLES:
|
||||
logger.info("Challenge detected, attempting to solve...")
|
||||
# Solve the captcha
|
||||
await wait_for(
|
||||
dep.solver.solve_captcha( # pyright: ignore[reportUnknownMemberType,reportUnknownArgumentType]
|
||||
captcha_container=dep.page,
|
||||
captcha_type=CaptchaType.CLOUDFLARE_INTERSTITIAL,
|
||||
wait_checkbox_attempts=1,
|
||||
wait_checkbox_delay=0.5,
|
||||
),
|
||||
timeout=timer.remaining(),
|
||||
)
|
||||
status = HTTPStatus.OK
|
||||
logger.debug("Challenge solved successfully.")
|
||||
else:
|
||||
try:
|
||||
await dep.page.wait_for_load_state(
|
||||
"networkidle", timeout=timer.remaining() * 1000
|
||||
)
|
||||
except PlaywrightTimeoutError:
|
||||
logger.info(
|
||||
"networkidle timed out after domcontentloaded; continuing with loaded page"
|
||||
)
|
||||
except (TimeoutError, PlaywrightTimeoutError) as e:
|
||||
logger.error("Timed out while loading the page or solving the challenge")
|
||||
raise HTTPException(
|
||||
status_code=408,
|
||||
detail="Timed out while loading the page or solving the challenge",
|
||||
) from e
|
||||
|
||||
cookies = await dep.context.cookies()
|
||||
|
||||
content_type = "text/html"
|
||||
response_content = ""
|
||||
|
||||
async def build_response_content(
|
||||
dep: BrowserDep,
|
||||
request: LinkRequest,
|
||||
page_request: object,
|
||||
*,
|
||||
challenge_detected: bool,
|
||||
page_html: str | None,
|
||||
) -> tuple[str, str]:
|
||||
"""Build (content_type, response_content) from the settled page."""
|
||||
if request.return_only_cookies:
|
||||
response_content = ""
|
||||
elif page_request and page_request.headers.get("content-type", "").startswith(
|
||||
return "text/html", ""
|
||||
|
||||
if page_request and page_request.headers.get("content-type", "").startswith(
|
||||
"application/pdf"
|
||||
):
|
||||
content_type = "application/pdf"
|
||||
try:
|
||||
fetch_response = await dep.page.request.fetch(dep.page.url)
|
||||
response_content = base64.b64encode(
|
||||
await fetch_response.body()
|
||||
).decode("ascii")
|
||||
except Exception:
|
||||
logger.exception("Failed to fetch PDF bytes, falling back to viewer HTML")
|
||||
content_type = "text/html"
|
||||
response_content = await dep.page.content()
|
||||
else:
|
||||
response_content = await dep.page.content()
|
||||
return await _fetch_pdf_content(dep)
|
||||
|
||||
return LinkResponse(
|
||||
message="Success",
|
||||
solution=Solution(
|
||||
user_agent=await dep.page.evaluate("navigator.userAgent"),
|
||||
url=final_url if final_url is not None else dep.page.url,
|
||||
status=status,
|
||||
cookies=cookies,
|
||||
headers=page_request.headers if page_request else {},
|
||||
response=response_content,
|
||||
content_type=content_type,
|
||||
),
|
||||
start_timestamp=start_time,
|
||||
response_content = (
|
||||
page_html
|
||||
if page_html is not None and not challenge_detected
|
||||
else await dep.page.content()
|
||||
)
|
||||
return "text/html", response_content
|
||||
|
||||
|
||||
async def _fetch_pdf_content(dep: BrowserDep) -> tuple[str, str]:
|
||||
"""Fetch raw PDF bytes as base64, falling back to viewer HTML on failure."""
|
||||
try:
|
||||
fetch_response = await dep.page.request.fetch(dep.page.url)
|
||||
response_content = base64.b64encode(
|
||||
await fetch_response.body()
|
||||
).decode("ascii")
|
||||
except Exception:
|
||||
logger.exception("Failed to fetch PDF bytes, falling back to viewer HTML")
|
||||
return "text/html", await dep.page.content()
|
||||
return "application/pdf", response_content
|
||||
|
||||
+1
-1
@@ -78,7 +78,7 @@ class LinkResponse(BaseModel):
|
||||
version: str = consts.VERSION
|
||||
|
||||
@classmethod
|
||||
def invalid(cls, url: str):
|
||||
def invalid(cls, url: str) -> LinkResponse:
|
||||
"""
|
||||
Return an invalid LinkResponse with default error values.
|
||||
|
||||
|
||||
+1
-1
@@ -69,7 +69,7 @@ async def load_urls(
|
||||
except PlaywrightTimeoutError:
|
||||
logger.debug("networkidle timed out for %s; extracting anyway", url)
|
||||
content = await _extract_content(dep.page)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to load %s: %s", url, exc)
|
||||
content = ""
|
||||
results.append(LoadResult(page_content=content, metadata={"source": url}))
|
||||
|
||||
+2
-1
@@ -13,6 +13,7 @@ from playwright_captcha import (
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from src.consts import (
|
||||
BROWSER_LOCALE,
|
||||
LOG_LEVEL,
|
||||
MAX_ATTEMPTS,
|
||||
PROXY_PASSWORD,
|
||||
@@ -94,7 +95,7 @@ async def get_browser(
|
||||
headless=True,
|
||||
proxy=proxy_config,
|
||||
humanize=True,
|
||||
locale="auto",
|
||||
locale=BROWSER_LOCALE or "auto",
|
||||
) as browser_raw:
|
||||
# InvisiblePlaywright yields a Browser instance
|
||||
browser = cast("Browser", browser_raw)
|
||||
|
||||
+7
-2
@@ -1,3 +1,4 @@
|
||||
import base64
|
||||
from http import HTTPStatus
|
||||
from json import JSONDecodeError
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
@@ -83,7 +84,6 @@ def test_pdf_handling():
|
||||
if solution.get("contentType") != "application/pdf":
|
||||
pytest.skip("Skipping PDF test - PDF bytes could not be fetched (upstream issue)")
|
||||
assert solution["response"] # non-empty base64
|
||||
import base64
|
||||
|
||||
decoded = base64.b64decode(solution["response"])
|
||||
assert decoded[:5] == b"%PDF-"
|
||||
@@ -116,6 +116,9 @@ def fake_dep(*, fail_states: set[str] | None = None) -> BrowserDepClass:
|
||||
page.title.return_value = "Login"
|
||||
page.evaluate.return_value = "UnitTestBrowser/1.0"
|
||||
page.content.return_value = "<html><title>Login</title></html>"
|
||||
locator = MagicMock()
|
||||
locator.count = AsyncMock(return_value=0)
|
||||
page.locator = MagicMock(return_value=locator)
|
||||
|
||||
def wait_for_load_state(state: str, **_kwargs: object) -> None:
|
||||
"""Fail the wait when asked for a configured state."""
|
||||
@@ -133,14 +136,16 @@ def fake_dep(*, fail_states: set[str] | None = None) -> BrowserDepClass:
|
||||
@pytest.mark.asyncio
|
||||
async def test_networkidle_timeout_after_domcontentloaded_returns_content():
|
||||
"""Pages that never go idle after DOM load must still return their content."""
|
||||
dep = fake_dep(fail_states={"networkidle"})
|
||||
response = await read_item(
|
||||
LinkRequest(url="https://example.test/login"),
|
||||
fake_dep(fail_states={"networkidle"}),
|
||||
dep,
|
||||
)
|
||||
|
||||
assert response.status == "ok"
|
||||
assert response.solution.status == HTTPStatus.OK
|
||||
assert response.solution.response == "<html><title>Login</title></html>"
|
||||
dep.solver.solve_captcha.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -109,7 +109,7 @@ dev = [
|
||||
{ name = "ruff", specifier = "==0.16.*" },
|
||||
]
|
||||
test = [
|
||||
{ name = "httpx2", specifier = "==2.9.*" },
|
||||
{ name = "httpx2", specifier = "==2.10.*" },
|
||||
{ name = "pytest", specifier = "==9.1.*" },
|
||||
{ name = "pytest-asyncio", specifier = "==1.4.*" },
|
||||
{ name = "pytest-retry", specifier = "==1.7.*" },
|
||||
@@ -450,15 +450,15 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "httpcore2"
|
||||
version = "2.9.1"
|
||||
version = "2.10.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "h11" },
|
||||
{ name = "truststore" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/39/a8/20ed1ed79cbc2ecdf5301c0968ab7c85547212e2a7bd126ddd2d986e206e/httpcore2-2.9.1.tar.gz", hash = "sha256:4d8acbf8b306f48c9d6046591fd5ba4037d1b1b1000d140fc2c3eab1e9a0c0e2", size = 67089, upload-time = "2026-07-24T09:21:03.867Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a9/83/a896fc59940fc5a6e2aff3a4be1d92fa890112936803b331cae75a993c34/httpcore2-2.10.0.tar.gz", hash = "sha256:13c0cc3d1919d4f28457f60cd2c2abe04113a8af184ccf1142811beba936f9dc", size = 67427, upload-time = "2026-08-09T09:11:32.123Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/9f/fb/46c52b781975c335a2bcf1072c7bbc007cbdc8d674217f5ee1daba2c848b/httpcore2-2.9.1-py3-none-any.whl", hash = "sha256:6182472379e855fe4221246a2bb7ecede403bc61c6798062ae1787d051ccde26", size = 82809, upload-time = "2026-07-24T09:21:01.178Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e5/4f/d149104195a35e2853a2fc203a8e3477747e58c80e17dda686dace174383/httpcore2-2.10.0-py3-none-any.whl", hash = "sha256:7df06cfb34070cae4f7c89be69dc1095eca138e9704ceffb98d25c1912ab6f01", size = 83000, upload-time = "2026-08-09T09:11:29.555Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -500,17 +500,27 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "httpx2"
|
||||
version = "2.9.1"
|
||||
version = "2.10.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "anyio" },
|
||||
{ name = "httpcore2" },
|
||||
{ name = "anyio", marker = "sys_platform != 'emscripten'" },
|
||||
{ name = "httpcore2", marker = "sys_platform != 'emscripten'" },
|
||||
{ name = "httpx2-jsfetch", marker = "sys_platform == 'emscripten'" },
|
||||
{ name = "idna" },
|
||||
{ name = "truststore" },
|
||||
{ name = "truststore", marker = "sys_platform != 'emscripten'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/21/14/38128fbafd7e0ed41d874df6c9a653d47c2d111cfe59e2b4ac95161b4abd/httpx2-2.9.1.tar.gz", hash = "sha256:1932a768737e3666291582833da748cc4e563c337cf96706fccc04fa6e58764a", size = 95458, upload-time = "2026-07-24T09:21:04.972Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/bd/3d/f9a8c07a3884f3e5b26205e8436a18b3af61c5d53192c3bea235574dbbec/httpx2-2.10.0.tar.gz", hash = "sha256:8741d7329fe2c7885fc9ceb61c8217acfb87a85f75723714b89ebf7ad7196338", size = 98749, upload-time = "2026-08-09T09:11:33.24Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/13/b8/cfd91c4ab9134d386d48f0b6ac662ff3d4be6efdee59ee1c67ebc3c0487c/httpx2-2.9.1-py3-none-any.whl", hash = "sha256:1820fe14a9ab1107bfeff39259987429450b070ec0ff38cc87eb0d8c97fdc71a", size = 91191, upload-time = "2026-07-24T09:21:02.6Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b9/6d/a637d52449d98a6892d9a4dc0262587afdb6a66f201871842dce5a97b1c1/httpx2-2.10.0-py3-none-any.whl", hash = "sha256:5e3194a432701e1cc6f69a8b1b2fa199ef907013fede8d9a09a2c5b7b8141a18", size = 94355, upload-time = "2026-08-09T09:11:30.882Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "httpx2-jsfetch"
|
||||
version = "1.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/cd/c4/0e5636363151a2a1795e0a77617168b9ca438e1748ec05fc9b5687f93d64/httpx2_jsfetch-1.0.tar.gz", hash = "sha256:70a0e3eabfef7cce5ad9c629f7d01ca05e418f586646f4ddf14782e4c1454c60", size = 6872, upload-time = "2026-08-07T00:13:07.492Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/9b/43/832f631d32e4f1211caa2ba368317739fe71f0b8530e4c9d15dc454bac2a/httpx2_jsfetch-1.0-py3-none-any.whl", hash = "sha256:cb916b707601e69a07721aabc8f3f6659be3a6893bc1ff5c6f9e02241df2da32", size = 6382, upload-time = "2026-08-07T00:13:06.567Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
Reference in New Issue
Block a user