chore(deps): lock file maintenance

Added a sponsor badge to the README.

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# Supported funding model platforms
+
+github: [ThePhaseless]

AGENTS.md

@@ -2,14 +2,14 @@
 
 ## Project overview
 
-- FastAPI service that mimics FlareSolverr-style API for bypassing anti-bot pages using Camoufox + Playwright captcha solver.
-- Entry point: main app in main.py; routes and request flow defined in src/endpoints.py and src/models.py.
-- Browser lifecycle is owned by get_camoufox() in src/utils.py, which yields a page, solver, and context for each request.
+- FastAPI service that mimics FlareSolverr-style API for bypassing anti-bot pages using invisible_playwright.
+- Entry point: main app in main.py; routes and request flow in src/endpoints.py, challenge handling in src/challenge.py, response bodies in src/content.py, models in src/models.py.
+- Browser lifecycle is owned by get_browser() in src/utils.py, which yields a page and context for each request.
 
 ## Architecture and data flow
 
-- Request flow: POST /v1 -> read_item() -> browser.goto() -> wait for load states -> detect challenge title -> solve captcha -> return LinkResponse.
-- Challenge detection uses CHALLENGE_TITLES from src/consts.py; update this map when adding providers.
+- Request flow: POST /v1 -> read_item() -> page.goto() -> wait for load states -> detect the interstitial -> click its checkbox until it clears -> return LinkResponse.
+- Challenge detection uses detect_cloudflare_challenge() from playwright_captcha; the challenge is over when its markup goes, not when a solver says so.
 - Health check hits /v1 internally with <https://google.com> and fails if status is not OK.
 - Logging: LogRequest middleware logs only POST /v1 timing and outcome; other paths pass through.
 
@@ -28,7 +28,7 @@
 
 - Local run: uv sync && uv run main.py
 - Init mode: uv run main.py --init (pre-warms health check via browser setup)
-- Tests: uv sync --group test && uv run pytest --retries 3
+- Tests: uv sync --group test && uv run pytest --retries 5
 - Docker troubleshooting: docker build --target test .
 
 ## Tests and external dependencies

Dockerfile

@@ -51,7 +51,7 @@ RUN mkdir -p /home/byparr &&\
 FROM app AS test
 RUN \
     uv sync --group test &&\
-    uv run pytest -rs --retries 3
+    uv run pytest -rs --retries 5
 
 FROM app
 ARG VERSION

README.md

@@ -1,4 +1,4 @@
-# Byparr
+# Byparr [![Sponsor](https://img.shields.io/badge/Sponsor-%E2%9D%A4-ea4aaa?logo=github-sponsors)](https://github.com/sponsors/ThePhaseless)
 
 <p align="center">
    <img src="icon/logo-byparr.svg" alt="Byparr logo" width="120" />

pyproject.toml

@@ -10,7 +10,7 @@ readme = "README.md"
 dependencies = [
     "fastapi[standard]==0.141.*",
     "invisible-playwright>=0.6.1",
-    "playwright==1.60.*",
+    "playwright==1.63.*",
     "playwright-captcha==0.1.*",
     "pydantic==2.*",
     "pydantic-settings==2.*",
@@ -20,7 +20,7 @@ urls = { repository = "https://github.com/ThePhaseless/Byparr" }
 [dependency-groups]
 
 test = [
-    "httpx2==2.10.*",
+    "httpx2==2.13.*",
     "pytest==9.1.*",
     "pytest-asyncio==1.4.*",
     "pytest-retry==1.7.*",

src/challenge.py

@@ -0,0 +1,117 @@
+import time
+from asyncio import sleep
+from contextlib import suppress
+
+from playwright.async_api import Error as PlaywrightError
+from playwright.async_api import FloatRect, Page
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+from playwright_captcha.solvers.click.cloudflare.utils.detection import (
+    CF_INTERSTITIAL_INDICATORS_SELECTORS,
+    detect_cloudflare_challenge,
+)
+
+from src.utils import TimeoutTimer, logger
+
+__all__ = [
+    "CF_INTERSTITIAL_INDICATORS_SELECTORS",
+    "challenge_present",
+    "solve_challenge",
+]
+
+POLL_INTERVAL = 0.25
+CLICK_SETTLE = 1.5
+CLICK_COOLDOWN = 4
+PROBE_INTERVAL = 0.5
+TOKEN_READ_TIMEOUT = 1000
+BOX_READ_TIMEOUT = 1000
+CHECKBOX_INSET = 25
+TURNSTILE_INPUT = 'input[name="cf-turnstile-response"]'
+WIDGET_ANCESTOR_DEPTHS = (1, 2, 3, 4)
+WIDGET_MIN_WIDTH = 40
+WIDGET_MIN_HEIGHT = 20
+WIDGET_MAX_HEIGHT = 120
+
+
+async def challenge_present(page: Page) -> bool:
+    """Report whether the Cloudflare interstitial is up."""
+    return await detect_cloudflare_challenge(page, "interstitial")
+
+
+async def widget_box(page: Page) -> FloatRect | None:
+    """Measure the widget container with locators; running page scripts resets the challenge."""
+    for depth in WIDGET_ANCESTOR_DEPTHS:
+        widget = page.locator(f"{TURNSTILE_INPUT} >> xpath=ancestor::div[{depth}]")
+        with suppress(PlaywrightError, PlaywrightTimeoutError):
+            if await widget.count() == 0:
+                continue
+            box = await widget.first.bounding_box(timeout=BOX_READ_TIMEOUT)
+            if (
+                box
+                and box["width"] > WIDGET_MIN_WIDTH
+                and WIDGET_MIN_HEIGHT < box["height"] < WIDGET_MAX_HEIGHT
+            ):
+                return box
+    return None
+
+
+async def click_checkbox(page: Page) -> bool:
+    """Click the checkbox through its container, leaving its closed shadow root alone."""
+    box = await widget_box(page)
+    if box is None:
+        return False
+    await page.mouse.move(box["x"] + CHECKBOX_INSET, box["y"] + box["height"] / 2)
+    try:
+        await page.mouse.down()
+    finally:
+        await page.mouse.up()
+    return True
+
+
+async def checkbox_already_answered(page: Page) -> bool:
+    """Report whether Turnstile has already filled in its response token."""
+    token = page.locator(TURNSTILE_INPUT)
+    with suppress(PlaywrightError, PlaywrightTimeoutError):
+        if await token.count() > 0:
+            return bool(await token.first.input_value(timeout=TOKEN_READ_TIMEOUT))
+    return False
+
+
+async def challenge_is_gone(page: Page) -> bool:
+    """Confirm the interstitial is really gone and not just between navigations."""
+    if await challenge_present(page):
+        return False
+    await sleep(POLL_INTERVAL)
+    return not await challenge_present(page)
+
+
+async def solve_challenge(page: Page, timer: TimeoutTimer) -> None:
+    """Wait out the interstitial, clicking its checkbox whenever one is offered."""
+    logger.info("Challenge detected, waiting for it to clear...")
+    clicks = 0
+    next_click = 0.0
+    while True:
+        if await challenge_is_gone(page):
+            logger.debug("Challenge cleared.")
+            if clicks:
+                await sleep(min(CLICK_SETTLE, timer.remaining()))
+            return
+
+        if time.perf_counter() >= next_click:
+            landed = False
+            with suppress(PlaywrightError, PlaywrightTimeoutError):
+                landed = not await checkbox_already_answered(
+                    page
+                ) and await click_checkbox(page)
+            if landed:
+                clicks += 1
+                logger.info("Clicked the challenge checkbox (attempt %d).", clicks)
+            next_click = time.perf_counter() + (
+                CLICK_COOLDOWN if landed else PROBE_INTERVAL
+            )
+
+        if timer.remaining() <= 0:
+            break
+        await sleep(POLL_INTERVAL)
+
+    message = "Challenge still present when the request budget ran out"
+    raise TimeoutError(message)

src/consts.py

@@ -9,13 +9,6 @@ class Settings(BaseSettings):
     log_level: str = "INFO"
     version: str = "unknown"
 
-    # The solver retries whenever it cannot reach the challenge widget, and
-    # that failure is usually structural rather than transient -- an
-    # unreachable widget stays unreachable. sys.maxsize meant a single request
-    # burned its whole max_timeout on ~1300 identical failed attempts before
-    # reporting a 408. Give it a handful of tries, then let the caller know.
-    max_attempts: int = 5
-
     proxy_server: str | None = None
     proxy_username: str | None = None
     proxy_password: str | None = None
@@ -34,8 +27,6 @@ settings = Settings()
 LOG_LEVEL = logging.getLevelNamesMapping()[settings.log_level.upper()]
 VERSION = settings.version.removeprefix("v")
 
-MAX_ATTEMPTS = settings.max_attempts
-
 PROXY_SERVER = settings.proxy_server
 PROXY_USERNAME = settings.proxy_username
 PROXY_PASSWORD = settings.proxy_password

src/content.py

@@ -0,0 +1,42 @@
+import base64
+
+from playwright.async_api import Page
+
+from src.models import LinkRequest
+from src.utils import logger
+
+
+async def build_response_content(
+    page: Page,
+    request: LinkRequest,
+    page_request: object,
+    *,
+    challenge_detected: bool,
+    page_html: str | None,
+) -> tuple[str, str]:
+    """Build (content_type, response_content) from the settled page."""
+    if request.return_only_cookies:
+        return "text/html", ""
+
+    if page_request and page_request.headers.get("content-type", "").startswith(
+        "application/pdf"
+    ):
+        return await fetch_pdf_content(page)
+
+    response_content = (
+        page_html
+        if page_html is not None and not challenge_detected
+        else await page.content()
+    )
+    return "text/html", response_content
+
+
+async def fetch_pdf_content(page: Page) -> tuple[str, str]:
+    """Fetch raw PDF bytes as base64, falling back to viewer HTML on failure."""
+    try:
+        fetch_response = await page.request.fetch(page.url)
+        response_content = base64.b64encode(await fetch_response.body()).decode("ascii")
+    except Exception:
+        logger.exception("Failed to fetch PDF bytes, falling back to viewer HTML")
+        return "text/html", await page.content()
+    return "application/pdf", response_content

src/endpoints.py

@@ -1,30 +1,28 @@
-import base64
 import time
 import warnings
-from asyncio import sleep, wait_for
 from http import HTTPStatus
 from typing import Annotated
 
 from fastapi import APIRouter, Depends, HTTPException
 from fastapi.responses import RedirectResponse
-from playwright.async_api import Page
+from playwright.async_api import Error as PlaywrightError
 from playwright.async_api import TimeoutError as PlaywrightTimeoutError
-from playwright_captcha import CaptchaType
-from playwright_captcha.solvers.click.cloudflare.utils.detection import (
-    detect_cloudflare_challenge,
-)
-from playwright_captcha.utils.exceptions import (
-    CaptchaDetectionError,
-    CaptchaSolvingError,
-)
 
+from src.challenge import challenge_present, solve_challenge
+from src.content import build_response_content
 from src.models import (
     HealthcheckResponse,
     LinkRequest,
     LinkResponse,
     Solution,
 )
-from src.utils import BrowserDepClass, TimeoutTimer, get_browser, logger
+from src.utils import (
+    BrowserDepClass,
+    TimeoutTimer,
+    get_browser,
+    logger,
+    remaining_ms,
+)
 
 warnings.filterwarnings("ignore", category=SyntaxWarning)
 
@@ -33,24 +31,6 @@ router = APIRouter()
 
 BrowserDep = Annotated[BrowserDepClass, Depends(get_browser)]
 
-# Markup only an unsolved challenge has. Two near misses to avoid:
-#
-#   script[src*="/cdn-cgi/challenge-platform/"] on its own also matches the jsd
-#   bot-scoring beacon Cloudflare serves from that path on ordinary pages, so it
-#   has to be narrowed to the challenge orchestrator (chl_page).
-#
-#   iframe[src*="challenges.cloudflare.com"] looks like the widget but outlives
-#   it: a cleared nowsecure.nl carries two of them with no challenge in sight.
-CHALLENGE_MARKERS = (
-    'script[src*="/cdn-cgi/challenge-platform/"][src*="chl_page"], '
-    "#challenge-error-text, #challenge-running, #challenge-stage"
-)
-
-# How long to let Cloudflare verify a click before trying again, and how often
-# to look. Verification took 5-15s in testing.
-CHALLENGE_SETTLE_SECONDS = 20.0
-CHALLENGE_POLL_SECONDS = 1.0
-
 
 @router.get("/", include_in_schema=False)
 def read_root():
@@ -86,7 +66,7 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse:
     await setup_routes(request, dep)
 
     try:
-        challenge_detected, page_html, page_request, status = await _navigate_and_solve(
+        challenge_detected, page_html, page_request = await _navigate_and_solve(
             dep, request, timer
         )
     except (TimeoutError, PlaywrightTimeoutError) as e:
@@ -95,24 +75,32 @@ async def read_item(request: LinkRequest, dep: BrowserDep) -> LinkResponse:
             status_code=408,
             detail="Timed out while loading the page or solving the challenge",
         ) from e
+    except PlaywrightError as e:
+        logger.error("Could not reach the target: %s", e)
+        raise HTTPException(
+            status_code=502,
+            detail=f"Could not reach the target: {e}",
+        ) from e
 
     cookies = await dep.context.cookies()
     content_type, response_content = await build_response_content(
-        dep,
+        dep.page,
         request,
         page_request,
         challenge_detected=challenge_detected,
         page_html=page_html,
     )
 
-    user_agent = page_request.request.headers.get("user-agent") if page_request else ""
+    user_agent = (
+        page_request.request.headers.get("user-agent") or "" if page_request else ""
+    )
 
     return LinkResponse(
         message="Success",
         solution=Solution(
             user_agent=user_agent,
             url=dep.page.url,
-            status=status,
+            status=HTTPStatus.OK,
             cookies=cookies,
             headers=page_request.headers if page_request else {},
             response=response_content,
@@ -139,149 +127,29 @@ async def _navigate_and_solve(
     dep: BrowserDep,
     request: LinkRequest,
     timer: TimeoutTimer,
-) -> tuple[bool, str | None, object, HTTPStatus]:
+) -> tuple[bool, str | None, object]:
     """Navigate to the URL, then solve a challenge or wait for network idle."""
     page_html: str | None = None
-    page_request = await dep.page.goto(request.url, timeout=timer.remaining() * 1000)
-    status = page_request.status if page_request else HTTPStatus.OK
+    page_request = await dep.page.goto(request.url, timeout=remaining_ms(timer))
     await dep.page.wait_for_load_state(
-        state="domcontentloaded", timeout=timer.remaining() * 1000
+        state="domcontentloaded", timeout=remaining_ms(timer)
     )
 
-    challenge_active = await detect_cloudflare_challenge(
-        dep.page, "interstitial"
-    ) or await detect_cloudflare_challenge(dep.page, "turnstile")
-    if not challenge_active:
+    if not await challenge_present(dep.page):
         page_html = await dep.page.content()
         await _wait_for_networkidle(dep, timer)
-        return False, page_html, page_request, status
+        return False, page_html, page_request
 
-    await _solve_challenge(dep, timer)
-    status = HTTPStatus.OK
-    return True, page_html, page_request, status
-
-
-async def _solve_challenge(dep: BrowserDep, timer: TimeoutTimer) -> None:
-    """
-    Attempt to solve a detected Cloudflare interstitial challenge.
-
-    The solver's own verdict is not trusted. It judges its click by waiting for
-    networkidle, which returns the moment the network happens to be quiet --
-    measured at 9ms after the click -- and then reports "challenge still
-    present". Cloudflare needs seconds: it replaces the checkbox with
-    "verifying you are human" before the page turns over. So try, then wait for
-    the challenge itself to go away, and only give up once the budget is gone.
-
-    That bounds what used to be unbounded. With max_attempts at sys.maxsize the
-    solver retried an unreachable widget ~1300 times per request and the caller
-    waited out the whole max_timeout for a 408 it was always going to get.
-    """
-    logger.info("Challenge detected, attempting to solve...")
-    while timer.remaining() > 0:
-        try:
-            await wait_for(
-                dep.solver.solve_captcha(  # pyright: ignore[reportUnknownMemberType,reportUnknownArgumentType]
-                    captcha_container=dep.page,
-                    captcha_type=CaptchaType.CLOUDFLARE_INTERSTITIAL,
-                    wait_checkbox_attempts=1,
-                    wait_checkbox_delay=0.5,
-                ),
-                timeout=timer.remaining(),
-            )
-        except (CaptchaDetectionError, CaptchaSolvingError) as e:
-            # Both mean "not solved yet", not "cannot be solved": the widget is
-            # mid-verification and momentarily absent, or the click has landed
-            # and the verdict was taken too early.
-            logger.debug(f"Solver attempt inconclusive: {e}")
-
-        if await _challenge_cleared(dep, timer):
-            logger.info("Challenge cleared")
-            return
-
-    message = "Challenge still present when the request budget ran out"
-    raise TimeoutError(message)
-
-
-async def _challenge_cleared(dep: BrowserDep, timer: TimeoutTimer) -> bool:
-    """
-    Wait out Cloudflare's post-click verification, up to CHALLENGE_SETTLE_SECONDS.
-
-    Polls rather than waiting for a load event: Cloudflare swaps the widget for
-    a spinner in place and only navigates once it is satisfied, so there is no
-    single event to await.
-    """
-    deadline = min(CHALLENGE_SETTLE_SECONDS, timer.remaining())
-    waited = 0.0
-    while waited < deadline:
-        if not await _challenge_visible(dep.page):
-            return True
-        await sleep(CHALLENGE_POLL_SECONDS)
-        waited += CHALLENGE_POLL_SECONDS
-    return not await _challenge_visible(dep.page)
-
-
-async def _challenge_visible(page: Page) -> bool:
-    """
-    Report whether an unsolved challenge is still on the page.
-
-    Cloudflare serves two different scripts from /cdn-cgi/challenge-platform/:
-    the challenge orchestrator on an interstitial, and the jsd bot-scoring
-    beacon on ordinary pages once a visitor is cleared. detect_cloudflare_
-    challenge() matches both, so on its own it never reports success. Match the
-    orchestrator and the widget instead.
-    """
-    try:
-        return await page.locator(CHALLENGE_MARKERS).count() > 0
-    except Exception:
-        # A navigation tore down the execution context mid-check, which only
-        # happens once Cloudflare has moved us on.
-        logger.debug("Challenge lookup interrupted by a navigation")
-        return False
+    await solve_challenge(dep.page, timer)
+    await _wait_for_networkidle(dep, timer)
+    return True, page_html, page_request
 
 
 async def _wait_for_networkidle(dep: BrowserDep, timer: TimeoutTimer) -> None:
     """Wait for network idle, tolerating post-DOM-load stalls."""
     try:
-        await dep.page.wait_for_load_state(
-            "networkidle", timeout=timer.remaining() * 1000
-        )
+        await dep.page.wait_for_load_state("networkidle", timeout=remaining_ms(timer))
     except PlaywrightTimeoutError:
         logger.info(
             "networkidle timed out after domcontentloaded; continuing with loaded page"
         )
-
-
-async def build_response_content(
-    dep: BrowserDep,
-    request: LinkRequest,
-    page_request: object,
-    *,
-    challenge_detected: bool,
-    page_html: str | None,
-) -> tuple[str, str]:
-    """Build (content_type, response_content) from the settled page."""
-    if request.return_only_cookies:
-        return "text/html", ""
-
-    if page_request and page_request.headers.get("content-type", "").startswith(
-        "application/pdf"
-    ):
-        return await _fetch_pdf_content(dep)
-
-    response_content = (
-        page_html
-        if page_html is not None and not challenge_detected
-        else await dep.page.content()
-    )
-    return "text/html", response_content
-
-
-async def _fetch_pdf_content(dep: BrowserDep) -> tuple[str, str]:
-    """Fetch raw PDF bytes as base64, falling back to viewer HTML on failure."""
-    try:
-        fetch_response = await dep.page.request.fetch(dep.page.url)
-        response_content = base64.b64encode(await fetch_response.body()).decode("ascii")
-    except Exception:
-        logger.exception("Failed to fetch PDF bytes, falling back to viewer HTML")
-        return "text/html", await dep.page.content()
-    return "application/pdf", response_content

src/utils.py

@@ -6,16 +6,11 @@ from typing import Annotated, NamedTuple, cast
 from fastapi import Header
 from invisible_playwright.async_api import InvisiblePlaywright
 from playwright.async_api import Browser, BrowserContext, Page
-from playwright_captcha import (
-    ClickSolver,
-    FrameworkType,
-)
 from pydantic import BaseModel, Field
 
 from src.consts import (
     BROWSER_LOCALE,
     LOG_LEVEL,
-    MAX_ATTEMPTS,
     PROXY_PASSWORD,
     PROXY_SERVER,
     PROXY_USERNAME,
@@ -35,31 +30,6 @@ if len(logger.handlers) == 0:
     logger.addHandler(logging.StreamHandler())
 
 
-# Cloudflare embeds its challenge widget in an iframe carrying
-# allow="cross-origin-isolated". Firefox honours that by moving the iframe into
-# a cross-origin-isolated content process, where Juggler sees a frame with no
-# docShell and no URL, so content_frame() raises "Permission denied to access
-# property docShell on cross-origin object" and the solver never reaches the
-# checkbox.
-#
-# Turning the two policies off (as upstream Playwright's Firefox does, and as
-# v2.1.0 did via camoufox's disable_coop=True) restores that access, and the
-# solver then clicks the checkbox successfully.
-#
-# It is not a demonstrated win: from a datacenter IP Cloudflare rejects the
-# click regardless -- measured across eight sites, nine clicks, and an
-# undetectable shadow-root patch -- so no outcome changed here. It is kept for
-# parity with v2, which users report worked on these sites, because reaching
-# the checkbox is a precondition for ever passing an interactive challenge and
-# Byparr mostly runs from residential addresses that Cloudflare treats far
-# better than CI does.
-BROWSER_PREFS = {
-    "devtools.jsonview.enabled": False,
-    "browser.tabs.remote.useCrossOriginOpenerPolicy": False,
-    "browser.tabs.remote.useCrossOriginEmbedderPolicy": False,
-}
-
-
 class TimeoutTimer(BaseModel):
     duration: int  # in seconds
     start_time: float = Field(default_factory=time.perf_counter)
@@ -69,9 +39,16 @@ class TimeoutTimer(BaseModel):
         return max(0, self.duration - (time.perf_counter() - self.start_time))
 
 
+MIN_WAIT_MS = 1.0
+
+
+def remaining_ms(timer: TimeoutTimer) -> float:
+    """Milliseconds left, never 0 - Playwright reads that as no timeout at all."""
+    return max(MIN_WAIT_MS, timer.remaining() * 1000)
+
+
 class BrowserDepClass(NamedTuple):
     page: Page
-    solver: ClickSolver
     context: BrowserContext
 
 
@@ -121,22 +98,14 @@ async def get_browser(
         proxy=proxy_config,
         humanize=True,
         locale=BROWSER_LOCALE or "auto",
-        extra_prefs=BROWSER_PREFS,
+        extra_prefs={
+            "devtools.jsonview.enabled": False,
+            "browser.tabs.remote.useCrossOriginOpenerPolicy": False,
+            "browser.tabs.remote.useCrossOriginEmbedderPolicy": False,
+        },
     ) as browser_raw:
         # InvisiblePlaywright yields a Browser instance
         browser = cast("Browser", browser_raw)
         context = await browser.new_context()
         page = await context.new_page()
-        async with ClickSolver(
-            # Not PATCHRIGHT: that path skips the unlockShadowRoot init script
-            # and injects it over CDP instead, which Firefox has no session for
-            # ("CDP session is only available in Chromium"). Cloudflare builds
-            # its widget inside a closed shadow root, so without that script
-            # nothing -- not the solver, not page.locator -- can see the
-            # challenge iframe, and every solve attempt fails outright.
-            framework=FrameworkType.PLAYWRIGHT,
-            page=page,
-            max_attempts=MAX_ATTEMPTS,
-            attempt_delay=1,
-        ) as solver:
-            yield BrowserDepClass(page, solver, context)
+        yield BrowserDepClass(page, context)

tests/main_test.py

@@ -1,6 +1,4 @@
 import base64
-import json
-import re
 from http import HTTPStatus
 from json import JSONDecodeError
 from unittest.mock import AsyncMock, MagicMock
@@ -8,53 +6,35 @@ from unittest.mock import AsyncMock, MagicMock
 import httpx2
 import pytest
 from fastapi import HTTPException
+from playwright.async_api import Error as PlaywrightError
 from playwright.async_api import TimeoutError as PlaywrightTimeoutError
-from playwright_captcha.utils.exceptions import (
-    CaptchaDetectionError,
-    CaptchaSolvingError,
-)
 from starlette.testclient import TestClient
 
 from main import app
-from src.endpoints import CHALLENGE_MARKERS, read_item
+from src.challenge import CF_INTERSTITIAL_INDICATORS_SELECTORS
+from src.endpoints import read_item
 from src.models import LinkRequest
-from src.utils import BrowserDepClass
+from src.utils import BrowserDepClass, TimeoutTimer, remaining_ms
 
 client = TestClient(app)
 
-# Real Firefox advertises 16 cipher suites; Playwright's HTTP client advertised
-# 52. A small margin absorbs Firefox version drift without letting 52 through.
-FIREFOX_CIPHER_SUITE_CEILING = 20
-
-# Sites Byparr clears from any network, datacenter ranges included. These carry
-# the hard assertion: if the bypass breaks, one of these goes red.
 test_websites = [
-    # Purpose-built Cloudflare challenge target. Serves a real interstitial and
-    # hands back a cf_clearance cookie once it is passed, so a pass here means
-    # the challenge was solved rather than never presented.
-    "https://nowsecure.nl/",
-    'https://www.yggtorrent.top/engine/search?do=search&order=desc&sort=publish_date&name="UNESCAPED"+"DOUBLEQUOTES"&category=2145',
-]
-
-# Cloudflare hands these its interactive checkbox challenge and then refuses the
-# click from datacenter ranges: the widget goes to "verifying you are human" and
-# comes back as a fresh unchecked box, indefinitely. Measured over four fresh
-# navigations and nine clicks, and reproduced from two unrelated hosting
-# providers on two architectures -- it is the visitor's IP being judged, not our
-# code. They still run rather than being skipped, so a real regression is
-# visible in the report and a pass is recorded as xpass, but the runner's luck
-# with Cloudflare cannot turn the build red.
-datacenter_hostile_websites = [
     "https://ext.to/",
     # "https://www.ygg.re/",
     "https://extratorrent.st/",
     "https://speed.cd/login",
+    'https://www.yggtorrent.top/engine/search?do=search&order=desc&sort=publish_date&name="UNESCAPED"+"DOUBLEQUOTES"&category=2145',
     "https://1337x.to/home/",
 ]
 
 
-def _bypass(website: str) -> None:
-    """Ask Byparr for the page and require a clean answer."""
+@pytest.mark.parametrize("website", test_websites)
+def test_bypass(website: str):
+    """
+    Tests if the service can bypass cloudflare/DDOS-GUARD on given websites.
+
+    This test is skipped if the website is not reachable or does not have cloudflare/DDOS-GUARD.
+    """
     test_request = httpx2.get(
         website,
     )
@@ -72,30 +52,20 @@ def _bypass(website: str) -> None:
 
     response = client.post(
         "/v1",
-        json=LinkRequest.model_construct(url=website, cmd="request.get").model_dump(),
+        json=LinkRequest.model_construct(
+            url=website, cmd="request.get", max_timeout=60
+        ).model_dump(),
     )
 
     assert response.status_code == HTTPStatus.OK
-
-
-@pytest.mark.parametrize("website", test_websites)
-def test_bypass(website: str):
-    """Tests if the service can bypass cloudflare/DDOS-GUARD on given websites."""
-    _bypass(website)
-
-
-@pytest.mark.xfail(
-    reason="Cloudflare refuses the checkbox click from datacenter IPs",
-    strict=False,
-)
-@pytest.mark.parametrize("website", datacenter_hostile_websites)
-def test_bypass_datacenter_hostile(website: str):
-    """Same check against sites Cloudflare guards hardest, outcome permitting."""
-    _bypass(website)
+    solution = response.json()["solution"]
+    assert "_cf_chl_opt" not in solution["response"]
+    assert "__cf_chl" not in solution["url"]
 
 
 def test_json_api():
-    """JSON APIs must return 200, not crash on the UA evaluate.
+    """
+    JSON APIs must return 200, not crash on the UA evaluate.
 
     Firefox renders application/json in a built-in viewer whose CSP blocks
     Playwright's eval-based evaluate() (issue #394). The browser must be
@@ -122,38 +92,6 @@ def test_json_api():
     assert '"ip"' in solution["response"]
 
 
-def test_tls_handshake_looks_like_firefox():
-    """
-    The handshake must be Firefox's, not the HTTP client's (#398).
-
-    route.fetch() re-issued navigations through Playwright's own HTTP stack, so
-    the ClientHello advertised 52 cipher suites where Firefox offers 16 -- a
-    fingerprint no amount of header spoofing hides. Unlike a Cloudflare verdict
-    this is deterministic, so it pins the regression that motivated this branch.
-    """
-    url = "https://www.howsmyssl.com/a/check"
-    if httpx2.get(url).status_code >= HTTPStatus.INTERNAL_SERVER_ERROR:
-        pytest.skip("Skipping TLS check - howsmyssl is down")
-
-    response = client.post(
-        "/v1",
-        json=LinkRequest.model_construct(url=url, cmd="request.get").model_dump(),
-    )
-    assert response.status_code == HTTPStatus.OK
-
-    body = response.json()["solution"]["response"]
-    report = json.loads(
-        re.sub(r"<[^>]+>", "", re.search(r"\{.*\}", body, re.DOTALL).group(0))
-    )
-    suites = len(report["given_cipher_suites"])
-
-    # Firefox offers 16; Playwright's client offered 52. Anything in between
-    # means the navigation is no longer going through the browser.
-    assert suites <= FIREFOX_CIPHER_SUITE_CEILING, (
-        f"{suites} cipher suites offered - the handshake is not Firefox's"
-    )
-
-
 def test_health_check():
     """
     Tests the health check endpoint.
@@ -208,37 +146,32 @@ def fake_dep(
     fail_states: set[str] | None = None,
     challenged: bool = False,
     marker_counts: list[int] | None = None,
+    widget_box: dict[str, float] | None = None,
+    user_agent: str | None = "UnitTestBrowser/1.0",
 ) -> BrowserDepClass:
-    """
-    Build a browser dependency triple backed by mocks.
-
-    `challenged` makes the detector report a Cloudflare challenge.
-    `marker_counts` drives the "is it still up?" check that runs after each
-    solve attempt: one entry per look, the last one repeating forever.
-    """
+    """Build a browser dependency pair backed by mocks."""
     page = AsyncMock()
     page.url = "https://example.test/login"
     page.goto.return_value = MagicMock(
         status=HTTPStatus.OK,
         headers={"content-type": "text/html"},
-        request=MagicMock(headers={"user-agent": "UnitTestBrowser/1.0"}),
+        request=MagicMock(headers={"user-agent": user_agent} if user_agent else {}),
     )
     page.title.return_value = "Login"
-    page.evaluate.return_value = "UnitTestBrowser/1.0"
     page.content.return_value = "<html><title>Login</title></html>"
-
     remaining = list(marker_counts or [])
 
     def count_for(selector: str) -> int:
-        """Answer the marker check from the script, everything else from `challenged`."""
-        if selector != CHALLENGE_MARKERS or not remaining:
+        """Answer the marker check from the script, else from `challenged`."""
+        if selector not in CF_INTERSTITIAL_INDICATORS_SELECTORS or not remaining:
             return 1 if challenged else 0
         return remaining.pop(0) if len(remaining) > 1 else remaining[0]
 
     def locator(selector: str) -> MagicMock:
         handle = MagicMock()
-        handle.count = AsyncMock(return_value=None)
-        handle.count.side_effect = lambda: count_for(selector)
+        handle.count = AsyncMock(side_effect=lambda: count_for(selector))
+        handle.first.bounding_box = AsyncMock(return_value=widget_box)
+        handle.first.input_value = AsyncMock(return_value="")
         return handle
 
     page.locator = MagicMock(side_effect=locator)
@@ -253,7 +186,7 @@ def fake_dep(
 
     context = AsyncMock()
     context.cookies.return_value = []
-    return BrowserDepClass(page=page, solver=AsyncMock(), context=context)
+    return BrowserDepClass(page=page, context=context)
 
 
 @pytest.mark.asyncio
@@ -266,9 +199,7 @@ async def test_networkidle_timeout_after_domcontentloaded_returns_content():
     )
 
     assert response.status == "ok"
-    assert response.solution.status == HTTPStatus.OK
     assert response.solution.response == "<html><title>Login</title></html>"
-    dep.solver.solve_captcha.assert_not_called()
 
 
 @pytest.mark.asyncio
@@ -284,18 +215,59 @@ async def test_domcontentloaded_timeout_returns_408():
 
 
 @pytest.mark.asyncio
-async def test_challenge_that_clears_after_the_click_succeeds():
-    """
-    The solver's own "challenge still present" verdict must not end the request.
+async def test_unreachable_host_is_a_502_not_a_500():
+    """An upstream we cannot reach is a gateway failure, never a Byparr crash."""
+    dep = fake_dep()
+    dep.page.goto.side_effect = PlaywrightError("Page.goto: NS_ERROR_UNKNOWN_HOST")
 
-    It judges its click by waiting for networkidle, which returns as soon as the
-    network happens to be quiet -- 9ms after the click, in practice -- while
-    Cloudflare is still showing "verifying you are human". Byparr has to wait
-    for the challenge markup itself to go away.
-    """
-    dep = fake_dep(challenged=True, marker_counts=[1, 0])
-    dep.solver.solve_captcha.side_effect = CaptchaSolvingError(
-        "challenge still present or expected content not detected"
+    with pytest.raises(HTTPException) as exc:
+        await read_item(LinkRequest(url="https://nope.invalid/"), dep)
+
+    assert exc.value.status_code == HTTPStatus.BAD_GATEWAY
+
+
+@pytest.mark.asyncio
+async def test_status_is_always_ok_like_flaresolverr():
+    """FlareSolverr hardcodes 200 because Selenium cannot report the real code."""
+    dep = fake_dep()
+    dep.page.goto.return_value = MagicMock(
+        status=HTTPStatus.FORBIDDEN,
+        headers={"content-type": "text/html"},
+        request=MagicMock(headers={"user-agent": "UnitTestBrowser/1.0"}),
+    )
+
+    response = await read_item(LinkRequest(url="https://example.test/login"), dep)
+
+    assert response.solution.status == HTTPStatus.OK
+
+
+def test_exhausted_budget_never_disables_playwright_timeouts():
+    """Playwright reads timeout=0 as no timeout at all, so the floor must hold."""
+    spent = TimeoutTimer(duration=0)
+
+    assert spent.remaining() == 0
+    assert remaining_ms(spent) > 0
+
+
+@pytest.mark.asyncio
+async def test_missing_user_agent_header_is_not_a_500():
+    """A request without a user-agent header degrades to empty, never a 500 (#394)."""
+    response = await read_item(
+        LinkRequest(url="https://example.test/login"),
+        fake_dep(user_agent=None),
+    )
+
+    assert response.status == "ok"
+    assert response.solution.user_agent == ""
+
+
+@pytest.mark.asyncio
+async def test_checkbox_is_clicked_while_the_challenge_is_up():
+    """A measurable widget gets a humanised press, not a raw synthetic click."""
+    dep = fake_dep(
+        challenged=True,
+        marker_counts=[1, 1, 0],
+        widget_box={"x": 100.0, "y": 200.0, "width": 300.0, "height": 60.0},
     )
 
     response = await read_item(
@@ -303,16 +275,32 @@ async def test_challenge_that_clears_after_the_click_succeeds():
     )
 
     assert response.status == "ok"
-    assert response.solution.status == HTTPStatus.OK
+    dep.page.mouse.move.assert_awaited_once_with(125.0, 230.0)
+    dep.page.mouse.down.assert_awaited_once()
+    dep.page.mouse.up.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+async def test_challenge_that_clears_on_its_own_is_never_clicked():
+    """A challenge is over when its markup goes, and until then we keep our hands off."""
+    dep = fake_dep(
+        challenged=True,
+        marker_counts=[1, 0],
+        widget_box={"x": 100.0, "y": 200.0, "width": 300.0, "height": 60.0},
+    )
+
+    response = await read_item(
+        LinkRequest(url="https://example.test/login", max_timeout=5), dep
+    )
+
+    assert response.status == "ok"
+    dep.page.mouse.down.assert_not_awaited()
 
 
 @pytest.mark.asyncio
 async def test_challenge_that_never_clears_returns_408():
     """A challenge still up when the budget runs out is a timeout, not a 500."""
     dep = fake_dep(challenged=True, marker_counts=[1])
-    dep.solver.solve_captcha.side_effect = CaptchaDetectionError(
-        "Cloudflare iframes not found"
-    )
 
     with pytest.raises(HTTPException) as exc:
         await read_item(
@@ -323,19 +311,13 @@ async def test_challenge_that_never_clears_returns_408():
 
 
 @pytest.mark.asyncio
-async def test_user_agent_survives_csp_blocked_evaluate():
-    """UA comes from request headers when page CSP blocks evaluate (#394).
+async def test_marker_vanishing_mid_navigation_is_not_a_solved_challenge():
+    """The marker drops out between challenge rounds; one clear read proves nothing."""
+    dep = fake_dep(challenged=True, marker_counts=[1, 0, 1])
 
-    No CSP configuration (header, meta tag, or internal viewer document) may
-    turn /v1 into a 500.
-    """
-    dep = fake_dep()
-    dep.page.evaluate.side_effect = Exception("call to eval() blocked by CSP")
+    with pytest.raises(HTTPException) as exc:
+        await read_item(
+            LinkRequest(url="https://example.test/login", max_timeout=2), dep
+        )
 
-    response = await read_item(
-        LinkRequest(url="https://example.test/login"),
-        dep,
-    )
-
-    assert response.status == "ok"
-    assert response.solution.user_agent == "UnitTestBrowser/1.0"
+    assert exc.value.status_code == HTTPStatus.REQUEST_TIMEOUT

tests/owui_test.py

@@ -89,7 +89,7 @@ def fake_dep(*, html: str = ARTICLE_HTML) -> BrowserDepClass:
             raise PlaywrightTimeoutError(message)
 
     page.wait_for_load_state.side_effect = wait_for_load_state
-    return BrowserDepClass(page=page, solver=AsyncMock(), context=AsyncMock())
+    return BrowserDepClass(page=page, context=AsyncMock())
 
 
 @pytest.mark.asyncio