From d0b013029cfc96ff719982da8bfb9601f9d76747 Mon Sep 17 00:00:00 2001 From: ThePhaseless Date: Tue, 18 Aug 2026 03:41:58 +0200 Subject: [PATCH] fix: load the page the challenge was hiding, and stop trusting a stray box Nothing waited for the destination once the interstitial let go: page_html stayed unset and the networkidle wait only ran on the branch that never saw a challenge, so a challenge clearing on its own returned whatever had loaded by then. Wait for it on both branches. An exhausted budget produced timeout=0, which Playwright reads as no timeout at all, turning every remaining wait unbounded exactly when it should fail fast. Floor it instead. Scroll the widget into view before measuring it, since bounding_box reports viewport coordinates and an off-screen widget was clicked at a point that hit nothing, and reject containers taller than a checkbox row so a full-page wrapper cannot pass for one - both reported success while clicking blank space. Drop max_attempts, which nothing reads now that the solver is gone, and refresh AGENTS.md, which still described camoufox and a solver in the dependency. Co-Authored-By: Claude Opus 5 --- AGENTS.md | 10 +++++----- src/consts.py | 5 ----- src/endpoints.py | 21 +++++++++++++++------ tests/main_test.py | 16 +++++++++++++--- 4 files changed, 33 insertions(+), 19 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b3c0c61..adcdb92 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,14 +2,14 @@ ## Project overview -- FastAPI service that mimics FlareSolverr-style API for bypassing anti-bot pages using Camoufox + Playwright captcha solver. +- FastAPI service that mimics FlareSolverr-style API for bypassing anti-bot pages using invisible_playwright. - Entry point: main app in main.py; routes and request flow defined in src/endpoints.py and src/models.py. -- Browser lifecycle is owned by get_camoufox() in src/utils.py, which yields a page, solver, and context for each request. +- Browser lifecycle is owned by get_browser() in src/utils.py, which yields a page and context for each request. ## Architecture and data flow -- Request flow: POST /v1 -> read_item() -> browser.goto() -> wait for load states -> detect challenge title -> solve captcha -> return LinkResponse. -- Challenge detection uses CHALLENGE_TITLES from src/consts.py; update this map when adding providers. +- Request flow: POST /v1 -> read_item() -> page.goto() -> wait for load states -> detect the interstitial -> click its checkbox until it clears -> return LinkResponse. +- Challenge detection uses detect_cloudflare_challenge() from playwright_captcha; the challenge is over when its markup goes, not when a solver says so. - Health check hits /v1 internally with and fails if status is not OK. - Logging: LogRequest middleware logs only POST /v1 timing and outcome; other paths pass through. @@ -28,7 +28,7 @@ - Local run: uv sync && uv run main.py - Init mode: uv run main.py --init (pre-warms health check via browser setup) -- Tests: uv sync --group test && uv run pytest --retries 3 +- Tests: uv sync --group test && uv run pytest --retries 5 - Docker troubleshooting: docker build --target test . ## Tests and external dependencies diff --git a/src/consts.py b/src/consts.py index eb30c7c..d2f2a34 100644 --- a/src/consts.py +++ b/src/consts.py @@ -1,5 +1,4 @@ import logging -import sys from pydantic_settings import BaseSettings, SettingsConfigDict @@ -10,8 +9,6 @@ class Settings(BaseSettings): log_level: str = "INFO" version: str = "unknown" - max_attempts: int = sys.maxsize - proxy_server: str | None = None proxy_username: str | None = None proxy_password: str | None = None @@ -30,8 +27,6 @@ settings = Settings() LOG_LEVEL = logging.getLevelNamesMapping()[settings.log_level.upper()] VERSION = settings.version.removeprefix("v") -MAX_ATTEMPTS = settings.max_attempts - PROXY_SERVER = settings.proxy_server PROXY_USERNAME = settings.proxy_username PROXY_PASSWORD = settings.proxy_password diff --git a/src/endpoints.py b/src/endpoints.py index 6ba329d..5468b87 100644 --- a/src/endpoints.py +++ b/src/endpoints.py @@ -123,10 +123,10 @@ async def _navigate_and_solve( ) -> tuple[bool, str | None, object, HTTPStatus]: """Navigate to the URL, then solve a challenge or wait for network idle.""" page_html: str | None = None - page_request = await dep.page.goto(request.url, timeout=timer.remaining() * 1000) + page_request = await dep.page.goto(request.url, timeout=_remaining_ms(timer)) status = page_request.status if page_request else HTTPStatus.OK await dep.page.wait_for_load_state( - state="domcontentloaded", timeout=timer.remaining() * 1000 + state="domcontentloaded", timeout=_remaining_ms(timer) ) if not await detect_cloudflare_challenge(dep.page, "interstitial"): @@ -135,10 +135,19 @@ async def _navigate_and_solve( return False, page_html, page_request, status await _solve_challenge(dep, timer) + await _wait_for_networkidle(dep, timer) status = HTTPStatus.OK return True, page_html, page_request, status +MIN_WAIT_MS = 1.0 + + +def _remaining_ms(timer: TimeoutTimer) -> float: + """Milliseconds left, never 0 - Playwright reads that as no timeout at all.""" + return max(MIN_WAIT_MS, timer.remaining() * 1000) + + CHALLENGE_POLL_INTERVAL = 0.25 CHALLENGE_CLICK_SETTLE = 1.5 CHECKBOX_CLICK_COOLDOWN = 4 @@ -150,6 +159,7 @@ TURNSTILE_INPUT = 'input[name="cf-turnstile-response"]' WIDGET_ANCESTOR_DEPTHS = (1, 2, 3, 4) WIDGET_MIN_WIDTH = 40 WIDGET_MIN_HEIGHT = 20 +WIDGET_MAX_HEIGHT = 120 async def _challenge_widget_box(page: Page) -> FloatRect | None: @@ -159,11 +169,12 @@ async def _challenge_widget_box(page: Page) -> FloatRect | None: with suppress(PlaywrightError, PlaywrightTimeoutError): if await widget.count() == 0: continue + await widget.first.scroll_into_view_if_needed(timeout=BOX_READ_TIMEOUT) box = await widget.first.bounding_box(timeout=BOX_READ_TIMEOUT) if ( box and box["width"] > WIDGET_MIN_WIDTH - and box["height"] > WIDGET_MIN_HEIGHT + and WIDGET_MIN_HEIGHT < box["height"] < WIDGET_MAX_HEIGHT ): return box return None @@ -235,9 +246,7 @@ async def _solve_challenge(dep: BrowserDep, timer: TimeoutTimer) -> None: async def _wait_for_networkidle(dep: BrowserDep, timer: TimeoutTimer) -> None: """Wait for network idle, tolerating post-DOM-load stalls.""" try: - await dep.page.wait_for_load_state( - "networkidle", timeout=timer.remaining() * 1000 - ) + await dep.page.wait_for_load_state("networkidle", timeout=_remaining_ms(timer)) except PlaywrightTimeoutError: logger.info( "networkidle timed out after domcontentloaded; continuing with loaded page" diff --git a/tests/main_test.py b/tests/main_test.py index d8218a9..553305f 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -13,9 +13,9 @@ from playwright_captcha.solvers.click.cloudflare.utils.detection import ( from starlette.testclient import TestClient from main import app -from src.endpoints import read_item +from src.endpoints import _remaining_ms, read_item from src.models import LinkRequest -from src.utils import BrowserDepClass +from src.utils import BrowserDepClass, TimeoutTimer client = TestClient(app) @@ -65,7 +65,8 @@ def test_bypass(website: str): def test_json_api(): - """JSON APIs must return 200, not crash on the UA evaluate. + """ + JSON APIs must return 200, not crash on the UA evaluate. Firefox renders application/json in a built-in viewer whose CSP blocks Playwright's eval-based evaluate() (issue #394). The browser must be @@ -171,6 +172,7 @@ def fake_dep( def locator(selector: str) -> MagicMock: handle = MagicMock() handle.count = AsyncMock(side_effect=lambda: count_for(selector)) + handle.first.scroll_into_view_if_needed = AsyncMock(return_value=None) handle.first.bounding_box = AsyncMock(return_value=widget_box) handle.first.input_value = AsyncMock(return_value="") return handle @@ -216,6 +218,14 @@ async def test_domcontentloaded_timeout_returns_408(): assert exc.value.status_code == HTTPStatus.REQUEST_TIMEOUT +def test_exhausted_budget_never_disables_playwright_timeouts(): + """Playwright reads timeout=0 as no timeout at all, so the floor must hold.""" + spent = TimeoutTimer(duration=0) + + assert spent.remaining() == 0 + assert _remaining_ms(spent) > 0 + + @pytest.mark.asyncio async def test_missing_user_agent_header_is_not_a_500(): """A request without a user-agent header degrades to empty, never a 500 (#394)."""