diff --git a/AGENTS.md b/AGENTS.md index b3c0c61..adcdb92 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,14 +2,14 @@ ## Project overview -- FastAPI service that mimics FlareSolverr-style API for bypassing anti-bot pages using Camoufox + Playwright captcha solver. +- FastAPI service that mimics FlareSolverr-style API for bypassing anti-bot pages using invisible_playwright. - Entry point: main app in main.py; routes and request flow defined in src/endpoints.py and src/models.py. -- Browser lifecycle is owned by get_camoufox() in src/utils.py, which yields a page, solver, and context for each request. +- Browser lifecycle is owned by get_browser() in src/utils.py, which yields a page and context for each request. ## Architecture and data flow -- Request flow: POST /v1 -> read_item() -> browser.goto() -> wait for load states -> detect challenge title -> solve captcha -> return LinkResponse. -- Challenge detection uses CHALLENGE_TITLES from src/consts.py; update this map when adding providers. +- Request flow: POST /v1 -> read_item() -> page.goto() -> wait for load states -> detect the interstitial -> click its checkbox until it clears -> return LinkResponse. +- Challenge detection uses detect_cloudflare_challenge() from playwright_captcha; the challenge is over when its markup goes, not when a solver says so. - Health check hits /v1 internally with and fails if status is not OK. - Logging: LogRequest middleware logs only POST /v1 timing and outcome; other paths pass through. @@ -28,7 +28,7 @@ - Local run: uv sync && uv run main.py - Init mode: uv run main.py --init (pre-warms health check via browser setup) -- Tests: uv sync --group test && uv run pytest --retries 3 +- Tests: uv sync --group test && uv run pytest --retries 5 - Docker troubleshooting: docker build --target test . ## Tests and external dependencies diff --git a/src/consts.py b/src/consts.py index eb30c7c..d2f2a34 100644 --- a/src/consts.py +++ b/src/consts.py @@ -1,5 +1,4 @@ import logging -import sys from pydantic_settings import BaseSettings, SettingsConfigDict @@ -10,8 +9,6 @@ class Settings(BaseSettings): log_level: str = "INFO" version: str = "unknown" - max_attempts: int = sys.maxsize - proxy_server: str | None = None proxy_username: str | None = None proxy_password: str | None = None @@ -30,8 +27,6 @@ settings = Settings() LOG_LEVEL = logging.getLevelNamesMapping()[settings.log_level.upper()] VERSION = settings.version.removeprefix("v") -MAX_ATTEMPTS = settings.max_attempts - PROXY_SERVER = settings.proxy_server PROXY_USERNAME = settings.proxy_username PROXY_PASSWORD = settings.proxy_password diff --git a/src/endpoints.py b/src/endpoints.py index 6ba329d..5468b87 100644 --- a/src/endpoints.py +++ b/src/endpoints.py @@ -123,10 +123,10 @@ async def _navigate_and_solve( ) -> tuple[bool, str | None, object, HTTPStatus]: """Navigate to the URL, then solve a challenge or wait for network idle.""" page_html: str | None = None - page_request = await dep.page.goto(request.url, timeout=timer.remaining() * 1000) + page_request = await dep.page.goto(request.url, timeout=_remaining_ms(timer)) status = page_request.status if page_request else HTTPStatus.OK await dep.page.wait_for_load_state( - state="domcontentloaded", timeout=timer.remaining() * 1000 + state="domcontentloaded", timeout=_remaining_ms(timer) ) if not await detect_cloudflare_challenge(dep.page, "interstitial"): @@ -135,10 +135,19 @@ async def _navigate_and_solve( return False, page_html, page_request, status await _solve_challenge(dep, timer) + await _wait_for_networkidle(dep, timer) status = HTTPStatus.OK return True, page_html, page_request, status +MIN_WAIT_MS = 1.0 + + +def _remaining_ms(timer: TimeoutTimer) -> float: + """Milliseconds left, never 0 - Playwright reads that as no timeout at all.""" + return max(MIN_WAIT_MS, timer.remaining() * 1000) + + CHALLENGE_POLL_INTERVAL = 0.25 CHALLENGE_CLICK_SETTLE = 1.5 CHECKBOX_CLICK_COOLDOWN = 4 @@ -150,6 +159,7 @@ TURNSTILE_INPUT = 'input[name="cf-turnstile-response"]' WIDGET_ANCESTOR_DEPTHS = (1, 2, 3, 4) WIDGET_MIN_WIDTH = 40 WIDGET_MIN_HEIGHT = 20 +WIDGET_MAX_HEIGHT = 120 async def _challenge_widget_box(page: Page) -> FloatRect | None: @@ -159,11 +169,12 @@ async def _challenge_widget_box(page: Page) -> FloatRect | None: with suppress(PlaywrightError, PlaywrightTimeoutError): if await widget.count() == 0: continue + await widget.first.scroll_into_view_if_needed(timeout=BOX_READ_TIMEOUT) box = await widget.first.bounding_box(timeout=BOX_READ_TIMEOUT) if ( box and box["width"] > WIDGET_MIN_WIDTH - and box["height"] > WIDGET_MIN_HEIGHT + and WIDGET_MIN_HEIGHT < box["height"] < WIDGET_MAX_HEIGHT ): return box return None @@ -235,9 +246,7 @@ async def _solve_challenge(dep: BrowserDep, timer: TimeoutTimer) -> None: async def _wait_for_networkidle(dep: BrowserDep, timer: TimeoutTimer) -> None: """Wait for network idle, tolerating post-DOM-load stalls.""" try: - await dep.page.wait_for_load_state( - "networkidle", timeout=timer.remaining() * 1000 - ) + await dep.page.wait_for_load_state("networkidle", timeout=_remaining_ms(timer)) except PlaywrightTimeoutError: logger.info( "networkidle timed out after domcontentloaded; continuing with loaded page" diff --git a/tests/main_test.py b/tests/main_test.py index d8218a9..553305f 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -13,9 +13,9 @@ from playwright_captcha.solvers.click.cloudflare.utils.detection import ( from starlette.testclient import TestClient from main import app -from src.endpoints import read_item +from src.endpoints import _remaining_ms, read_item from src.models import LinkRequest -from src.utils import BrowserDepClass +from src.utils import BrowserDepClass, TimeoutTimer client = TestClient(app) @@ -65,7 +65,8 @@ def test_bypass(website: str): def test_json_api(): - """JSON APIs must return 200, not crash on the UA evaluate. + """ + JSON APIs must return 200, not crash on the UA evaluate. Firefox renders application/json in a built-in viewer whose CSP blocks Playwright's eval-based evaluate() (issue #394). The browser must be @@ -171,6 +172,7 @@ def fake_dep( def locator(selector: str) -> MagicMock: handle = MagicMock() handle.count = AsyncMock(side_effect=lambda: count_for(selector)) + handle.first.scroll_into_view_if_needed = AsyncMock(return_value=None) handle.first.bounding_box = AsyncMock(return_value=widget_box) handle.first.input_value = AsyncMock(return_value="") return handle @@ -216,6 +218,14 @@ async def test_domcontentloaded_timeout_returns_408(): assert exc.value.status_code == HTTPStatus.REQUEST_TIMEOUT +def test_exhausted_budget_never_disables_playwright_timeouts(): + """Playwright reads timeout=0 as no timeout at all, so the floor must hold.""" + spent = TimeoutTimer(duration=0) + + assert spent.remaining() == 0 + assert _remaining_ms(spent) > 0 + + @pytest.mark.asyncio async def test_missing_user_agent_header_is_not_a_500(): """A request without a user-agent header degrades to empty, never a 500 (#394)."""