From e0b1efa560eb55e72a4d7ba161abc41d29046188 Mon Sep 17 00:00:00 2001 From: ThePhaseless Date: Sun, 16 Aug 2026 21:16:20 +0200 Subject: [PATCH] style: trim the challenge-solver comments to what is load-bearing Cuts ~100 lines of commentary that restated the diff or recorded dead investigation, and merges _press_point back into _press_checkbox now that the checked guard is one condition rather than the extra return that tripped the too-many-returns lint. Corrects the COOP/COEP note, which claimed the pair changed no outcome. Without those prefs the widget's iframe never appears in page.frames at all: measured on ext.to, eight presses land with them and none without. No behaviour change. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TDMac4vGGcBhoUB5V6bvFK --- src/consts.py | 7 +-- src/endpoints.py | 109 ++++++++++++++------------------------------- src/utils.py | 35 ++++----------- tests/main_test.py | 60 +++++-------------------- 4 files changed, 57 insertions(+), 154 deletions(-) diff --git a/src/consts.py b/src/consts.py index b77ffe8..04f301e 100644 --- a/src/consts.py +++ b/src/consts.py @@ -9,11 +9,8 @@ class Settings(BaseSettings): log_level: str = "INFO" version: str = "unknown" - # The solver retries whenever it cannot reach the challenge widget, and - # that failure is usually structural rather than transient -- an - # unreachable widget stays unreachable. sys.maxsize meant a single request - # burned its whole max_timeout on ~1300 identical failed attempts before - # reporting a 408. Give it a handful of tries, then let the caller know. + # sys.maxsize burned a request's whole max_timeout on ~1300 failed solver + # attempts before returning 408. An unreachable widget stays unreachable. max_attempts: int = 5 proxy_server: str | None = None diff --git a/src/endpoints.py b/src/endpoints.py index 0a8a3e7..e65e997 100644 --- a/src/endpoints.py +++ b/src/endpoints.py @@ -28,29 +28,20 @@ router = APIRouter() BrowserDep = Annotated[BrowserDepClass, Depends(get_browser)] -# Markup only an unsolved challenge has. Two near misses to avoid: -# -# script[src*="/cdn-cgi/challenge-platform/"] on its own also matches the jsd -# bot-scoring beacon Cloudflare serves from that path on ordinary pages, so it -# has to be narrowed to the challenge orchestrator (chl_page). -# -# iframe[src*="challenges.cloudflare.com"] looks like the widget but outlives -# it: a cleared nowsecure.nl carries two of them with no challenge in sight. +# Markup only an unsolved challenge has. chl_page is required: the bare +# challenge-platform path also matches the jsd beacon served on cleared pages, +# and the widget iframe outlives the challenge (a cleared nowsecure.nl has two). CHALLENGE_MARKERS = ( 'script[src*="/cdn-cgi/challenge-platform/"][src*="chl_page"], ' "#challenge-error-text, #challenge-running, #challenge-stage" ) -# The widget lives in an iframe served from here; the checkbox is an invisible -# input inside it, so it is pressed by position rather than by locator. 30px in -# from the widget's left edge is the middle of the box Cloudflare draws. +# 30px in from the widget's left edge is the middle of the box Cloudflare draws. CF_WIDGET_HOST = "challenges.cloudflare.com" CHECKBOX_SELECTOR = 'input[type="checkbox"]' CHECKBOX_OFFSET_X = 30 -# How often to look, and how long to leave a press alone before trying again. -# Verification takes 5-15s, and pressing over the top of it just restarts the -# cycle. +# Verification takes 5-15s; pressing over the top of it restarts the cycle. CHALLENGE_POLL_SECONDS = 1.0 PRESS_INTERVAL_SECONDS = 12.0 @@ -166,19 +157,11 @@ async def _navigate_and_solve( async def _solve_challenge(dep: BrowserDep, timer: TimeoutTimer) -> None: """ - Attempt to solve a detected Cloudflare interstitial challenge. + Clear a Cloudflare challenge, whether or not it needs the checkbox pressed. - Handles both shapes Cloudflare serves: the non-interactive challenge, which - clears itself given a few seconds, and the interactive one, which needs the - checkbox pressed. Both are covered by the same loop -- watch for the - challenge markup to disappear, and press whenever a checkbox is on offer. - - playwright-captcha's solver is deliberately not used here. It clicks the - checkbox input directly, and that input is invisible, so the click reports - success while `checked` never flips. It also judges the result by waiting - for networkidle, which returned 9ms after the click while Cloudflare was - still verifying, so it reported failure on challenges that were about to - pass. + playwright-captcha's solver is not used: it clicks the invisible input, so + the click reports success while `checked` never flips, and it judges the + result on networkidle, which returns while Cloudflare is still verifying. """ logger.info("Challenge detected, attempting to solve...") last_press = -PRESS_INTERVAL_SECONDS @@ -207,55 +190,34 @@ def _cloudflare_frame(page: Page) -> object | None: return None -async def _press_point(page: Page) -> tuple[float, float] | None: - """ - Where to press, or None when there is nothing to press right now. - - Cloudflare cycles between "checking if you are human", where the widget - frame holds no input at all, and the state where the checkbox is offered, - so the input has to exist before reaching for the mouse. A box that is - already checked is skipped too: a press has landed and is being verified, - and pressing over the top restarts that verification -- which is how ext.to - and speed.cd stayed on "performing security verification" for a full 300s - budget while being pressed a dozen times. - - The point is measured from the iframe rather than the input, because the - input is invisible: it sits under a styled overlay, so Playwright reports a - successful click on it while `checked` never flips. That is why - playwright-captcha's own click has never solved one of these. - """ - frame = _cloudflare_frame(page) - if frame is None: - return None - try: - checkbox = frame.locator(CHECKBOX_SELECTOR) - if not await checkbox.count() or await checkbox.first.is_checked(): - return None - element = await frame.frame_element() - box = await element.bounding_box() - except Exception: - # The widget is mid-swap; try again on the next poll. - return None - if not box: - return None - return box["x"] + CHECKBOX_OFFSET_X, box["y"] + box["height"] / 2 - - async def _press_checkbox(page: Page) -> bool: """ Press the widget's visible pixels. True when a press actually happened. - Measured against extratorrent.st and ext.to on a residential connection, - this clears the challenge and returns a cf_clearance cookie, where clicking - the input never did. + Cloudflare cycles through a state where the frame holds no input, and an + already-checked box means a press is being verified -- pressing again + restarts that. The point comes from the iframe, not the input, because the + input is invisible beneath a styled overlay. """ - point = await _press_point(page) - if point is None: + frame = _cloudflare_frame(page) + if frame is None: return False - x, y = point try: - # Approach before pressing: a cursor that teleports onto the target is - # itself a signal. + checkbox = frame.locator(CHECKBOX_SELECTOR) + if not await checkbox.count() or await checkbox.first.is_checked(): + return False + element = await frame.frame_element() + box = await element.bounding_box() + except Exception: + # The widget is mid-swap; try again on the next poll. + return False + if not box: + return False + + x = box["x"] + CHECKBOX_OFFSET_X + y = box["y"] + box["height"] / 2 + try: + # A cursor that teleports onto the target is itself a signal. await page.mouse.move(x - 180, y - 120) await sleep(0.3) await page.mouse.move(x - 45, y - 20, steps=18) @@ -276,17 +238,14 @@ async def _challenge_visible(page: Page) -> bool: """ Report whether an unsolved challenge is still on the page. - Cloudflare serves two different scripts from /cdn-cgi/challenge-platform/: - the challenge orchestrator on an interstitial, and the jsd bot-scoring - beacon on ordinary pages once a visitor is cleared. detect_cloudflare_ - challenge() matches both, so on its own it never reports success. Match the - orchestrator and the widget instead. + detect_cloudflare_challenge() matches the jsd beacon served on cleared pages + too, so on its own it never reports success. """ try: return await page.locator(CHALLENGE_MARKERS).count() > 0 except Exception: - # A navigation tore down the execution context mid-check, which only - # happens once Cloudflare has moved us on. + # A navigation tore down the context mid-check; that only happens once + # Cloudflare has moved us on. logger.debug("Challenge lookup interrupted by a navigation") return False diff --git a/src/utils.py b/src/utils.py index 054b7c3..81b486f 100644 --- a/src/utils.py +++ b/src/utils.py @@ -35,26 +35,12 @@ if len(logger.handlers) == 0: logger.addHandler(logging.StreamHandler()) -# Cloudflare embeds its challenge widget in an iframe carrying -# allow="cross-origin-isolated". Firefox honours that by moving the iframe into -# a cross-origin-isolated content process, where Juggler sees a frame with no -# docShell and no URL, so content_frame() raises "Permission denied to access -# property docShell on cross-origin object" and the solver never reaches the -# checkbox. -# -# Turning the two policies off (as upstream Playwright's Firefox does, and as -# v2.1.0 did via camoufox's disable_coop=True) restores that access, and the -# solver then clicks the checkbox successfully. -# -# The COOP/COEP pair is not a demonstrated win: toggling it changed no outcome -# on any site measured, from either a datacenter or a residential address. It is -# kept for parity with v2.1.0, which clears the interactive challenge where this -# stack does not, because reaching the checkbox is a precondition for ever -# passing one. -# -# devtools.jsonview.enabled is load-bearing and must stay. Firefox renders -# application/json in a viewer whose CSP blocks eval, so page.evaluate() dies -# with "call to eval() blocked by CSP" and /v1 500s on every JSON API (#394). +# The widget's iframe carries allow="cross-origin-isolated", which Firefox +# honours by moving it into an isolated process where Juggler sees no docShell. +# With the two policies on, that frame never appears in page.frames at all; +# turning them off (as camoufox's disable_coop=True did in v2.1.0) brings it +# back. devtools.jsonview.enabled keeps page.evaluate() alive on JSON responses, +# whose viewer CSP blocks eval (#394). BROWSER_PREFS = { "devtools.jsonview.enabled": False, "browser.tabs.remote.useCrossOriginOpenerPolicy": False, @@ -130,12 +116,9 @@ async def get_browser( context = await browser.new_context() page = await context.new_page() async with ClickSolver( - # Not PATCHRIGHT: that path skips the unlockShadowRoot init script - # and injects it over CDP instead, which Firefox has no session for - # ("CDP session is only available in Chromium"). Cloudflare builds - # its widget inside a closed shadow root, so without that script - # nothing -- not the solver, not page.locator -- can see the - # challenge iframe, and every solve attempt fails outright. + # PATCHRIGHT injects the shadow-root unlock over CDP, which Firefox + # has no session for, leaving Cloudflare's closed shadow root sealed + # and the challenge iframe invisible to every locator. framework=FrameworkType.PLAYWRIGHT, page=page, max_attempts=MAX_ATTEMPTS, diff --git a/tests/main_test.py b/tests/main_test.py index c6c4dc9..e30039d 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -35,29 +35,15 @@ test_websites = [ 'https://www.yggtorrent.top/engine/search?do=search&order=desc&sort=publish_date&name="UNESCAPED"+"DOUBLEQUOTES"&category=2145', ] -# Cloudflare hands these its interactive checkbox challenge. The press lands on -# the widget's visible pixels and Cloudflare declines it, returning a fresh -# unchecked box indefinitely. +# Cloudflare declines the press on these and re-serves a fresh unchecked box. +# It is the browser stack, not the IP: from one datacenter IP, v2.1.0's camoufox +# cleared ext.to, speed.cd and extratorrent.st in under 20s each, where no +# configuration of invisible_playwright clears any of them. Neither the JS +# fingerprint nor the TLS handshake explains it -- reproducing camoufox's JA4 +# exactly (t13d1717h2_5b57614c22b0) changed nothing. # -# This is our browser stack, not the visitor's address. Measured 2026-08-16 from -# one datacenter IP within the same hour: byparr v2.1.0 -# (ghcr.io/thephaseless/byparr:2.1.0, camoufox) cleared ext.to in 18s, -# speed.cd/login in 20s and extratorrent.st in 19s, each returning cf_clearance. -# No configuration of the current invisible_playwright stack clears any of them: -# tested with and without new_context(), with and without the shadow-root init -# script, with and without the COOP/COEP prefs, and with both locator.click() -# and a pixel press. -# -# Two candidate explanations were measured and eliminated. The JS fingerprint is -# not it -- camoufox is the less coherent of the two (no WebGL at all, oscpu -# leaking Linux under a Windows UA) and passes anyway. The TLS handshake is not -# it either -- setting security.ssl3.ecdhe_ecdsa_aes_128_sha=True reproduces -# camoufox's JA4 byte for byte (t13d1717h2_5b57614c22b0_3cbfd9057e0d) and the -# challenge is still refused. -# -# They run rather than being skipped, so a real regression stays visible in the -# report and a pass is recorded as xpass, but a Cloudflare verdict we do not yet -# understand cannot turn the build red. +# They run rather than skip, so a regression still shows and a pass is an xpass, +# but a verdict we cannot yet explain must not turn the build red. datacenter_hostile_websites = [ "https://ext.to/", # "https://www.ygg.re/", @@ -99,10 +85,7 @@ def test_bypass(website: str): @pytest.mark.xfail( - reason=( - "Cloudflare's interactive challenge refuses the press on " - "invisible_playwright; v2.1.0's camoufox clears these from the same IP" - ), + reason="Cloudflare declines the press on invisible_playwright; camoufox clears it", strict=False, ) @pytest.mark.parametrize("website", datacenter_hostile_websites) @@ -329,14 +312,7 @@ async def test_domcontentloaded_timeout_returns_408(): @pytest.mark.asyncio async def test_challenge_that_clears_is_reported_as_success(): - """ - A challenge is over when its markup goes away, not when a solver says so. - - playwright-captcha judged its own click by waiting for networkidle, which - returns as soon as the network happens to be quiet -- 9ms after the click, - in practice -- while Cloudflare is still showing "verifying you are human", - and then reported failure on challenges that were about to pass. - """ + """A challenge is over when its markup goes, not when a solver says so.""" dep = fake_dep(challenged=True, marker_counts=[1, 0]) response = await read_item( @@ -349,13 +325,7 @@ async def test_challenge_that_clears_is_reported_as_success(): @pytest.mark.asyncio async def test_unchecked_box_is_pressed_on_the_widgets_visible_pixels(): - """ - The press must land on the widget, not on the input. - - The input is invisible -- it sits under a styled overlay -- so a click on it - reports success while `checked` never flips. Pressing the pixels Cloudflare - actually draws is what clears the challenge. - """ + """The press must land on the widget, not on the invisible input.""" dep = fake_dep(challenged=True, marker_counts=[1, 1, 0], checkbox="unchecked") await read_item(LinkRequest(url="https://example.test/login", max_timeout=5), dep) @@ -369,13 +339,7 @@ async def test_unchecked_box_is_pressed_on_the_widgets_visible_pixels(): @pytest.mark.asyncio async def test_checked_box_is_left_alone_while_cloudflare_verifies(): - """ - Pressing a box that is already checked restarts Cloudflare's verification. - - ext.to and speed.cd sat on "performing security verification" for a full - 300s budget while being pressed a dozen times, never getting far enough - into the check to finish it. - """ + """Pressing an already-checked box restarts Cloudflare's verification.""" dep = fake_dep(challenged=True, marker_counts=[1, 1, 0], checkbox="checked") await read_item(LinkRequest(url="https://example.test/login", max_timeout=5), dep)