From 691aaa6f3f370653f29d7e77fdc763a127741446 Mon Sep 17 00:00:00 2001 From: ThePhaseless Date: Mon, 17 Aug 2026 11:06:09 +0200 Subject: [PATCH] refactor: strip the challenge-solver changes to the minimum Removes every explanatory comment added by this branch, inlines the browser prefs rather than holding them in a module constant, folds _cloudflare_frame into its only caller, and cuts the added docstrings to one line each. No behaviour change: 13 unit tests pass, and removing the checked-box guard still fails its test. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TDMac4vGGcBhoUB5V6bvFK --- src/consts.py | 2 -- src/endpoints.py | 52 +++++++--------------------------------------- src/utils.py | 22 +++++--------------- tests/main_test.py | 42 ++++--------------------------------- 4 files changed, 17 insertions(+), 101 deletions(-) diff --git a/src/consts.py b/src/consts.py index 04f301e..fa3329f 100644 --- a/src/consts.py +++ b/src/consts.py @@ -9,8 +9,6 @@ class Settings(BaseSettings): log_level: str = "INFO" version: str = "unknown" - # sys.maxsize burned a request's whole max_timeout on ~1300 failed solver - # attempts before returning 408. An unreachable widget stays unreachable. max_attempts: int = 5 proxy_server: str | None = None diff --git a/src/endpoints.py b/src/endpoints.py index e65e997..d9aef3f 100644 --- a/src/endpoints.py +++ b/src/endpoints.py @@ -28,20 +28,13 @@ router = APIRouter() BrowserDep = Annotated[BrowserDepClass, Depends(get_browser)] -# Markup only an unsolved challenge has. chl_page is required: the bare -# challenge-platform path also matches the jsd beacon served on cleared pages, -# and the widget iframe outlives the challenge (a cleared nowsecure.nl has two). CHALLENGE_MARKERS = ( 'script[src*="/cdn-cgi/challenge-platform/"][src*="chl_page"], ' "#challenge-error-text, #challenge-running, #challenge-stage" ) - -# 30px in from the widget's left edge is the middle of the box Cloudflare draws. CF_WIDGET_HOST = "challenges.cloudflare.com" CHECKBOX_SELECTOR = 'input[type="checkbox"]' CHECKBOX_OFFSET_X = 30 - -# Verification takes 5-15s; pressing over the top of it restarts the cycle. CHALLENGE_POLL_SECONDS = 1.0 PRESS_INTERVAL_SECONDS = 12.0 @@ -156,13 +149,7 @@ async def _navigate_and_solve( async def _solve_challenge(dep: BrowserDep, timer: TimeoutTimer) -> None: - """ - Clear a Cloudflare challenge, whether or not it needs the checkbox pressed. - - playwright-captcha's solver is not used: it clicks the invisible input, so - the click reports success while `checked` never flips, and it judges the - result on networkidle, which returns while Cloudflare is still verifying. - """ + """Clear the challenge, pressing the checkbox whenever one is offered.""" logger.info("Challenge detected, attempting to solve...") last_press = -PRESS_INTERVAL_SECONDS while timer.remaining() > 0: @@ -182,34 +169,20 @@ async def _solve_challenge(dep: BrowserDep, timer: TimeoutTimer) -> None: raise TimeoutError(message) -def _cloudflare_frame(page: Page) -> object | None: - """Find the turnstile widget's frame; None while Cloudflare is between states.""" - for frame in page.frames: - if CF_WIDGET_HOST in frame.url and not frame.is_detached(): - return frame - return None - - async def _press_checkbox(page: Page) -> bool: - """ - Press the widget's visible pixels. True when a press actually happened. - - Cloudflare cycles through a state where the frame holds no input, and an - already-checked box means a press is being verified -- pressing again - restarts that. The point comes from the iframe, not the input, because the - input is invisible beneath a styled overlay. - """ - frame = _cloudflare_frame(page) + """Press the widget's visible pixels; True when a press happened.""" + frame = next( + (f for f in page.frames if CF_WIDGET_HOST in f.url and not f.is_detached()), + None, + ) if frame is None: return False try: checkbox = frame.locator(CHECKBOX_SELECTOR) if not await checkbox.count() or await checkbox.first.is_checked(): return False - element = await frame.frame_element() - box = await element.bounding_box() + box = await (await frame.frame_element()).bounding_box() except Exception: - # The widget is mid-swap; try again on the next poll. return False if not box: return False @@ -217,7 +190,6 @@ async def _press_checkbox(page: Page) -> bool: x = box["x"] + CHECKBOX_OFFSET_X y = box["y"] + box["height"] / 2 try: - # A cursor that teleports onto the target is itself a signal. await page.mouse.move(x - 180, y - 120) await sleep(0.3) await page.mouse.move(x - 45, y - 20, steps=18) @@ -235,18 +207,10 @@ async def _press_checkbox(page: Page) -> bool: async def _challenge_visible(page: Page) -> bool: - """ - Report whether an unsolved challenge is still on the page. - - detect_cloudflare_challenge() matches the jsd beacon served on cleared pages - too, so on its own it never reports success. - """ + """Report whether an unsolved challenge is still on the page.""" try: return await page.locator(CHALLENGE_MARKERS).count() > 0 except Exception: - # A navigation tore down the context mid-check; that only happens once - # Cloudflare has moved us on. - logger.debug("Challenge lookup interrupted by a navigation") return False diff --git a/src/utils.py b/src/utils.py index 81b486f..54a7c98 100644 --- a/src/utils.py +++ b/src/utils.py @@ -35,19 +35,6 @@ if len(logger.handlers) == 0: logger.addHandler(logging.StreamHandler()) -# The widget's iframe carries allow="cross-origin-isolated", which Firefox -# honours by moving it into an isolated process where Juggler sees no docShell. -# With the two policies on, that frame never appears in page.frames at all; -# turning them off (as camoufox's disable_coop=True did in v2.1.0) brings it -# back. devtools.jsonview.enabled keeps page.evaluate() alive on JSON responses, -# whose viewer CSP blocks eval (#394). -BROWSER_PREFS = { - "devtools.jsonview.enabled": False, - "browser.tabs.remote.useCrossOriginOpenerPolicy": False, - "browser.tabs.remote.useCrossOriginEmbedderPolicy": False, -} - - class TimeoutTimer(BaseModel): duration: int # in seconds start_time: float = Field(default_factory=time.perf_counter) @@ -109,16 +96,17 @@ async def get_browser( proxy=proxy_config, humanize=True, locale=BROWSER_LOCALE or "auto", - extra_prefs=BROWSER_PREFS, + extra_prefs={ + "devtools.jsonview.enabled": False, + "browser.tabs.remote.useCrossOriginOpenerPolicy": False, + "browser.tabs.remote.useCrossOriginEmbedderPolicy": False, + }, ) as browser_raw: # InvisiblePlaywright yields a Browser instance browser = cast("Browser", browser_raw) context = await browser.new_context() page = await context.new_page() async with ClickSolver( - # PATCHRIGHT injects the shadow-root unlock over CDP, which Firefox - # has no session for, leaving Cloudflare's closed shadow root sealed - # and the challenge iframe invisible to every locator. framework=FrameworkType.PLAYWRIGHT, page=page, max_attempts=MAX_ATTEMPTS, diff --git a/tests/main_test.py b/tests/main_test.py index e30039d..b4274cc 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -18,32 +18,15 @@ from src.utils import BrowserDepClass client = TestClient(app) -# Real Firefox advertises 16 cipher suites; Playwright's HTTP client advertised -# 52. A small margin absorbs Firefox version drift without letting 52 through. FIREFOX_CIPHER_SUITE_CEILING = 20 -# The turnstile iframe's geometry, as measured on a real challenge page. WIDGET_BOX = {"x": 512.0, "y": 304.0, "width": 300.0, "height": 65.0} -# Sites Byparr clears from any network, datacenter ranges included. These carry -# the hard assertion: if the bypass breaks, one of these goes red. test_websites = [ - # Purpose-built Cloudflare challenge target. Serves a real interstitial and - # hands back a cf_clearance cookie once it is passed, so a pass here means - # the challenge was solved rather than never presented. "https://nowsecure.nl/", 'https://www.yggtorrent.top/engine/search?do=search&order=desc&sort=publish_date&name="UNESCAPED"+"DOUBLEQUOTES"&category=2145', ] -# Cloudflare declines the press on these and re-serves a fresh unchecked box. -# It is the browser stack, not the IP: from one datacenter IP, v2.1.0's camoufox -# cleared ext.to, speed.cd and extratorrent.st in under 20s each, where no -# configuration of invisible_playwright clears any of them. Neither the JS -# fingerprint nor the TLS handshake explains it -- reproducing camoufox's JA4 -# exactly (t13d1717h2_5b57614c22b0) changed nothing. -# -# They run rather than skip, so a regression still shows and a pass is an xpass, -# but a verdict we cannot yet explain must not turn the build red. datacenter_hostile_websites = [ "https://ext.to/", # "https://www.ygg.re/", @@ -123,14 +106,7 @@ def test_json_api(): def test_tls_handshake_looks_like_firefox(): - """ - The handshake must be Firefox's, not the HTTP client's (#398). - - route.fetch() re-issued navigations through Playwright's own HTTP stack, so - the ClientHello advertised 52 cipher suites where Firefox offers 16 -- a - fingerprint no amount of header spoofing hides. Unlike a Cloudflare verdict - this is deterministic, so it pins the regression that motivated this branch. - """ + """The handshake must be Firefox's, not the HTTP client's (#398).""" url = "https://www.howsmyssl.com/a/check" if httpx2.get(url).status_code >= HTTPStatus.INTERNAL_SERVER_ERROR: pytest.skip("Skipping TLS check - howsmyssl is down") @@ -147,8 +123,6 @@ def test_tls_handshake_looks_like_firefox(): ) suites = len(report["given_cipher_suites"]) - # Firefox offers 16; Playwright's client offered 52. Anything in between - # means the navigation is no longer going through the browser. assert suites <= FIREFOX_CIPHER_SUITE_CEILING, ( f"{suites} cipher suites offered - the handshake is not Firefox's" ) @@ -204,7 +178,7 @@ def test_max_timeout_normalization(payload: dict, expected: int): def fake_cloudflare_frame(*, checked: bool) -> MagicMock: - """Build a turnstile widget frame offering one checkbox in the given state.""" + """Build a widget frame offering one checkbox in the given state.""" frame = MagicMock() frame.url = ( "https://challenges.cloudflare.com/cdn-cgi/challenge-platform/h/b/turnstile" @@ -229,15 +203,7 @@ def fake_dep( marker_counts: list[int] | None = None, checkbox: str | None = None, ) -> BrowserDepClass: - """ - Build a browser dependency triple backed by mocks. - - `challenged` makes the detector report a Cloudflare challenge. - `marker_counts` drives the "is it still up?" check that runs on each poll: - one entry per look, the last one repeating forever. - `checkbox` puts a widget frame on the page with the box "checked" or - "unchecked"; without it the page carries no widget at all. - """ + """Build a browser dependency pair backed by mocks.""" page = AsyncMock() page.url = "https://example.test/login" page.goto.return_value = MagicMock( @@ -252,7 +218,7 @@ def fake_dep( remaining = list(marker_counts or []) def count_for(selector: str) -> int: - """Answer the marker check from the script, everything else from `challenged`.""" + """Answer the marker check from the script, else from `challenged`.""" if selector != CHALLENGE_MARKERS or not remaining: return 1 if challenged else 0 return remaining.pop(0) if len(remaining) > 1 else remaining[0]