add cache, pip, progress and fix bypass

Bump docker/metadata-action from 5.5.1 to 5.6.1 in /.github/workflows

.github/workflows/docker-publish.yml

@@ -30,6 +30,7 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: "3.12"
+          cache: "pip"
 
       - name: Set up Poetry
         run: pip install poetry
@@ -55,7 +56,13 @@ jobs:
           rm ./google-chrome-stable_current_amd64.deb
 
       - name: Run tests
-        run: poetry run pytest
+        run: poetry run pytest --retries 2 --show-progress
+
+      - name: Upload screenshots if tests fail
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          path: screenshots
 
   build:
     needs: test
@@ -99,7 +106,7 @@ jobs:
       # https://github.com/docker/metadata-action
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
+        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1
         with:
           tags: |
             type=ref,event=branch

Dockerfile

@@ -27,4 +27,5 @@ COPY pyproject.toml poetry.lock ./
 RUN poetry install
 
 COPY . .
-CMD [". .venv/bin/activate && python3 main.py"]
+HEALTHCHECK --interval=60s --timeout=30s --start-period=5s --retries=3 CMD [ "curl", "http://localhost:8191/health" ]
+CMD ["./cmd.sh"]

cmd.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+. .venv/bin/activate && python3 main.py

fix_nodriver.py

@@ -1,65 +0,0 @@
-# https://github.com/ultrafunkamsterdam/undetected-chromedriver/issues/1954
-# Fix for nodriver in .venv/lib/python3.11/site-packages/nodriver/core/browser.py
-from __future__ import annotations
-
-import logging
-import os
-from pathlib import Path
-from platform import python_version
-
-env_path = os.getenv("VIRTUAL_ENV")
-if env_path is None:
-    env_path = Path(os.__file__).parent.parent.parent.as_posix()
-python_version = python_version().split(".")[0:2]
-nodriver_path = Path(env_path + f"/lib/python{'.'.join(python_version)}/site-packages/nodriver/cdp/network.py")
-if not nodriver_path.exists():
-    msg = f"{nodriver_path} not found"
-    raise FileNotFoundError(msg)
-
-new_cookie_partition_key = """\
-        if isinstance(json, str):
-            return cls(top_level_site=json, has_cross_site_ancestor=False)
-        elif isinstance(json, dict):
-            return cls(
-                top_level_site=str(json["topLevelSite"]),
-                has_cross_site_ancestor=bool(json["hasCrossSiteAncestor"]),
-            )
-"""
-
-logger = logging.getLogger(__name__)
-handler = logging.StreamHandler()
-logger.addHandler(handler)
-logger.setLevel(logging.INFO)
-logger.info(f"Fixing nodriver in {nodriver_path}")
-# delete CookiePartitionKey declaration
-with nodriver_path.open("r+") as f:
-    lines = f.readlines()
-    found_def = False
-    found_body = False
-    i = -1
-    while i < len(lines):
-        i += 1
-        line = lines[i]
-        strip_line = line.strip("\n")
-        if not found_def and line.startswith("class CookiePartitionKey:"):
-            logger.info(f"Found line {i}: {strip_line}")
-            found_def = True
-            continue
-        if found_def:
-            if line.startswith("    def from_json"):
-                logger.info(f"Found line {i}: {strip_line}")
-                found_body = True
-                continue
-            if found_body:
-                if line.startswith(("\t\t", "        ")):
-                    logger.info(f"Removing line {i}: {strip_line}")
-                    lines.pop(i)
-                    i -= 1
-                    continue
-                else:
-                    lines = lines[:i] + [new_cookie_partition_key] + lines[i:]
-                    break
-
-
-with nodriver_path.open("w") as f:
-    f.writelines(lines)

main.py

@@ -30,9 +30,16 @@ def read_root():
 async def health_check():
     """Health check endpoint."""
     logger.info("Health check")
-    # browser: Chrome = await new_browser()
-    # browser.get("https://google.com")
-    # browser.stop()
+
+    health_check_request = read_item(
+        LinkRequest.model_construct(url="https://prowlarr.servarr.com/v1/ping")
+    )
+    if health_check_request.solution.status != 200:
+        raise HTTPException(
+            status_code=500,
+            detail="Health check failed",
+        )
+
     return {"status": "ok"}
 
 
@@ -45,37 +52,44 @@ def read_item(request: LinkRequest):
     response: LinkResponse
 
     # start_time = int(time.time() * 1000)
-    with SB(uc=True, locale_code="en", test=False, xvfb=True, ad_block=True) as sb:
-        sb: BaseCase
-        sb.uc_open_with_reconnect(request.url)
-        sb.uc_gui_click_captcha()
-        logger.info(f"Got webpage: {request.url}")
-        sb.save_screenshot("screenshot.png")
-        logger.info(f"Got webpage: {request.url}")
+    try:
+        with SB(uc=True, locale_code="en", test=False, xvfb=True, ad_block=True) as sb:
+            sb: BaseCase
+            sb.uc_open_with_reconnect(request.url)
+            source = sb.get_page_source()
+            source_bs = BeautifulSoup(source, "html.parser")
+            title_tag = source_bs.title
+            logger.info(f"Got webpage: {request.url}")
+            if title_tag and title_tag.string in src.utils.consts.CHALLENGE_TITLES:
+                logger.info("Challenge detected")
+                sb.uc_gui_click_captcha()
+                logger.info("Clicked captcha")
 
-        source = sb.get_page_source()
-        source_bs = BeautifulSoup(source, "html.parser")
-        title_tag = source_bs.title
-        if title_tag is None:
-            raise HTTPException(status_code=500, detail="Title tag not found")
+            source = sb.get_page_source()
+            source_bs = BeautifulSoup(source, "html.parser")
+            title_tag = source_bs.title
 
-        if title_tag.string in src.utils.consts.CHALLENGE_TITLES:
-            raise HTTPException(status_code=500, detail="Could not bypass challenge")
+            if title_tag and title_tag.string in src.utils.consts.CHALLENGE_TITLES:
+                sb.save_screenshot(f"./screenshots/{request.url}.png")
+                raise HTTPException(
+                    status_code=500, detail="Could not bypass challenge"
+                )
 
-        title = title_tag.string
-        logger.info(f"Title: {title}")
-        response = LinkResponse(
-            message="Success",
-            solution=Solution(
-                userAgent=sb.get_user_agent(),
-                url=sb.get_current_url(),
-                status=200,
-                cookies=sb.get_cookies(),
-                headers={},
-                response=source,
-            ),
-            startTimestamp=start_time,
-        )
+            response = LinkResponse(
+                message="Success",
+                solution=Solution(
+                    userAgent=sb.get_user_agent(),
+                    url=sb.get_current_url(),
+                    status=200,
+                    cookies=sb.get_cookies(),
+                    headers={},
+                    response=source,
+                ),
+                startTimestamp=start_time,
+            )
+    except Exception as e:
+        logger.error(f"Error: {e}")
+        raise HTTPException(status_code=500, detail="Unknown error, check logs") from e
 
     return response
 

poetry.lock

@@ -1324,6 +1324,19 @@ files = [
 [package.dependencies]
 pytest = "*"
 
+[[package]]
+name = "pytest-progress"
+version = "1.3.0"
+description = "pytest plugin for instant test progress status"
+optional = false
+python-versions = "*"
+files = [
+    {file = "pytest_progress-1.3.0.tar.gz", hash = "sha256:b2a6cd0b0cd8b50b19f56777402835e546dc8404eb7fa77ac2ace9dc719ec50e"},
+]
+
+[package.dependencies]
+pytest = ">=2.7"
+
 [[package]]
 name = "pytest-rerunfailures"
 version = "14.0"
@@ -1339,6 +1352,23 @@ files = [
 packaging = ">=17.1"
 pytest = ">=7.2"
 
+[[package]]
+name = "pytest-retry"
+version = "1.6.3"
+description = "Adds the ability to retry flaky tests in CI environments"
+optional = false
+python-versions = ">=3.9"
+files = [
+    {file = "pytest_retry-1.6.3-py3-none-any.whl", hash = "sha256:e96f7df77ee70b0838d1085f9c3b8b5b7d74bf8947a0baf32e2b8c71b27683c8"},
+    {file = "pytest_retry-1.6.3.tar.gz", hash = "sha256:36ccfa11c8c8f9ddad5e20375182146d040c20c4a791745139c5a99ddf1b557d"},
+]
+
+[package.dependencies]
+pytest = ">=7.0.0"
+
+[package.extras]
+dev = ["black", "flake8", "isort", "mypy"]
+
 [[package]]
 name = "pytest-xdist"
 version = "3.6.1"
@@ -2165,4 +2195,4 @@ h11 = ">=0.9.0,<1"
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.12"
-content-hash = "1dcc6c3a9ff83a4e27c96b1047a388e1ddd9a7c326b20ef07793c0721f9594dc"
+content-hash = "b14c2514cac9868f24705f98bfe5aaae06ed4b21508b9e4622ada2938338b525"

pyproject.toml

@@ -17,6 +17,8 @@ ruff = "^0.8.0"
 seleniumbase = "^4.32.12"
 pyautogui = "^0.9.54"
 beautifulsoup4 = "^4.12.3"
+pytest-retry = "^1.6.3"
+pytest-progress = "^1.3.0"
 
 
 [build-system]

src/models/requests.py

@@ -4,13 +4,13 @@ import time
 from http import HTTPStatus
 from typing import Any
 
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 
 
 class LinkRequest(BaseModel):
-    cmd: str
+    cmd: str = "get"
     url: str
-    maxTimeout: int  # noqa: N815 # Ignore to preserve compatibility
+    max_timeout: int = Field(30, alias="maxTimeout")
 
 
 class ProtectionTriggeredError(Exception):

tests/main_test.py

@@ -39,7 +39,9 @@ def test_bypass(website: str):
 
     response = client.post(
         "/v1",
-        json=LinkRequest(url=website, maxTimeout=30, cmd="request.get").model_dump(),
+        json=LinkRequest.model_construct(
+            url=website, max_timeout=30, cmd="request.get"
+        ).model_dump(),
     )
 
     assert response.status_code == HTTPStatus.OK