Bug fixes for Login and WELIB (#242)

Fix login (when using latest CWA in dev)
Add PRIORITIZE_WELIB option
libgen seems to be back, revert the url rewrite

Should fix #242 and #226

.github/workflows/build-and-publish-docker-image.yml

@@ -1,20 +1,14 @@
 name: Create and publish Docker images
-
 on:
   push:
     branches:
       - 'main'
     tags:
       - 'v*'
-  pull_request:
-    branches:
-      - 'main'  
   workflow_dispatch:
-
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
-
 jobs:
   build-and-push-images:
     runs-on: ubuntu-latest
@@ -23,12 +17,23 @@ jobs:
       packages: write
       attestations: write
       id-token: write
+    strategy:
+      matrix:
+        include:
+          - suffix: ""
+            target: cwa-bd
+            image_name_suffix: ""
+          - suffix: "-tor"
+            target: cwa-bd-tor
+            image_name_suffix: "-tor"
     steps:
       - name: Get current date
         id: date
-        run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+        run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
+      
       - name: Checkout repository
         uses: actions/checkout@v4
+      
       - name: Log in to the Container registry
         uses: docker/login-action@v3
         with:
@@ -36,74 +41,39 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       
-      # Build and push main image
-      - name: Extract metadata for main image
-        id: meta-main
+      - name: Extract metadata for ${{ matrix.target }} image
+        id: meta
         uses: docker/metadata-action@v5
         with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}${{ matrix.image_name_suffix }}
           tags: |
-            type=raw,value=latest,enable={{is_default_branch}}
+            type=raw,value=dev,enable={{is_default_branch}}
+            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/') }}
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
             type=sha
-            type=ref,event=branch
             type=ref,event=tag
       
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
       
-      - name: Build and push main Docker image
-        id: push-main
+      - name: Build and push ${{ matrix.target }} Docker image
+        id: push
         uses: docker/build-push-action@v5
         with:
           platforms: linux/amd64,linux/arm64
           context: .
-          target: cwa-bd
-          push: true
+          target: ${{ matrix.target }}
+          push: ${{ github.event_name != 'pull_request' }}
           build-args: |
             BUILD_VERSION=${{ steps.date.outputs.date }}-${{ github.sha }}
-          tags: ${{ steps.meta-main.outputs.tags }}
-          labels: ${{ steps.meta-main.outputs.labels }}
-
-      - name: Generate artifact attestation for main image
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+      
+      - name: Generate artifact attestation for ${{ matrix.target }} image
+        if: github.event_name != 'pull_request'
         uses: actions/attest-build-provenance@v2
         with:
-          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-          subject-digest: ${{ steps.push-main.outputs.digest }}
-          push-to-registry: true
-
-      # Build and push tor image
-      - name: Extract metadata for tor image
-        id: meta-tor
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-tor
-          tags: |
-            type=raw,value=latest,enable={{is_default_branch}}
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=sha
-            type=ref,event=branch
-            type=ref,event=tag
-
-      - name: Build and push tor Docker image
-        id: push-tor
-        uses: docker/build-push-action@v5
-        with:
-          platforms: linux/amd64,linux/arm64
-          context: .
-          target: cwa-bd-tor
-          push: true
-          build-args: |
-            BUILD_VERSION=${{ steps.date.outputs.date }}-${{ github.sha }}
-          tags: ${{ steps.meta-tor.outputs.tags }}
-          labels: ${{ steps.meta-tor.outputs.labels }}
-
-      - name: Generate artifact attestation for tor image
-        uses: actions/attest-build-provenance@v2
-        with:
-          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-tor
-          subject-digest: ${{ steps.push-tor.outputs.digest }}
-          push-to-registry: true
-
+          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}${{ matrix.image_name_suffix }}
+          subject-digest: ${{ steps.push.outputs.digest }}
+          push-to-registry: true

app.py

@@ -357,7 +357,10 @@ def authenticate() -> bool:
 
     # Validate credentials against database
     try:
-        conn = sqlite3.connect(CWA_DB_PATH)
+        # Open database in true read-only mode to avoid journal/WAL writes on RO mounts
+        db_path = os.fspath(CWA_DB_PATH)
+        db_uri = f"file:{db_path}?mode=ro&immutable=1"
+        conn = sqlite3.connect(db_uri, uri=True)
         cur = conn.cursor()
         cur.execute("SELECT password FROM user WHERE name = ?", (username,))
         row = cur.fetchone()

book_manager.py

@@ -9,7 +9,7 @@ from bs4 import BeautifulSoup, Tag, NavigableString, ResultSet
 import downloader
 from logger import setup_logger
 from config import SUPPORTED_FORMATS, BOOK_LANGUAGE, AA_BASE_URL
-from env import AA_DONATOR_KEY, USE_CF_BYPASS
+from env import AA_DONATOR_KEY, USE_CF_BYPASS, PRIORITIZE_WELIB
 from models import BookInfo, SearchFilters
 
 logger = setup_logger(__name__)
@@ -176,6 +176,7 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo:
     slow_urls_with_waitlist = set()
     external_urls_libgen = set()
     external_urls_z_lib = set()
+    external_urls_welib = set()
 
     for url in every_url:
         try:
@@ -197,7 +198,7 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo:
             ):
                 libgen_url = url["href"]
                 # TODO : Temporary fix ? Maybe get URLs from https://open-slum.org/ ?
-                libgen_url = libgen_url = re.sub(r'libgen\.(\w+)', 'libgen.bz', url["href"])
+                libgen_url = libgen_url = re.sub(r'libgen\.(\w+)', 'libgen.gs', url["href"])
                 external_urls_libgen.add(libgen_url)
             elif url.text.strip().lower().startswith("z-lib"):
                 if ".onion/" not in url["href"]:
@@ -205,10 +206,13 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo:
         except:
             pass
 
+    external_urls_welib = _get_download_urls_from_welib(book_id) if USE_CF_BYPASS else set()
 
     urls = []
+    urls += list(external_urls_welib) if PRIORITIZE_WELIB else []
     urls += list(slow_urls_no_waitlist) if USE_CF_BYPASS else []
     urls += list(external_urls_libgen)
+    urls += list(external_urls_welib) if not PRIORITIZE_WELIB else []
     urls += list(slow_urls_with_waitlist)  if USE_CF_BYPASS else []
     urls += list(external_urls_z_lib)
 
@@ -242,6 +246,19 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo:
 
     return book_info
 
+def _get_download_urls_from_welib(book_id: str) -> set[str]:
+    """Get download urls from welib.org."""
+    url = f"https://welib.org/md5/{book_id}"
+    logger.info(f"Getting download urls from welib.org for {book_id}. While this uses the bypasser, it will not start downloading them yet.")
+    html = downloader.html_get_page(url, use_bypasser=True)
+    if not html:
+        return []
+    soup = BeautifulSoup(html, "html.parser")
+    download_links = soup.find_all("a", href=True)
+    download_links = [link["href"] for link in download_links]
+    download_links = [link for link in download_links if "/slow_download/" in link]
+    download_links = [downloader.get_absolute_url(url, link) for link in download_links]
+    return set(download_links)
 
 def _extract_book_metadata(
     metadata_divs
@@ -347,7 +364,7 @@ def _get_download_url(link: str, title: str) -> str:
             download_link = soup.find_all("a", href=True, class_="addDownloadedBook")
             if download_link:
                 url = download_link[0]["href"]
-        elif link.startswith(f"{AA_BASE_URL}/slow_download/"):
+        elif "/slow_download/" in link:
             download_links = soup.find_all("a", href=True, string="📚 Download now")
             if not download_links:
                 countdown = soup.find_all("span", class_="js-partner-countdown")

docker-compose.yml

@@ -1,6 +1,7 @@
 services:
   calibre-web-automated-book-downloader:
     image: ghcr.io/calibrain/calibre-web-automated-book-downloader:latest
+    container_name: calibre-web-automated-book-downloader
     environment:
       FLASK_PORT: 8084
       LOG_LEVEL: info
@@ -20,4 +21,4 @@ services:
       - /tmp/data/calibre-web/ingest:/cwa-book-ingest
       # This is the location of CWA's app.db, which contains authentication
       # details
-      - /cwa/config/path/app.db:/auth/app.db:ro
+      #- /cwa/config/path/app.db:/auth/app.db:ro

env.py

@@ -25,7 +25,9 @@ _BOOK_LANGUAGE = os.getenv("BOOK_LANGUAGE", "en").lower()
 _CUSTOM_SCRIPT = os.getenv("CUSTOM_SCRIPT", "").strip()
 FLASK_HOST = os.getenv("FLASK_HOST", "0.0.0.0")
 FLASK_PORT = int(os.getenv("FLASK_PORT", "8084"))
-DEBUG = string_to_bool(os.getenv("DEBUG", "False"))
+DEBUG = string_to_bool(os.getenv("DEBUG", "false"))
+PRIORITIZE_WELIB = string_to_bool(os.getenv("PRIORITIZE_WELIB", "false"))
+
 # If debug is true, we want to log everything
 if DEBUG:
     LOG_LEVEL = "DEBUG"

readme.md

@@ -82,6 +82,7 @@ Note that if using TOR, the TZ will be calculated automatically based on IP.
 | `BOOK_LANGUAGE`        | Preferred language for books                              | `en`                              |
 | `AA_DONATOR_KEY`       | Optional Donator key for Anna's Archive fast download API | ``                                |
 | `USE_BOOK_TITLE`       | Use book title as filename instead of ID                  | `false`                           |
+| `PRIORITIZE_WELIB`     | When downloading, download from WELIB first instead of AA | `false`                           |
 
 If you change `BOOK_LANGUAGE`, you can add multiple comma separated languages, such as `en,fr,ru` etc.  
 
@@ -90,7 +91,7 @@ If you change `BOOK_LANGUAGE`, you can add multiple comma separated languages, s
 | Variable               | Description                                               | Default Value                     |
 | ---------------------- | --------------------------------------------------------- | --------------------------------- |
 | `AA_BASE_URL`          | Base URL of Annas-Archive (could be changed for a proxy)  | `https://annas-archive.org`       |
-| `USE_CF_BYPASS`        | Disable CF bypass and use alternative links instead       | `true`                           |
+| `USE_CF_BYPASS`        | Disable CF bypass and use alternative links instead       | `true`                            |
 
 If you are a donator on AA, you can use your Key in `AA_DONATOR_KEY` to speed up downloads and bypass the wait times.
 If disabling the cloudflare bypass, you will be using alternative download hosts, such as libgen or z-lib, but they usually have a delay before getting the more recent books and their collection is not as big as aa's. But this setting should work for the majority of books.

requirements.txt

@@ -5,7 +5,7 @@ tqdm
 pyvirtualdisplay
 dnspython
 pyautogui
-seleniumbase
+seleniumbase>=4.41.1
 gunicorn
 python-xlib
 psutil