Use matrix build for main and tor github action (#237)

- Fix branch tagging and use :dev for main and :latest for tagged
release
- Fix latest tag check for release

.github/workflows/build-and-publish-docker-image.yml

@@ -1,20 +1,14 @@
 name: Create and publish Docker images
-
 on:
   push:
     branches:
       - 'main'
     tags:
       - 'v*'
-  pull_request:
-    branches:
-      - 'main'  
   workflow_dispatch:
-
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
-
 jobs:
   build-and-push-images:
     runs-on: ubuntu-latest
@@ -23,12 +17,23 @@ jobs:
       packages: write
       attestations: write
       id-token: write
+    strategy:
+      matrix:
+        include:
+          - suffix: ""
+            target: cwa-bd
+            image_name_suffix: ""
+          - suffix: "-tor"
+            target: cwa-bd-tor
+            image_name_suffix: "-tor"
     steps:
       - name: Get current date
         id: date
-        run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+        run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
+      
       - name: Checkout repository
         uses: actions/checkout@v4
+      
       - name: Log in to the Container registry
         uses: docker/login-action@v3
         with:
@@ -36,74 +41,39 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       
-      # Build and push main image
-      - name: Extract metadata for main image
-        id: meta-main
+      - name: Extract metadata for ${{ matrix.target }} image
+        id: meta
         uses: docker/metadata-action@v5
         with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}${{ matrix.image_name_suffix }}
           tags: |
-            type=raw,value=latest,enable={{is_default_branch}}
+            type=raw,value=dev,enable={{is_default_branch}}
+            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/') }}
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
             type=sha
-            type=ref,event=branch
             type=ref,event=tag
       
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
       
-      - name: Build and push main Docker image
-        id: push-main
+      - name: Build and push ${{ matrix.target }} Docker image
+        id: push
         uses: docker/build-push-action@v5
         with:
           platforms: linux/amd64,linux/arm64
           context: .
-          target: cwa-bd
-          push: true
+          target: ${{ matrix.target }}
+          push: ${{ github.event_name != 'pull_request' }}
           build-args: |
             BUILD_VERSION=${{ steps.date.outputs.date }}-${{ github.sha }}
-          tags: ${{ steps.meta-main.outputs.tags }}
-          labels: ${{ steps.meta-main.outputs.labels }}
-
-      - name: Generate artifact attestation for main image
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+      
+      - name: Generate artifact attestation for ${{ matrix.target }} image
+        if: github.event_name != 'pull_request'
         uses: actions/attest-build-provenance@v2
         with:
-          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-          subject-digest: ${{ steps.push-main.outputs.digest }}
-          push-to-registry: true
-
-      # Build and push tor image
-      - name: Extract metadata for tor image
-        id: meta-tor
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-tor
-          tags: |
-            type=raw,value=latest,enable={{is_default_branch}}
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=sha
-            type=ref,event=branch
-            type=ref,event=tag
-
-      - name: Build and push tor Docker image
-        id: push-tor
-        uses: docker/build-push-action@v5
-        with:
-          platforms: linux/amd64,linux/arm64
-          context: .
-          target: cwa-bd-tor
-          push: true
-          build-args: |
-            BUILD_VERSION=${{ steps.date.outputs.date }}-${{ github.sha }}
-          tags: ${{ steps.meta-tor.outputs.tags }}
-          labels: ${{ steps.meta-tor.outputs.labels }}
-
-      - name: Generate artifact attestation for tor image
-        uses: actions/attest-build-provenance@v2
-        with:
-          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-tor
-          subject-digest: ${{ steps.push-tor.outputs.digest }}
-          push-to-registry: true
-
+          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}${{ matrix.image_name_suffix }}
+          subject-digest: ${{ steps.push.outputs.digest }}
+          push-to-registry: true

book_manager.py

@@ -176,6 +176,7 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo:
     slow_urls_with_waitlist = set()
     external_urls_libgen = set()
     external_urls_z_lib = set()
+    external_urls_welib = set()
 
     for url in every_url:
         try:
@@ -209,6 +210,7 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo:
     urls = []
     urls += list(slow_urls_no_waitlist) if USE_CF_BYPASS else []
     urls += list(external_urls_libgen)
+    urls += list( _get_download_urls_from_welib(book_id))  if USE_CF_BYPASS else []
     urls += list(slow_urls_with_waitlist)  if USE_CF_BYPASS else []
     urls += list(external_urls_z_lib)
 
@@ -242,6 +244,18 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo:
 
     return book_info
 
+def _get_download_urls_from_welib(book_id: str) -> List[str]:
+    """Get download urls from welib.org."""
+    url = f"https://welib.org/md5/{book_id}"
+    html = downloader.html_get_page(url, use_bypasser=True)
+    if not html:
+        return []
+    soup = BeautifulSoup(html, "html.parser")
+    download_links = soup.find_all("a", href=True)
+    download_links = [link["href"] for link in download_links]
+    download_links = [link for link in download_links if "/slow_download/" in link]
+    download_links = [downloader.get_absolute_url(url, link) for link in download_links]
+    return set(download_links)
 
 def _extract_book_metadata(
     metadata_divs
@@ -347,7 +361,7 @@ def _get_download_url(link: str, title: str) -> str:
             download_link = soup.find_all("a", href=True, class_="addDownloadedBook")
             if download_link:
                 url = download_link[0]["href"]
-        elif link.startswith(f"{AA_BASE_URL}/slow_download/"):
+        elif "/slow_download/" in link:
             download_links = soup.find_all("a", href=True, string="📚 Download now")
             if not download_links:
                 countdown = soup.find_all("span", class_="js-partner-countdown")

docker-compose.yml

@@ -1,6 +1,7 @@
 services:
   calibre-web-automated-book-downloader:
     image: ghcr.io/calibrain/calibre-web-automated-book-downloader:latest
+    container_name: calibre-web-automated-book-downloader
     environment:
       FLASK_PORT: 8084
       LOG_LEVEL: info
@@ -20,4 +21,4 @@ services:
       - /tmp/data/calibre-web/ingest:/cwa-book-ingest
       # This is the location of CWA's app.db, which contains authentication
       # details
-      - /cwa/config/path/app.db:/auth/app.db:ro
+      #- /cwa/config/path/app.db:/auth/app.db:ro

requirements.txt

@@ -5,7 +5,7 @@ tqdm
 pyvirtualdisplay
 dnspython
 pyautogui
-seleniumbase
+seleniumbase==4.41
 gunicorn
 python-xlib
 psutil