fix seleniumbase (#230)

- Fix typo

book_manager.py

@@ -166,61 +166,43 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo:
         else:
             preview = preview_value
 
-    data = data.select_one("div:nth-of-type(1)")
-    while "🔍" not in data.text:
-        data = data.parent
-    # Find the start of book information
-    divs = data.find_all("div")
-    start_div_id = next((i for i, div in enumerate(divs) if "🔍" in div.text), 3)
-
-    format_div = divs[start_div_id - 1].text
-    format_parts = format_div.split(".")
-    if len(format_parts) > 1:
-        format = format_parts[1].split(",")[0].strip().lower()
-    else:
-        format = None
-
-    size = next(
-        (
-            token.strip()
-            for token in format_div.split(",")
-            if token.strip() and token.strip()[0].isnumeric()
-        ),
-        None,
-    )
+    data = soup.find_all("div", {"class": "main-inner"})[0].find_next("div")
+    divs = list(data.children)
+    format = divs[13].text.split(" · ")[1].strip().lower()
+    size = divs[13].text.split(" · ")[2].strip().lower()
 
     every_url = soup.find_all("a")
     slow_urls_no_waitlist = set()
     slow_urls_with_waitlist = set()
     external_urls_libgen = set()
     external_urls_z_lib = set()
+    external_urls_welib = set()
 
     for url in every_url:
         try:
-            if url.parent.text.strip().lower().startswith("option #"):
-                if url.text.strip().lower().startswith("slow partner server"):
-                    if (
-                        url.next is not None
-                        and url.next.next is not None
-                        and "waitlist" in url.next.next.strip().lower()
-                    ):
-                        internal_text = url.next.next.strip().lower()
-                        if "no waitlist" in internal_text:
-                            slow_urls_no_waitlist.add(url["href"])
-                        else:
-                            slow_urls_with_waitlist.add(url["href"])
-                elif (
+            if url.text.strip().lower().startswith("slow partner server"):
+                if (
                     url.next is not None
                     and url.next.next is not None
-                    and "click “GET” at the top" in url.next.next.text.strip()
+                    and "waitlist" in url.next.next.strip().lower()
                 ):
-                    libgen_url = url["href"]
-                    # TODO : Temporary fix ? Maybe get URLs from https://open-slum.org/ ?
-                    libgen_url = libgen_url = re.sub(r'libgen\.(\w+)', 'libgen.bz', url["href"])
-                    external_urls_libgen.add(libgen_url)
-                elif url.text.strip().lower().startswith("z-lib"):
-                    if ".onion/" not in url["href"]:
-                        external_urls_z_lib.add(url["href"])
+                    internal_text = url.next.next.strip().lower()
+                    if "no waitlist" in internal_text:
+                        slow_urls_no_waitlist.add(url["href"])
+                    else:
+                        slow_urls_with_waitlist.add(url["href"])
+            elif (
+                url.next is not None
+                and url.next.next is not None
+                and "click “GET” at the top" in url.next.next.text.strip()
+            ):
+                libgen_url = url["href"]
+                # TODO : Temporary fix ? Maybe get URLs from https://open-slum.org/ ?
+                libgen_url = libgen_url = re.sub(r'libgen\.(\w+)', 'libgen.bz', url["href"])
+                external_urls_libgen.add(libgen_url)
+            elif url.text.strip().lower().startswith("z-lib"):
+                if ".onion/" not in url["href"]:
+                    external_urls_z_lib.add(url["href"])
         except:
             pass
 
@@ -228,26 +210,30 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo:
     urls = []
     urls += list(slow_urls_no_waitlist) if USE_CF_BYPASS else []
     urls += list(external_urls_libgen)
+    urls += list( _get_download_urls_from_welib(book_id))  if USE_CF_BYPASS else []
     urls += list(slow_urls_with_waitlist)  if USE_CF_BYPASS else []
     urls += list(external_urls_z_lib)
 
     for i in range(len(urls)):
         urls[i] = downloader.get_absolute_url(AA_BASE_URL, urls[i])
 
+    # Remove empty urls
+    urls = [url for url in urls if url != ""]
+
     # Extract basic information
     book_info = BookInfo(
         id=book_id,
         preview=preview,
-        title=divs[start_div_id].next,
-        publisher=divs[start_div_id + 1].next,
-        author=divs[start_div_id + 2].next,
+        title=divs[7].next.strip(),
+        publisher=divs[11].text.strip(),
+        author=divs[9].text.strip(),
         format=format,
         size=size,
         download_urls=urls,
     )
 
     # Extract additional metadata
-    info = _extract_book_metadata(divs[start_div_id + 3 :])
+    info = _extract_book_metadata(divs[-6])
     book_info.info = info
 
     # Set language and year from metadata if available
@@ -258,35 +244,41 @@ def _parse_book_info_page(soup: BeautifulSoup, book_id: str) -> BookInfo:
 
     return book_info
 
+def _get_download_urls_from_welib(book_id: str) -> List[str]:
+    """Get download urls from welib.org."""
+    url = f"https://welib.org/md5/{book_id}"
+    html = downloader.html_get_page(url, use_bypasser=True)
+    if not html:
+        return []
+    soup = BeautifulSoup(html, "html.parser")
+    download_links = soup.find_all("a", href=True)
+    download_links = [link["href"] for link in download_links]
+    download_links = [link for link in download_links if "/slow_download/" in link]
+    download_links = [downloader.get_absolute_url(url, link) for link in download_links]
+    return set(download_links)
 
 def _extract_book_metadata(
-    metadata_divs: Union[ResultSet[Tag], List[Tag]],
+    metadata_divs
 ) -> Dict[str, List[str]]:
     """Extract metadata from book info divs."""
     info: Dict[str, List[str]] = {}
 
     # Process the first set of metadata
-    sub_data = metadata_divs[0].find_all("div")
-    for i in range(0, len(sub_data) - 1, 2):
-        key = sub_data[i].next
-        value = sub_data[i + 1].next
+    sub_datas = metadata_divs.find_all("div")[0]
+    sub_datas = list(sub_datas.children)
+    for sub_data in sub_datas:
+        if sub_data.text.strip() == "":
+            continue
+        sub_data = list(sub_data.children)
+        key = sub_data[0].text.strip()
+        value = sub_data[1].text.strip()
         if key not in info:
-            info[key] = []
-        info[key].append(value)
-
-    # Process the second set of metadata (spans)
-    # Find elements where aria-label="code tabs"
-    meta_spans: List[Tag] = []
-    for div in metadata_divs:
-        if div.find_all("div", {"aria-label": "code tabs"}):
-            meta_spans = div.find_all("span")
-            break
-    for i in range(0, len(meta_spans) - 1, 2):
-        key = meta_spans[i].next
-        value = meta_spans[i + 1].next
-        if key not in info:
-            info[key] = []
-        info[key].append(value)
+            info[key] = set()
+        info[key].add(value)
+    
+    # make set into list
+    for key, value in info.items():
+        info[key] = list(value)
 
     # Filter relevant metadata
     relevant_prefixes = [
@@ -369,7 +361,7 @@ def _get_download_url(link: str, title: str) -> str:
             download_link = soup.find_all("a", href=True, class_="addDownloadedBook")
             if download_link:
                 url = download_link[0]["href"]
-        elif link.startswith(f"{AA_BASE_URL}/slow_download/"):
+        elif "/slow_download/" in link:
             download_links = soup.find_all("a", href=True, string="📚 Download now")
             if not download_links:
                 countdown = soup.find_all("span", class_="js-partner-countdown")

docker-compose.yml

@@ -1,6 +1,7 @@
 services:
   calibre-web-automated-book-downloader:
     image: ghcr.io/calibrain/calibre-web-automated-book-downloader:latest
+    container_name: calibre-web-automated-book-downloader
     environment:
       FLASK_PORT: 8084
       LOG_LEVEL: info
@@ -20,4 +21,4 @@ services:
       - /tmp/data/calibre-web/ingest:/cwa-book-ingest
       # This is the location of CWA's app.db, which contains authentication
       # details
-      - /cwa/config/path/app.db:/auth/app.db:ro
+      #- /cwa/config/path/app.db:/auth/app.db:ro

downloader.py

@@ -120,6 +120,8 @@ def get_absolute_url(base_url: str, url: str) -> str:
     """
     if url.strip() == "":
         return ""
+    if url.strip("#") == "":
+        return ""
     if url.startswith("http"):
         return url
     parsed_url = urlparse(url)

requirements.txt

@@ -5,7 +5,7 @@ tqdm
 pyvirtualdisplay
 dnspython
 pyautogui
-seleniumbase
+seleniumbase==4.41
 gunicorn
 python-xlib
 psutil