feat: implement Priority 4 Tubidy Search Fallback when spotDL and yt-dlp both fail

apps/musicseerr/tasks.py

@@ -68,6 +68,102 @@ def scrape_tubidy_link(url: str, proxy_url: str) -> str:
     raise ValueError("Could not find any MP3 download link on the Tubidy page.")
 
 
+def search_tubidy(query: str, proxy_url: str) -> list:
+    """
+    Searches Tubidy for the query and returns a list of results with ID, title, and link.
+    """
+    proxies = {"http": proxy_url, "https": proxy_url} if proxy_url else None
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+    }
+    
+    endpoint = "https://mp3.tubidy.cool"
+    search_url = f"{endpoint}/search.php"
+    params = {
+        "q": query,
+        "si": 7,
+        "pn": 1
+    }
+    
+    logger.info(f"Searching Tubidy for '{query}' via proxy: {proxy_url}")
+    try:
+        response = requests.get(search_url, headers=headers, params=params, proxies=proxies, timeout=15)
+        response.raise_for_status()
+        
+        soup = BeautifulSoup(response.text, "html.parser")
+        results = []
+        
+        for media_body in soup.find_all("div", class_="media-body"):
+            a_tag = media_body.find("a")
+            if a_tag:
+                href = a_tag.get("href")
+                if href:
+                    title = a_tag.get("aria-label") or a_tag.text.strip()
+                    # Extract ID from /watch/content_id
+                    match = re.search(r'/watch/([^/]+)', href)
+                    content_id = match.group(1) if match else None
+                    link = href
+                    if href.startswith("//"):
+                        link = f"https:{href}"
+                    elif href.startswith("/"):
+                        link = f"{endpoint}{href}"
+                    
+                    if content_id:
+                        results.append({
+                            "id": content_id,
+                            "title": title,
+                            "link": link
+                        })
+        return results
+    except Exception as e:
+        logger.error(f"Failed to search Tubidy: {e}")
+        return []
+
+
+def get_tubidy_download_link(content_id: str, proxy_url: str) -> str:
+    """
+    Queries watch.php for the given content ID to find the direct MP3 download URL.
+    """
+    proxies = {"http": proxy_url, "https": proxy_url} if proxy_url else None
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+    }
+    
+    endpoint = "https://mp3.tubidy.cool"
+    watch_url = f"{endpoint}/watch.php"
+    params = {
+        "id": content_id,
+        "p": "mp4",
+        "lnk": 6,
+        "act": "down",
+        "t": "ssl"
+    }
+    
+    logger.info(f"Fetching watch.php for content ID '{content_id}'")
+    try:
+        response = requests.get(watch_url, headers=headers, params=params, proxies=proxies, timeout=15)
+        response.raise_for_status()
+        
+        soup = BeautifulSoup(response.text, "html.parser")
+        for li in soup.find_all("li", class_=lambda x: x and "list-group-item" in x and "big" in x):
+            a_tag = li.find("a")
+            if a_tag:
+                text = a_tag.text.lower()
+                href = a_tag.get("href")
+                if href and "download" in text:
+                    return href
+                    
+        # Fallback to general scraping of the watch.php page if structure differs
+        for a_tag in soup.find_all("a", href=True):
+            text = a_tag.text.lower()
+            if "download" in text and "act=process" not in a_tag["href"]:
+                return a_tag["href"]
+    except Exception as e:
+        logger.error(f"Failed to fetch download link for {content_id}: {e}")
+        
+    raise ValueError("Could not find download link on watch.php page.")
+
+
 def download_file_stream(url: str, dest_path: str, proxy_url: str):
     """
     Downloads a file in chunks using requests through the proxy.
@@ -223,10 +319,9 @@ def run_download_task(task_id: str, query: str, proxy_url: str, jobs_dict: dict,
             if result.returncode == 0:
                 logger.info(f"[{task_id}] yt-dlp download succeeded.")
             else:
-                logger.error(f"[{task_id}] yt-dlp failed: {result.stderr or result.stdout}")
+                logger.warning(f"[{task_id}] yt-dlp failed: {result.stderr or result.stdout}. Falling back to Tubidy Search.")
-                raise Exception(f"yt-dlp failed to download: {result.stderr}")
                 
-        # Priority 3: Custom Tubidy Scraper
+        # Priority 3: Custom Tubidy Link Scraper (if URL is directly passed)
         if is_tubidy:
             jobs_dict[task_id]["priority_used"] = "Tubidy"
             logger.info(f"[{task_id}] Attempting Priority 3: Tubidy Scraper for '{query}'")
@@ -253,6 +348,45 @@ def run_download_task(task_id: str, query: str, proxy_url: str, jobs_dict: dict,
             tag_mp3(dest_path, filename)
             logger.info(f"[{task_id}] Tubidy ID3 tagging complete.")
             
+        # Priority 4: Tubidy Search Fallback (if spotDL and yt-dlp failed and no files are present)
+        downloaded_files = os.listdir(temp_dir)
+        if not downloaded_files and not is_tubidy:
+            jobs_dict[task_id]["priority_used"] = "Tubidy-Search"
+            logger.info(f"[{task_id}] Attempting Fallback Priority 4: Tubidy Search Scraper for '{query}'")
+            
+            search_results = search_tubidy(query, resolved_proxy_url)
+            if search_results:
+                first_result = search_results[0]
+                logger.info(f"[{task_id}] Found Tubidy match: '{first_result['title']}' (ID: {first_result['id']})")
+                
+                try:
+                    mp3_url = get_tubidy_download_link(first_result["id"], resolved_proxy_url)
+                    logger.info(f"[{task_id}] Resolved Tubidy download URL: {mp3_url}")
+                    
+                    filename = f"{first_result['title']}.mp3"
+                    # Clean filename
+                    filename = "".join([c for c in filename if c.isalnum() or c in " .-_()"]).strip()
+                    if not filename.endswith(".mp3"):
+                        filename += ".mp3"
+                    if not filename or filename == ".mp3":
+                        filename = "tubidy_fallback.mp3"
+                        
+                    dest_path = os.path.join(temp_dir, filename)
+                    
+                    # Download file stream
+                    download_file_stream(mp3_url, dest_path, resolved_proxy_url)
+                    logger.info(f"[{task_id}] Tubidy fallback download completed.")
+                    
+                    # Tag metadata
+                    tag_mp3(dest_path, filename)
+                    logger.info(f"[{task_id}] Tubidy fallback ID3 tagging complete.")
+                except Exception as e:
+                    logger.error(f"[{task_id}] Tubidy fallback download failed: {e}")
+                    raise Exception(f"All download options (spotDL, yt-dlp, Tubidy) failed. Last error: {e}")
+            else:
+                logger.error(f"[{task_id}] No Tubidy search results found for query '{query}'")
+                raise Exception("All download options (spotDL, yt-dlp, Tubidy) failed. No Tubidy search results found.")
+            
         # Verify files and move to /remote-music/music
         downloaded_files = os.listdir(temp_dir)
         if not downloaded_files: