fix(musicseerr): move downloaded files immediately to seedbox, bypass Cloudflare with curl_cffi, and fix Last.fm recommendations feed fetching

apps/musicseerr/Dockerfile

@@ -22,7 +22,8 @@ RUN pip install --no-cache-dir \
     beautifulsoup4 \
     mutagen \
     spotdl \
-    yt-dlp
+    yt-dlp \
+    curl_cffi
 
 # Copy application files
 COPY main.py tasks.py /app/

apps/musicseerr/main.py

@@ -5,6 +5,7 @@ from fastapi import FastAPI, BackgroundTasks, Request, Form
 from fastapi.responses import HTMLResponse, JSONResponse
 from pydantic import BaseModel
 import requests
+from curl_cffi import requests as cf_requests
 
 from tasks import run_download_task
 
@@ -460,18 +461,28 @@ def index_page():
 
 def run_lastfm_auto_download(username: str):
     url = f"https://lfm.xiffy.nl/{username}/recommended"
-    proxies = {"http": PROXY_URL, "https": PROXY_URL} if PROXY_URL else None
     headers = {
         "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
     }
     
     logger.info(f"Fetching Last.fm recommendations feed from {url}")
+    response = None
     try:
-        response = requests.get(url, headers=headers, proxies=proxies, timeout=20)
+        logger.info("Fetching Last.fm feed directly (no proxy)")
+        response = cf_requests.get(url, headers=headers, impersonate="chrome", timeout=20)
         response.raise_for_status()
     except Exception as e:
-        logger.error(f"Failed to fetch Last.fm recommendations feed: {e}")
-        return
+        logger.warning(f"Failed to fetch Last.fm feed directly: {e}. Retrying with proxy...")
+        if PROXY_URL:
+            proxies = {"http": PROXY_URL, "https": PROXY_URL}
+            try:
+                response = cf_requests.get(url, headers=headers, proxies=proxies, impersonate="chrome", timeout=20)
+                response.raise_for_status()
+            except Exception as e2:
+                logger.error(f"Failed to fetch Last.fm feed with proxy: {e2}")
+                return
+        else:
+            return
         
     try:
         import xml.etree.ElementTree as ET

apps/musicseerr/tasks.py

@@ -9,11 +9,35 @@ from bs4 import BeautifulSoup
 from urllib.parse import urljoin
 from mutagen.mp3 import EasyMP3
 from mutagen.id3 import ID3
+from curl_cffi import requests as cf_requests
 
 # Configure Logger
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger("musicseerr-tasks")
 
+def move_new_files_to_dest(temp_dir: str, dest_dir: str) -> list:
+    moved = []
+    if not os.path.exists(temp_dir):
+        return moved
+    os.makedirs(dest_dir, exist_ok=True)
+    for file_name in os.listdir(temp_dir):
+        src_path = os.path.join(temp_dir, file_name)
+        # Check if it is a regular file and not a symlink
+        if os.path.isfile(src_path) and not os.path.islink(src_path):
+            dest_path = os.path.join(dest_dir, file_name)
+            logger.info(f"Moving new file '{file_name}' to remote Seedbox directory: {dest_path}")
+            try:
+                shutil.move(src_path, dest_path)
+                moved.append(file_name)
+                # Recreate as symlink in temp_dir so subsequent tools can find it if needed
+                try:
+                    os.symlink(dest_path, src_path)
+                except Exception as sym_err:
+                    logger.warning(f"Failed to recreate symlink for {file_name}: {sym_err}")
+            except Exception as move_err:
+                logger.error(f"Failed to move file {file_name} to {dest_dir}: {move_err}")
+    return moved
+
 def scrape_tubidy_link(url: str, proxy_url: str) -> str:
     """
     Scrapes a Tubidy page to find the direct MP3 download URL.
@@ -24,7 +48,7 @@ def scrape_tubidy_link(url: str, proxy_url: str) -> str:
     }
     
     logger.info(f"Fetching Tubidy page: {url} via proxy: {proxy_url}")
-    response = requests.get(url, headers=headers, proxies=proxies, timeout=20)
+    response = cf_requests.get(url, headers=headers, proxies=proxies, impersonate="chrome", timeout=20)
     response.raise_for_status()
     
     soup = BeautifulSoup(response.text, "html.parser")
@@ -51,7 +75,7 @@ def scrape_tubidy_link(url: str, proxy_url: str) -> str:
             sub_url = urljoin(url, a["href"])
             logger.info(f"Following nested Tubidy download link: {sub_url}")
             try:
-                sub_resp = requests.get(sub_url, headers=headers, proxies=proxies, timeout=15)
+                sub_resp = cf_requests.get(sub_url, headers=headers, proxies=proxies, impersonate="chrome", timeout=15)
                 if sub_resp.status_code == 200:
                     sub_soup = BeautifulSoup(sub_resp.text, "html.parser")
                     for sub_a in sub_soup.find_all("a", href=True):
@@ -88,7 +112,7 @@ def search_tubidy(query: str, proxy_url: str) -> list:
     
     logger.info(f"Searching Tubidy for '{query}' via proxy: {proxy_url}")
     try:
-        response = requests.get(search_url, headers=headers, params=params, proxies=proxies, timeout=15)
+        response = cf_requests.get(search_url, headers=headers, params=params, proxies=proxies, impersonate="chrome", timeout=15)
         response.raise_for_status()
         
         soup = BeautifulSoup(response.text, "html.parser")
@@ -142,7 +166,7 @@ def get_tubidy_download_link(content_id: str, proxy_url: str) -> str:
     
     logger.info(f"Fetching watch.php for content ID '{content_id}'")
     try:
-        response = requests.get(watch_url, headers=headers, params=params, proxies=proxies, timeout=15)
+        response = cf_requests.get(watch_url, headers=headers, params=params, proxies=proxies, impersonate="chrome", timeout=15)
         response.raise_for_status()
         
         soup = BeautifulSoup(response.text, "html.parser")
@@ -175,7 +199,7 @@ def download_file_stream(url: str, dest_path: str, proxy_url: str):
     }
     
     logger.info(f"Streaming download from {url} to {dest_path}")
-    response = requests.get(url, headers=headers, proxies=proxies, stream=True, timeout=60)
+    response = cf_requests.get(url, headers=headers, proxies=proxies, impersonate="chrome", stream=True, timeout=60)
     response.raise_for_status()
     
     with open(dest_path, "wb") as f:
@@ -246,7 +270,7 @@ def extract_spotify_track_title(url: str, proxy_url: str) -> str:
     
     logger.info(f"Scraping Spotify track via embed URL: {embed_url}")
     try:
-        response = requests.get(embed_url, headers=headers, proxies=proxies, timeout=15)
+        response = cf_requests.get(embed_url, headers=headers, proxies=proxies, impersonate="chrome", timeout=15)
         if response.status_code == 200:
             next_data_match = re.search(r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>', response.text)
             if next_data_match:
@@ -287,7 +311,7 @@ def scrape_spotify_playlist(url: str, proxy_url: str) -> list:
         try:
             # Short jitter to avoid hitting Spotify exactly at the same time
             time.sleep(1)
-            response = requests.get(embed_url, headers=headers, proxies=current_proxies, timeout=15)
+            response = cf_requests.get(embed_url, headers=headers, proxies=current_proxies, impersonate="chrome", timeout=15)
             if response.status_code == 200:
                 next_data_match = re.search(r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>', response.text)
                 if next_data_match:
@@ -488,6 +512,11 @@ def run_download_task(task_id: str, query: str, proxy_url: str, jobs_dict: dict,
                         
                 if track_success:
                     download_count += 1
+                    new_moved = move_new_files_to_dest(temp_dir, "/remote-music/music")
+                    if "files" not in jobs_dict[task_id] or not jobs_dict[task_id]["files"]:
+                        jobs_dict[task_id]["files"] = []
+                    jobs_dict[task_id]["files"].extend(new_moved)
+                    logger.info(f"[{task_id}] Successfully downloaded and moved {len(new_moved)} files for track '{track_query}'")
                     
             if download_count > 0:
                 download_successful = True
@@ -638,38 +667,25 @@ def run_download_task(task_id: str, query: str, proxy_url: str, jobs_dict: dict,
         if not download_successful:
             raise Exception("Ingestion finished but no download was successful.")
             
-        downloaded_files = os.listdir(temp_dir)
-        actual_files = [f for f in downloaded_files if not os.path.islink(os.path.join(temp_dir, f))]
-        
-        if not actual_files:
-            if downloaded_files:
-                logger.info(f"[{task_id}] All tracks in this query were already downloaded (skipped duplicates).")
+        if download_successful and not is_playlist:
+            new_moved = move_new_files_to_dest(temp_dir, "/remote-music/music")
+            if "files" not in jobs_dict[task_id] or not jobs_dict[task_id]["files"]:
                 jobs_dict[task_id]["files"] = []
-                jobs_dict[task_id]["status"] = "completed"
-                if chat_id:
-                    send_telegram_notification(chat_id, f"✅ Successfully ingested request: '{query}'\n\n(All tracks already existed - skipped duplicates)")
-                return
-            else:
-                raise Exception("Ingestion finished successfully but no files were found in local temp folder.")
+            jobs_dict[task_id]["files"].extend(new_moved)
             
-        os.makedirs("/remote-music/music", exist_ok=True)
-        moved_files = []
-        
-        for file_name in actual_files:
-            src_file = os.path.join(temp_dir, file_name)
-            dest_file = os.path.join("/remote-music/music", file_name)
+        if "files" not in jobs_dict[task_id] or not jobs_dict[task_id]["files"]:
+            logger.info(f"[{task_id}] All tracks in this query were already downloaded (skipped duplicates).")
+            jobs_dict[task_id]["files"] = []
+            jobs_dict[task_id]["status"] = "completed"
+            if chat_id:
+                send_telegram_notification(chat_id, f"✅ Successfully ingested request: '{query}'\n\n(All tracks already existed - skipped duplicates)")
+            return
             
-            logger.info(f"[{task_id}] Moving new file '{file_name}' to remote Seedbox directory")
-            shutil.move(src_file, dest_file)
-            moved_files.append(file_name)
-
-            
-        jobs_dict[task_id]["files"] = moved_files
         jobs_dict[task_id]["status"] = "completed"
         logger.info(f"[{task_id}] Music Ingestion task fully completed.")
         
         if chat_id:
-            files_str = "\n".join(moved_files)
+            files_str = "\n".join(jobs_dict[task_id]["files"])
             send_telegram_notification(chat_id, f"✅ Successfully ingested request: '{query}'\n\nFiles imported to Navidrome:\n{files_str}")
         
     except Exception as e: