From 732f71700caf14e4287cd2a4200ffa8720f331d6 Mon Sep 17 00:00:00 2001 From: fchinembiri Date: Fri, 10 Jul 2026 20:19:05 +0200 Subject: [PATCH] fix(musicseerr): move downloaded files immediately to seedbox, bypass Cloudflare with curl_cffi, and fix Last.fm recommendations feed fetching --- apps/musicseerr/Dockerfile | 3 +- apps/musicseerr/main.py | 19 +++++++-- apps/musicseerr/tasks.py | 80 +++++++++++++++++++++++--------------- 3 files changed, 65 insertions(+), 37 deletions(-) diff --git a/apps/musicseerr/Dockerfile b/apps/musicseerr/Dockerfile index c118d8f..a74d279 100644 --- a/apps/musicseerr/Dockerfile +++ b/apps/musicseerr/Dockerfile @@ -22,7 +22,8 @@ RUN pip install --no-cache-dir \ beautifulsoup4 \ mutagen \ spotdl \ - yt-dlp + yt-dlp \ + curl_cffi # Copy application files COPY main.py tasks.py /app/ diff --git a/apps/musicseerr/main.py b/apps/musicseerr/main.py index bd39877..85c245d 100644 --- a/apps/musicseerr/main.py +++ b/apps/musicseerr/main.py @@ -5,6 +5,7 @@ from fastapi import FastAPI, BackgroundTasks, Request, Form from fastapi.responses import HTMLResponse, JSONResponse from pydantic import BaseModel import requests +from curl_cffi import requests as cf_requests from tasks import run_download_task @@ -460,18 +461,28 @@ def index_page(): def run_lastfm_auto_download(username: str): url = f"https://lfm.xiffy.nl/{username}/recommended" - proxies = {"http": PROXY_URL, "https": PROXY_URL} if PROXY_URL else None headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" } logger.info(f"Fetching Last.fm recommendations feed from {url}") + response = None try: - response = requests.get(url, headers=headers, proxies=proxies, timeout=20) + logger.info("Fetching Last.fm feed directly (no proxy)") + response = cf_requests.get(url, headers=headers, impersonate="chrome", timeout=20) response.raise_for_status() except Exception as e: - logger.error(f"Failed to fetch Last.fm recommendations feed: {e}") - return + logger.warning(f"Failed to fetch Last.fm feed directly: {e}. Retrying with proxy...") + if PROXY_URL: + proxies = {"http": PROXY_URL, "https": PROXY_URL} + try: + response = cf_requests.get(url, headers=headers, proxies=proxies, impersonate="chrome", timeout=20) + response.raise_for_status() + except Exception as e2: + logger.error(f"Failed to fetch Last.fm feed with proxy: {e2}") + return + else: + return try: import xml.etree.ElementTree as ET diff --git a/apps/musicseerr/tasks.py b/apps/musicseerr/tasks.py index 09f55c8..1f49211 100644 --- a/apps/musicseerr/tasks.py +++ b/apps/musicseerr/tasks.py @@ -9,11 +9,35 @@ from bs4 import BeautifulSoup from urllib.parse import urljoin from mutagen.mp3 import EasyMP3 from mutagen.id3 import ID3 +from curl_cffi import requests as cf_requests # Configure Logger logging.basicConfig(level=logging.INFO) logger = logging.getLogger("musicseerr-tasks") +def move_new_files_to_dest(temp_dir: str, dest_dir: str) -> list: + moved = [] + if not os.path.exists(temp_dir): + return moved + os.makedirs(dest_dir, exist_ok=True) + for file_name in os.listdir(temp_dir): + src_path = os.path.join(temp_dir, file_name) + # Check if it is a regular file and not a symlink + if os.path.isfile(src_path) and not os.path.islink(src_path): + dest_path = os.path.join(dest_dir, file_name) + logger.info(f"Moving new file '{file_name}' to remote Seedbox directory: {dest_path}") + try: + shutil.move(src_path, dest_path) + moved.append(file_name) + # Recreate as symlink in temp_dir so subsequent tools can find it if needed + try: + os.symlink(dest_path, src_path) + except Exception as sym_err: + logger.warning(f"Failed to recreate symlink for {file_name}: {sym_err}") + except Exception as move_err: + logger.error(f"Failed to move file {file_name} to {dest_dir}: {move_err}") + return moved + def scrape_tubidy_link(url: str, proxy_url: str) -> str: """ Scrapes a Tubidy page to find the direct MP3 download URL. @@ -24,7 +48,7 @@ def scrape_tubidy_link(url: str, proxy_url: str) -> str: } logger.info(f"Fetching Tubidy page: {url} via proxy: {proxy_url}") - response = requests.get(url, headers=headers, proxies=proxies, timeout=20) + response = cf_requests.get(url, headers=headers, proxies=proxies, impersonate="chrome", timeout=20) response.raise_for_status() soup = BeautifulSoup(response.text, "html.parser") @@ -51,7 +75,7 @@ def scrape_tubidy_link(url: str, proxy_url: str) -> str: sub_url = urljoin(url, a["href"]) logger.info(f"Following nested Tubidy download link: {sub_url}") try: - sub_resp = requests.get(sub_url, headers=headers, proxies=proxies, timeout=15) + sub_resp = cf_requests.get(sub_url, headers=headers, proxies=proxies, impersonate="chrome", timeout=15) if sub_resp.status_code == 200: sub_soup = BeautifulSoup(sub_resp.text, "html.parser") for sub_a in sub_soup.find_all("a", href=True): @@ -88,7 +112,7 @@ def search_tubidy(query: str, proxy_url: str) -> list: logger.info(f"Searching Tubidy for '{query}' via proxy: {proxy_url}") try: - response = requests.get(search_url, headers=headers, params=params, proxies=proxies, timeout=15) + response = cf_requests.get(search_url, headers=headers, params=params, proxies=proxies, impersonate="chrome", timeout=15) response.raise_for_status() soup = BeautifulSoup(response.text, "html.parser") @@ -142,7 +166,7 @@ def get_tubidy_download_link(content_id: str, proxy_url: str) -> str: logger.info(f"Fetching watch.php for content ID '{content_id}'") try: - response = requests.get(watch_url, headers=headers, params=params, proxies=proxies, timeout=15) + response = cf_requests.get(watch_url, headers=headers, params=params, proxies=proxies, impersonate="chrome", timeout=15) response.raise_for_status() soup = BeautifulSoup(response.text, "html.parser") @@ -175,7 +199,7 @@ def download_file_stream(url: str, dest_path: str, proxy_url: str): } logger.info(f"Streaming download from {url} to {dest_path}") - response = requests.get(url, headers=headers, proxies=proxies, stream=True, timeout=60) + response = cf_requests.get(url, headers=headers, proxies=proxies, impersonate="chrome", stream=True, timeout=60) response.raise_for_status() with open(dest_path, "wb") as f: @@ -246,7 +270,7 @@ def extract_spotify_track_title(url: str, proxy_url: str) -> str: logger.info(f"Scraping Spotify track via embed URL: {embed_url}") try: - response = requests.get(embed_url, headers=headers, proxies=proxies, timeout=15) + response = cf_requests.get(embed_url, headers=headers, proxies=proxies, impersonate="chrome", timeout=15) if response.status_code == 200: next_data_match = re.search(r'', response.text) if next_data_match: @@ -287,7 +311,7 @@ def scrape_spotify_playlist(url: str, proxy_url: str) -> list: try: # Short jitter to avoid hitting Spotify exactly at the same time time.sleep(1) - response = requests.get(embed_url, headers=headers, proxies=current_proxies, timeout=15) + response = cf_requests.get(embed_url, headers=headers, proxies=current_proxies, impersonate="chrome", timeout=15) if response.status_code == 200: next_data_match = re.search(r'', response.text) if next_data_match: @@ -488,6 +512,11 @@ def run_download_task(task_id: str, query: str, proxy_url: str, jobs_dict: dict, if track_success: download_count += 1 + new_moved = move_new_files_to_dest(temp_dir, "/remote-music/music") + if "files" not in jobs_dict[task_id] or not jobs_dict[task_id]["files"]: + jobs_dict[task_id]["files"] = [] + jobs_dict[task_id]["files"].extend(new_moved) + logger.info(f"[{task_id}] Successfully downloaded and moved {len(new_moved)} files for track '{track_query}'") if download_count > 0: download_successful = True @@ -638,38 +667,25 @@ def run_download_task(task_id: str, query: str, proxy_url: str, jobs_dict: dict, if not download_successful: raise Exception("Ingestion finished but no download was successful.") - downloaded_files = os.listdir(temp_dir) - actual_files = [f for f in downloaded_files if not os.path.islink(os.path.join(temp_dir, f))] - - if not actual_files: - if downloaded_files: - logger.info(f"[{task_id}] All tracks in this query were already downloaded (skipped duplicates).") + if download_successful and not is_playlist: + new_moved = move_new_files_to_dest(temp_dir, "/remote-music/music") + if "files" not in jobs_dict[task_id] or not jobs_dict[task_id]["files"]: jobs_dict[task_id]["files"] = [] - jobs_dict[task_id]["status"] = "completed" - if chat_id: - send_telegram_notification(chat_id, f"✅ Successfully ingested request: '{query}'\n\n(All tracks already existed - skipped duplicates)") - return - else: - raise Exception("Ingestion finished successfully but no files were found in local temp folder.") + jobs_dict[task_id]["files"].extend(new_moved) - os.makedirs("/remote-music/music", exist_ok=True) - moved_files = [] - - for file_name in actual_files: - src_file = os.path.join(temp_dir, file_name) - dest_file = os.path.join("/remote-music/music", file_name) + if "files" not in jobs_dict[task_id] or not jobs_dict[task_id]["files"]: + logger.info(f"[{task_id}] All tracks in this query were already downloaded (skipped duplicates).") + jobs_dict[task_id]["files"] = [] + jobs_dict[task_id]["status"] = "completed" + if chat_id: + send_telegram_notification(chat_id, f"✅ Successfully ingested request: '{query}'\n\n(All tracks already existed - skipped duplicates)") + return - logger.info(f"[{task_id}] Moving new file '{file_name}' to remote Seedbox directory") - shutil.move(src_file, dest_file) - moved_files.append(file_name) - - - jobs_dict[task_id]["files"] = moved_files jobs_dict[task_id]["status"] = "completed" logger.info(f"[{task_id}] Music Ingestion task fully completed.") if chat_id: - files_str = "\n".join(moved_files) + files_str = "\n".join(jobs_dict[task_id]["files"]) send_telegram_notification(chat_id, f"✅ Successfully ingested request: '{query}'\n\nFiles imported to Navidrome:\n{files_str}") except Exception as e: