832 lines
38 KiB
Python
832 lines
38 KiB
Python
import os
|
|
import shutil
|
|
import subprocess
|
|
import logging
|
|
import re
|
|
import requests
|
|
import json
|
|
import threading
|
|
from bs4 import BeautifulSoup
|
|
from urllib.parse import urljoin
|
|
from mutagen.mp3 import EasyMP3
|
|
from mutagen.id3 import ID3
|
|
from curl_cffi import requests as cf_requests
|
|
|
|
DOWNLOAD_SEMAPHORE = threading.Semaphore(5)
|
|
|
|
# Configure Logger
|
|
logging.basicConfig(level=logging.INFO)
|
|
logger = logging.getLogger("musicseerr-tasks")
|
|
|
|
JOBS_FILE = "/remote-music/.musicseerr_jobs.json"
|
|
|
|
JOBS_WRITE_LOCK = threading.Lock()
|
|
|
|
def save_jobs(jobs_dict):
|
|
with JOBS_WRITE_LOCK:
|
|
try:
|
|
import uuid
|
|
temp_file = f"{JOBS_FILE}.{uuid.uuid4()}.tmp"
|
|
with open(temp_file, "w") as f:
|
|
json.dump(jobs_dict, f, indent=2)
|
|
os.replace(temp_file, JOBS_FILE)
|
|
except Exception as e:
|
|
logger.error(f"Failed to save jobs database: {e}")
|
|
|
|
|
|
def move_new_files_to_dest(temp_dir: str, dest_dir: str) -> list:
|
|
moved = []
|
|
if not os.path.exists(temp_dir):
|
|
return moved
|
|
os.makedirs(dest_dir, exist_ok=True)
|
|
for file_name in os.listdir(temp_dir):
|
|
src_path = os.path.join(temp_dir, file_name)
|
|
# Check if it is a regular file and not a symlink
|
|
if os.path.isfile(src_path) and not os.path.islink(src_path):
|
|
dest_path = os.path.join(dest_dir, file_name)
|
|
logger.info(f"Moving new file '{file_name}' to remote Seedbox directory: {dest_path}")
|
|
try:
|
|
shutil.move(src_path, dest_path)
|
|
moved.append(file_name)
|
|
# Recreate as symlink in temp_dir so subsequent tools can find it if needed
|
|
try:
|
|
os.symlink(dest_path, src_path)
|
|
except Exception as sym_err:
|
|
logger.warning(f"Failed to recreate symlink for {file_name}: {sym_err}")
|
|
except Exception as move_err:
|
|
logger.error(f"Failed to move file {file_name} to {dest_dir}: {move_err}")
|
|
return moved
|
|
|
|
def scrape_tubidy_link(url: str, proxy_url: str) -> str:
|
|
"""
|
|
Scrapes a Tubidy page to find the direct MP3 download URL.
|
|
"""
|
|
proxies = {"http": proxy_url, "https": proxy_url} if proxy_url else None
|
|
headers = {
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
|
}
|
|
|
|
logger.info(f"Fetching Tubidy page: {url} via proxy: {proxy_url}")
|
|
response = cf_requests.get(url, headers=headers, proxies=proxies, impersonate="chrome", timeout=20)
|
|
response.raise_for_status()
|
|
|
|
soup = BeautifulSoup(response.text, "html.parser")
|
|
|
|
# 1. Search for direct mp3 links in audio tags
|
|
audio_tag = soup.find("audio")
|
|
if audio_tag:
|
|
source = audio_tag.find("source")
|
|
if source and source.get("src"):
|
|
return urljoin(url, source["src"])
|
|
if audio_tag.get("src"):
|
|
return urljoin(url, audio_tag["src"])
|
|
|
|
# 2. Search for <a> tags containing .mp3 in href
|
|
for a in soup.find_all("a", href=True):
|
|
href = a["href"]
|
|
if ".mp3" in href.lower():
|
|
return urljoin(url, href)
|
|
|
|
# 3. Follow potential download/mp3 buttons if nested
|
|
for a in soup.find_all("a", href=True):
|
|
text = a.text.lower()
|
|
if "mp3" in text or "download" in text:
|
|
sub_url = urljoin(url, a["href"])
|
|
logger.info(f"Following nested Tubidy download link: {sub_url}")
|
|
try:
|
|
sub_resp = cf_requests.get(sub_url, headers=headers, proxies=proxies, impersonate="chrome", timeout=15)
|
|
if sub_resp.status_code == 200:
|
|
sub_soup = BeautifulSoup(sub_resp.text, "html.parser")
|
|
for sub_a in sub_soup.find_all("a", href=True):
|
|
sub_href = sub_a["href"]
|
|
if ".mp3" in sub_href.lower():
|
|
return urljoin(sub_url, sub_href)
|
|
except Exception as e:
|
|
logger.warning(f"Failed to fetch nested link {sub_url}: {e}")
|
|
|
|
# 4. Fallback: search raw text for mp3 URLs
|
|
mp3_matches = re.findall(r'https?://[^\s"\'>]+\.mp3', response.text)
|
|
if mp3_matches:
|
|
return mp3_matches[0]
|
|
|
|
raise ValueError("Could not find any MP3 download link on the Tubidy page.")
|
|
|
|
|
|
def search_tubidy(query: str, proxy_url: str) -> list:
|
|
"""
|
|
Searches Tubidy for the query and returns a list of results with ID, title, and link.
|
|
Includes robust fallback variations if the initial search returns no results.
|
|
"""
|
|
proxies = {"http": proxy_url, "https": proxy_url} if proxy_url else None
|
|
headers = {
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
|
}
|
|
|
|
endpoint = "https://mp3.tubidy.cool"
|
|
search_url = f"{endpoint}/search.php"
|
|
params = {
|
|
"q": query,
|
|
"si": 7,
|
|
"pn": 1
|
|
}
|
|
|
|
logger.info(f"Searching Tubidy for '{query}' via proxy: {proxy_url}")
|
|
try:
|
|
response = cf_requests.get(search_url, headers=headers, params=params, proxies=proxies, impersonate="chrome", timeout=15)
|
|
response.raise_for_status()
|
|
|
|
def parse_results(html: str) -> list:
|
|
soup = BeautifulSoup(html, "html.parser")
|
|
results_list = []
|
|
for media_body in soup.find_all("div", class_="media-body"):
|
|
a_tag = media_body.find("a")
|
|
if a_tag:
|
|
href = a_tag.get("href")
|
|
if href:
|
|
title = a_tag.get("aria-label") or a_tag.text.strip()
|
|
match = re.search(r'/watch/([^/]+)', href)
|
|
content_id = match.group(1) if match else None
|
|
link = href
|
|
if href.startswith("//"):
|
|
link = f"https:{href}"
|
|
elif href.startswith("/"):
|
|
link = f"{endpoint}{href}"
|
|
|
|
if content_id:
|
|
results_list.append({
|
|
"id": content_id,
|
|
"title": title,
|
|
"link": link
|
|
})
|
|
return results_list
|
|
|
|
results = parse_results(response.text)
|
|
if results:
|
|
return results
|
|
|
|
# Fallback query variations if first search yielded no results
|
|
variations = []
|
|
|
|
# Variation 1: Clean basic symbols and parentheticals
|
|
clean1 = re.sub(r'[\(\[][^\)\]]*[\)\]]', '', query)
|
|
clean1 = clean1.replace("-", " ")
|
|
clean1 = re.sub(r'\s+', ' ', clean1).strip()
|
|
variations.append(clean1)
|
|
|
|
# Variation 2: Clean and also remove feat/ft
|
|
clean2 = re.sub(r'(?i)\b(feat|ft)\.?\b.*$', '', clean1)
|
|
clean2 = re.sub(r'[^\w\s]', '', clean2)
|
|
clean2 = re.sub(r'\s+', ' ', clean2).strip()
|
|
variations.append(clean2)
|
|
|
|
# Variation 3: Handle censored words (e.g. fucked -> f*cked, fuck -> f*ck)
|
|
if "fucked" in clean2.lower() or "fuck" in clean2.lower():
|
|
clean3 = re.sub(r'(?i)\bfucked\b', 'f*cked', clean2)
|
|
clean3 = re.sub(r'(?i)\bfuck\b', 'f*ck', clean3)
|
|
variations.append(clean3)
|
|
|
|
# Variation 4: Try just the artist and the first 2 words of the title
|
|
parts = query.split(" - ", 1)
|
|
if len(parts) == 2:
|
|
artist = re.sub(r'[^\w\s]', '', parts[0]).strip()
|
|
title_words = re.sub(r'[^\w\s]', '', parts[1]).strip().split()
|
|
if len(title_words) > 1:
|
|
clean4 = f"{artist} {' '.join(title_words[:1])}"
|
|
variations.append(clean4)
|
|
|
|
# Try each variation in order
|
|
for var in variations:
|
|
var = var.strip()
|
|
if not var or var == query:
|
|
continue
|
|
logger.info(f"Retrying Tubidy search with variation: '{var}'")
|
|
params["q"] = var
|
|
try:
|
|
response = cf_requests.get(search_url, headers=headers, params=params, proxies=proxies, impersonate="chrome", timeout=15)
|
|
response.raise_for_status()
|
|
new_results = parse_results(response.text)
|
|
if new_results:
|
|
logger.info(f"Found {len(new_results)} results for variation '{var}'")
|
|
return new_results
|
|
except Exception as e:
|
|
logger.warning(f"Variation search for '{var}' failed: {e}")
|
|
|
|
return []
|
|
except Exception as e:
|
|
logger.error(f"Failed to search Tubidy: {e}")
|
|
return []
|
|
|
|
|
|
def get_tubidy_download_link(content_id: str, proxy_url: str) -> str:
|
|
"""
|
|
Queries watch.php for the given content ID to find the direct MP3 download URL.
|
|
Supports initiating and polling the conversion process if the video is not yet processed.
|
|
"""
|
|
proxies = {"http": proxy_url, "https": proxy_url} if proxy_url else None
|
|
headers = {
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
|
}
|
|
|
|
endpoint = "https://mp3.tubidy.cool"
|
|
watch_url = f"{endpoint}/watch.php"
|
|
params = {
|
|
"id": content_id,
|
|
"p": "mp4",
|
|
"lnk": 6,
|
|
"act": "down",
|
|
"t": "ssl"
|
|
}
|
|
|
|
logger.info(f"Fetching watch.php for content ID '{content_id}'")
|
|
try:
|
|
session = cf_requests.Session()
|
|
if proxies:
|
|
session.proxies = proxies
|
|
session.headers.update(headers)
|
|
|
|
response = session.get(watch_url, params=params, impersonate="chrome", timeout=15)
|
|
response.raise_for_status()
|
|
|
|
def extract_link(html: str) -> str:
|
|
soup = BeautifulSoup(html, "html.parser")
|
|
# 1. Search in list-group-item big tags
|
|
for li in soup.find_all("li", class_=lambda x: x and "list-group-item" in x and "big" in x):
|
|
a_tag = li.find("a")
|
|
if a_tag:
|
|
text = a_tag.text.lower()
|
|
href = a_tag.get("href")
|
|
if href and "download" in text:
|
|
return urljoin(watch_url, href)
|
|
# 2. General fallback search for direct download/mp3 links
|
|
for a_tag in soup.find_all("a", href=True):
|
|
text = a_tag.text.lower()
|
|
href = a_tag["href"]
|
|
if "download" in text and "act=process" not in href and "act=down" not in href:
|
|
return urljoin(watch_url, href)
|
|
if (".mp3" in href.lower() or "video.mp3.tubidy.cool" in href.lower()) and "act=process" not in href:
|
|
return urljoin(watch_url, href)
|
|
return ""
|
|
|
|
link = extract_link(response.text)
|
|
if link:
|
|
return link
|
|
|
|
# If no link found, check if there is a process link
|
|
soup = BeautifulSoup(response.text, "html.parser")
|
|
process_link = None
|
|
for a_tag in soup.find_all("a", href=True):
|
|
if "act=process" in a_tag["href"]:
|
|
process_link = urljoin(watch_url, a_tag["href"])
|
|
break
|
|
|
|
if process_link:
|
|
logger.info(f"Triggering Tubidy processing step for content ID '{content_id}' via {process_link}")
|
|
proc_resp = session.get(process_link, impersonate="chrome", timeout=15)
|
|
proc_resp.raise_for_status()
|
|
|
|
# Wait for conversion (typically takes 5-10s on Tubidy server)
|
|
import time
|
|
wait_time = 10
|
|
logger.info(f"Waiting {wait_time} seconds for conversion to finish...")
|
|
time.sleep(wait_time)
|
|
|
|
# Re-fetch the download page
|
|
response = session.get(watch_url, params=params, impersonate="chrome", timeout=15)
|
|
response.raise_for_status()
|
|
link = extract_link(response.text)
|
|
if link:
|
|
return link
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to fetch download link for {content_id}: {e}")
|
|
|
|
raise ValueError("Could not find download link on watch.php page.")
|
|
|
|
|
|
def download_file_stream(url: str, dest_path: str, proxy_url: str):
|
|
"""
|
|
Downloads a file in chunks using requests through the proxy.
|
|
"""
|
|
proxies = {"http": proxy_url, "https": proxy_url} if proxy_url else None
|
|
headers = {
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
|
}
|
|
|
|
logger.info(f"Streaming download from {url} to {dest_path}")
|
|
response = cf_requests.get(url, headers=headers, proxies=proxies, impersonate="chrome", stream=True, timeout=60)
|
|
response.raise_for_status()
|
|
|
|
with open(dest_path, "wb") as f:
|
|
for chunk in response.iter_content(chunk_size=8192):
|
|
if chunk:
|
|
f.write(chunk)
|
|
|
|
|
|
def tag_mp3(file_path: str, filename: str):
|
|
"""
|
|
Parses the filename to extract artist/title and injects ID3 tags using Mutagen.
|
|
"""
|
|
base_name = os.path.splitext(os.path.basename(file_path))[0]
|
|
artist = "Unknown Artist"
|
|
title = base_name
|
|
|
|
# Try parsing "Artist - Title" format
|
|
if " - " in base_name:
|
|
parts = base_name.split(" - ", 1)
|
|
artist = parts[0].strip()
|
|
title = parts[1].strip()
|
|
|
|
logger.info(f"Applying tags - Title: '{title}', Artist: '{artist}' to {file_path}")
|
|
|
|
# Initialize ID3 tags if they do not exist
|
|
try:
|
|
audio = EasyMP3(file_path)
|
|
except Exception:
|
|
id3 = ID3()
|
|
id3.save(file_path)
|
|
audio = EasyMP3(file_path)
|
|
|
|
audio["title"] = title
|
|
audio["artist"] = artist
|
|
audio["album"] = "Tubidy Ingestion"
|
|
audio.save()
|
|
|
|
|
|
def send_telegram_notification(chat_id: int, text: str):
|
|
"""
|
|
Sends a message back to Telegram.
|
|
"""
|
|
token = os.getenv("TELEGRAM_BOT_TOKEN")
|
|
if not token or not chat_id:
|
|
return
|
|
url = f"https://api.telegram.org/bot{token}/sendMessage"
|
|
try:
|
|
requests.post(url, json={"chat_id": chat_id, "text": text}, timeout=10)
|
|
except Exception as e:
|
|
logger.error(f"Failed to send Telegram notification: {e}")
|
|
|
|
|
|
def extract_spotify_track_title(url: str, proxy_url: str) -> str:
|
|
"""
|
|
Scrapes a Spotify track embed page and returns the track search string ("Artist - Title").
|
|
"""
|
|
match = re.search(r'track/([a-zA-Z0-9]+)', url)
|
|
if not match:
|
|
logger.warning(f"Could not extract track ID from URL: {url}")
|
|
return ""
|
|
track_id = match.group(1)
|
|
embed_url = f"https://open.spotify.com/embed/track/{track_id}"
|
|
|
|
proxies = {"http": proxy_url, "https": proxy_url} if proxy_url else None
|
|
headers = {
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
|
}
|
|
|
|
logger.info(f"Scraping Spotify track via embed URL: {embed_url}")
|
|
try:
|
|
response = cf_requests.get(embed_url, headers=headers, proxies=proxies, impersonate="chrome", timeout=15)
|
|
if response.status_code == 200:
|
|
next_data_match = re.search(r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>', response.text)
|
|
if next_data_match:
|
|
data = json.loads(next_data_match.group(1))
|
|
entity = data['props']['pageProps']['state']['data']['entity']
|
|
name = entity.get('name', '').strip()
|
|
artists = [a.get('name', '') for a in entity.get('artists', [])]
|
|
artists = [a for a in artists if a]
|
|
if name and artists:
|
|
return f"{', '.join(artists)} - {name}"
|
|
elif name:
|
|
return name
|
|
except Exception as e:
|
|
logger.warning(f"Failed to extract Spotify track title: {e}")
|
|
return ""
|
|
|
|
|
|
def scrape_spotify_playlist(url: str, proxy_url: str) -> list:
|
|
"""
|
|
Scrapes a Spotify playlist embed page and returns a list of track search strings ("Artist - Title").
|
|
"""
|
|
match = re.search(r'playlist/([a-zA-Z0-9]+)', url)
|
|
if not match:
|
|
logger.warning(f"Could not extract playlist ID from URL: {url}")
|
|
return []
|
|
playlist_id = match.group(1)
|
|
embed_url = f"https://open.spotify.com/embed/playlist/{playlist_id}"
|
|
|
|
headers = {
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
|
}
|
|
import time
|
|
# Try with proxy first, then without proxy
|
|
for use_proxy in [True, False]:
|
|
current_proxies = {"http": proxy_url, "https": proxy_url} if (use_proxy and proxy_url) else None
|
|
proxy_desc = f"via proxy {proxy_url}" if current_proxies else "directly"
|
|
logger.info(f"Scraping Spotify playlist via embed URL: {embed_url} ({proxy_desc})")
|
|
try:
|
|
# Short jitter to avoid hitting Spotify exactly at the same time
|
|
time.sleep(1)
|
|
response = cf_requests.get(embed_url, headers=headers, proxies=current_proxies, impersonate="chrome", timeout=15)
|
|
if response.status_code == 200:
|
|
next_data_match = re.search(r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>', response.text)
|
|
if next_data_match:
|
|
data = json.loads(next_data_match.group(1))
|
|
page_props = data.get('props', {}).get('pageProps', {})
|
|
if 'state' not in page_props:
|
|
logger.warning(f"Spotify embed pageProps does not contain 'state' ({proxy_desc}). HTML snippet: {response.text[:250]}")
|
|
continue
|
|
entity = page_props['state']['data']['entity']
|
|
tracks = entity.get('tracks', entity.get('trackList', []))
|
|
track_queries = []
|
|
for t in tracks:
|
|
title = t.get('title', '').strip()
|
|
subtitle = t.get('subtitle', '').replace('\xa0', ' ').strip()
|
|
if title and subtitle:
|
|
track_queries.append(f"{subtitle} - {title}")
|
|
elif title:
|
|
track_queries.append(title)
|
|
return track_queries
|
|
logger.warning(f"Failed to scrape Spotify playlist {proxy_desc}: HTTP {response.status_code}")
|
|
except Exception as e:
|
|
logger.warning(f"Failed to scrape Spotify playlist {proxy_desc}: {e}")
|
|
|
|
return []
|
|
|
|
|
|
def symlink_existing_tracks(temp_dir: str):
|
|
"""
|
|
Finds all audio files recursively in /remote-music/music, extracts artist and title,
|
|
and creates symlinks named "Artist - Title.mp3" in temp_dir so spotDL skips them.
|
|
"""
|
|
remote_base = "/remote-music/music"
|
|
if not os.path.isdir(remote_base):
|
|
return
|
|
|
|
logger.info(f"Scanning {remote_base} to symlink existing tracks in {temp_dir}...")
|
|
count = 0
|
|
for root, dirs, files in os.walk(remote_base):
|
|
for file in files:
|
|
if file.lower().endswith((".mp3", ".flac", ".m4a", ".ogg", ".wav", ".opus")):
|
|
# Try to clean/extract artist and title
|
|
base_name = os.path.splitext(file)[0]
|
|
|
|
# Strip leading numbers like "01. ", "01 - ", etc.
|
|
cleaned = re.sub(r'^\d+[\s.-]+', '', base_name).strip()
|
|
|
|
# We expect the file to have a " - " separator
|
|
if " - " in cleaned:
|
|
parts = cleaned.split(" - ", 1)
|
|
artist = parts[0].strip()
|
|
title = parts[1].strip()
|
|
|
|
# Target filename that spotDL expects
|
|
target_name = f"{artist} - {title}.mp3"
|
|
else:
|
|
# Fallback to direct name with .mp3 extension
|
|
target_name = f"{cleaned}.mp3"
|
|
|
|
# Replaces invalid chars
|
|
target_name = "".join([c for c in target_name if c.isalnum() or c in " .-_()"]).strip()
|
|
if not target_name.endswith(".mp3"):
|
|
target_name += ".mp3"
|
|
|
|
dest_path = os.path.join(temp_dir, target_name)
|
|
if not os.path.exists(dest_path):
|
|
try:
|
|
src_path = os.path.join(root, file)
|
|
os.symlink(src_path, dest_path)
|
|
count += 1
|
|
except Exception as e:
|
|
logger.warning(f"Failed to create symlink for {file} -> {target_name}: {e}")
|
|
|
|
logger.info(f"Created {count} symlinks for existing tracks.")
|
|
|
|
|
|
def run_download_task(task_id: str, query: str, proxy_url: str, jobs_dict: dict, chat_id: int = None):
|
|
with DOWNLOAD_SEMAPHORE:
|
|
_run_download_task_impl(task_id, query, proxy_url, jobs_dict, chat_id)
|
|
|
|
|
|
def _run_download_task_impl(task_id: str, query: str, proxy_url: str, jobs_dict: dict, chat_id: int = None):
|
|
"""
|
|
Core download manager task that runs in the background.
|
|
"""
|
|
jobs_dict[task_id]["status"] = "downloading"
|
|
save_jobs(jobs_dict)
|
|
|
|
temp_dir = f"/tmp/downloads/{task_id}"
|
|
os.makedirs(temp_dir, exist_ok=True)
|
|
symlink_existing_tracks(temp_dir)
|
|
|
|
# Resolve proxy hostname to IP address for spotDL compatibility (spotDL requires IP in proxy URL)
|
|
resolved_proxy_url = proxy_url
|
|
if proxy_url:
|
|
try:
|
|
from urllib.parse import urlparse
|
|
import socket
|
|
parsed = urlparse(proxy_url)
|
|
if parsed.hostname and not parsed.hostname.replace('.', '').isdigit():
|
|
ip = socket.gethostbyname(parsed.hostname)
|
|
netloc = ip
|
|
if parsed.port:
|
|
netloc = f"{ip}:{parsed.port}"
|
|
if parsed.username or parsed.password:
|
|
auth = ""
|
|
if parsed.username:
|
|
auth += parsed.username
|
|
if parsed.password:
|
|
auth += f":{parsed.password}"
|
|
netloc = f"{auth}@{netloc}"
|
|
resolved_proxy_url = parsed._replace(netloc=netloc).geturl()
|
|
logger.info(f"Resolved proxy hostname '{parsed.hostname}' to IP '{ip}'. New proxy URL: '{resolved_proxy_url}'")
|
|
except Exception as e:
|
|
logger.error(f"Failed to resolve proxy hostname: {e}")
|
|
|
|
try:
|
|
download_successful = False
|
|
|
|
# If query is a Spotify track URL, resolve it to a text title first for fallback options
|
|
resolved_title = ""
|
|
if "spotify.com/track/" in query.lower():
|
|
resolved_title = extract_spotify_track_title(query, resolved_proxy_url)
|
|
if resolved_title:
|
|
logger.info(f"[{task_id}] Pre-resolved Spotify URL to text title: '{resolved_title}'")
|
|
query = resolved_title
|
|
|
|
is_playlist = "spotify.com/playlist/" in query.lower()
|
|
|
|
if is_playlist:
|
|
jobs_dict[task_id]["priority_used"] = "playlist-scraper"
|
|
save_jobs(jobs_dict)
|
|
logger.info(f"[{task_id}] Spotify playlist detected: '{query}'")
|
|
track_queries = scrape_spotify_playlist(query, resolved_proxy_url)
|
|
if not track_queries:
|
|
raise Exception("Could not scrape any tracks from the Spotify playlist embed page.")
|
|
|
|
logger.info(f"[{task_id}] Successfully scraped {len(track_queries)} tracks from playlist.")
|
|
download_count = 0
|
|
|
|
for idx, track_query in enumerate(track_queries):
|
|
logger.info(f"[{task_id}] [Track {idx+1}/{len(track_queries)}] Downloading '{track_query}'")
|
|
track_success = False
|
|
|
|
# 1. spotDL
|
|
env = os.environ.copy()
|
|
if resolved_proxy_url:
|
|
env["http_proxy"] = resolved_proxy_url
|
|
env["https_proxy"] = resolved_proxy_url
|
|
|
|
cmd = ["spotdl", "download", track_query]
|
|
if resolved_proxy_url:
|
|
cmd += ["--proxy", resolved_proxy_url]
|
|
|
|
logger.info(f"[{task_id}] [Track {idx+1}] Trying spotDL...")
|
|
result = subprocess.run(cmd, cwd=temp_dir, env=env, capture_output=True, text=True)
|
|
has_error = "AudioProviderError" in result.stdout or "AudioProviderError" in result.stderr or "LookupError" in result.stdout or "LookupError" in result.stderr or "JSONDecodeError" in result.stdout or "JSONDecodeError" in result.stderr
|
|
if result.returncode == 0 and not has_error:
|
|
logger.info(f"[{task_id}] [Track {idx+1}] spotDL succeeded.")
|
|
track_success = True
|
|
else:
|
|
logger.warning(f"[{task_id}] [Track {idx+1}] spotDL failed: {result.stderr or result.stdout}. Trying yt-dlp...")
|
|
|
|
# 2. yt-dlp
|
|
if not track_success:
|
|
search_query = f"ytsearch1:{track_query}"
|
|
cmd_ytdlp = [
|
|
"yt-dlp",
|
|
"-x",
|
|
"--audio-format", "mp3",
|
|
"--embed-metadata",
|
|
"--yes-playlist",
|
|
"--output", "%(title)s.%(ext)s"
|
|
]
|
|
if resolved_proxy_url:
|
|
cmd_ytdlp += ["--proxy", resolved_proxy_url]
|
|
cmd_ytdlp.append(search_query)
|
|
|
|
logger.info(f"[{task_id}] [Track {idx+1}] Trying yt-dlp...")
|
|
result_ytdlp = subprocess.run(cmd_ytdlp, cwd=temp_dir, env=env, capture_output=True, text=True)
|
|
if result_ytdlp.returncode == 0:
|
|
logger.info(f"[{task_id}] [Track {idx+1}] yt-dlp succeeded.")
|
|
track_success = True
|
|
else:
|
|
logger.warning(f"[{task_id}] [Track {idx+1}] yt-dlp failed. Trying Tubidy...")
|
|
|
|
# 3. Tubidy
|
|
if not track_success:
|
|
try:
|
|
logger.info(f"[{task_id}] [Track {idx+1}] Trying Tubidy...")
|
|
search_results = search_tubidy(track_query, resolved_proxy_url)
|
|
if search_results:
|
|
first_result = search_results[0]
|
|
mp3_url = get_tubidy_download_link(first_result["id"], resolved_proxy_url)
|
|
filename = f"{first_result['title']}.mp3"
|
|
filename = "".join([c for c in filename if c.isalnum() or c in " .-_()"]).strip()
|
|
if not filename.endswith(".mp3"):
|
|
filename += ".mp3"
|
|
dest_path = os.path.join(temp_dir, filename)
|
|
download_file_stream(mp3_url, dest_path, resolved_proxy_url)
|
|
tag_mp3(dest_path, filename)
|
|
track_success = True
|
|
logger.info(f"[{task_id}] [Track {idx+1}] Tubidy succeeded.")
|
|
except Exception as e:
|
|
logger.error(f"[{task_id}] [Track {idx+1}] Tubidy failed: {e}")
|
|
|
|
if track_success:
|
|
download_count += 1
|
|
new_moved = move_new_files_to_dest(temp_dir, "/remote-music/music")
|
|
if "files" not in jobs_dict[task_id] or not jobs_dict[task_id]["files"]:
|
|
jobs_dict[task_id]["files"] = []
|
|
jobs_dict[task_id]["files"].extend(new_moved)
|
|
save_jobs(jobs_dict)
|
|
logger.info(f"[{task_id}] Successfully downloaded and moved {len(new_moved)} files for track '{track_query}'")
|
|
|
|
if download_count > 0:
|
|
download_successful = True
|
|
logger.info(f"[{task_id}] Playlist download finished. Successfully downloaded {download_count} tracks.")
|
|
else:
|
|
raise Exception("Failed to download any tracks from the playlist.")
|
|
|
|
else:
|
|
is_tubidy = "tubidy" in query.lower()
|
|
is_spotify = "spotify.com" in query.lower()
|
|
is_youtube = "youtube.com" in query.lower() or "youtu.be" in query.lower()
|
|
|
|
# Priority 1: spotDL (text search or Spotify URL)
|
|
if not is_tubidy and not is_youtube:
|
|
jobs_dict[task_id]["priority_used"] = "spotDL"
|
|
save_jobs(jobs_dict)
|
|
logger.info(f"[{task_id}] Attempting Priority 1: spotDL for '{query}'")
|
|
|
|
env = os.environ.copy()
|
|
if resolved_proxy_url:
|
|
env["http_proxy"] = resolved_proxy_url
|
|
env["https_proxy"] = resolved_proxy_url
|
|
|
|
cmd = ["spotdl", "download", query]
|
|
if resolved_proxy_url:
|
|
cmd += ["--proxy", resolved_proxy_url]
|
|
|
|
# Run spotDL in the temp directory
|
|
result = subprocess.run(cmd, cwd=temp_dir, env=env, capture_output=True, text=True)
|
|
has_error = "AudioProviderError" in result.stdout or "AudioProviderError" in result.stderr or "LookupError" in result.stdout or "LookupError" in result.stderr or "JSONDecodeError" in result.stdout or "JSONDecodeError" in result.stderr
|
|
if result.returncode == 0 and not has_error:
|
|
logger.info(f"[{task_id}] spotDL download succeeded.")
|
|
download_successful = True
|
|
else:
|
|
logger.warning(f"[{task_id}] spotDL failed: {result.stderr or result.stdout}. Falling back to yt-dlp.")
|
|
# Clear state for P2 fallback
|
|
is_spotify = False
|
|
|
|
# Priority 2: yt-dlp (direct YouTube or fallback for failed spotDL)
|
|
if not is_tubidy and not is_spotify:
|
|
jobs_dict[task_id]["priority_used"] = "yt-dlp"
|
|
save_jobs(jobs_dict)
|
|
logger.info(f"[{task_id}] Attempting Priority 2: yt-dlp for '{query}'")
|
|
|
|
env = os.environ.copy()
|
|
if resolved_proxy_url:
|
|
env["http_proxy"] = resolved_proxy_url
|
|
env["https_proxy"] = resolved_proxy_url
|
|
|
|
# If not a link or if resolved from Spotify track URL, treat as search query
|
|
search_query = query
|
|
if resolved_title:
|
|
search_query = f"ytsearch1:{resolved_title}"
|
|
logger.info(f"[{task_id}] Using resolved Spotify track title for yt-dlp search: '{search_query}'")
|
|
elif not query.startswith("http"):
|
|
search_query = f"ytsearch1:{query}"
|
|
|
|
cmd = [
|
|
"yt-dlp",
|
|
"-x",
|
|
"--audio-format", "mp3",
|
|
"--embed-metadata",
|
|
"--yes-playlist",
|
|
"--output", "%(title)s.%(ext)s"
|
|
]
|
|
if resolved_proxy_url:
|
|
cmd += ["--proxy", resolved_proxy_url]
|
|
cmd.append(search_query)
|
|
|
|
result = subprocess.run(cmd, cwd=temp_dir, env=env, capture_output=True, text=True)
|
|
if result.returncode == 0:
|
|
logger.info(f"[{task_id}] yt-dlp download succeeded.")
|
|
download_successful = True
|
|
else:
|
|
logger.warning(f"[{task_id}] yt-dlp failed: {result.stderr or result.stdout}. Falling back to Tubidy Search.")
|
|
|
|
# Priority 3: Custom Tubidy Link Scraper (if URL is directly passed)
|
|
if is_tubidy:
|
|
jobs_dict[task_id]["priority_used"] = "Tubidy"
|
|
save_jobs(jobs_dict)
|
|
logger.info(f"[{task_id}] Attempting Priority 3: Tubidy Scraper for '{query}'")
|
|
|
|
# Scrape MP3 URL
|
|
mp3_url = scrape_tubidy_link(query, resolved_proxy_url)
|
|
logger.info(f"[{task_id}] Scraped Tubidy MP3 URL: {mp3_url}")
|
|
|
|
# Formulate filename
|
|
filename = query.split("/")[-1]
|
|
if not filename or ".html" in filename or "?" in filename:
|
|
filename = "tubidy_download"
|
|
filename = filename.replace(".html", "").replace(".php", "").strip()
|
|
if not filename.endswith(".mp3"):
|
|
filename += ".mp3"
|
|
|
|
dest_path = os.path.join(temp_dir, filename)
|
|
|
|
# Download file stream
|
|
download_file_stream(mp3_url, dest_path, resolved_proxy_url)
|
|
logger.info(f"[{task_id}] Tubidy download completed.")
|
|
download_successful = True
|
|
|
|
# Tag metadata
|
|
tag_mp3(dest_path, filename)
|
|
logger.info(f"[{task_id}] Tubidy ID3 tagging complete.")
|
|
|
|
# Priority 4: Tubidy Search Fallback (if spotDL and yt-dlp failed and no files are present)
|
|
if not download_successful and not is_tubidy:
|
|
jobs_dict[task_id]["priority_used"] = "Tubidy-Search"
|
|
save_jobs(jobs_dict)
|
|
logger.info(f"[{task_id}] Attempting Fallback Priority 4: Tubidy Search Scraper for '{query}'")
|
|
|
|
search_query = query
|
|
if resolved_title:
|
|
search_query = resolved_title
|
|
logger.info(f"[{task_id}] Using pre-resolved Spotify track title for Tubidy search: '{search_query}'")
|
|
|
|
search_results = search_tubidy(search_query, resolved_proxy_url)
|
|
if search_results:
|
|
first_result = search_results[0]
|
|
logger.info(f"[{task_id}] Found Tubidy match: '{first_result['title']}' (ID: {first_result['id']})")
|
|
|
|
try:
|
|
mp3_url = get_tubidy_download_link(first_result["id"], resolved_proxy_url)
|
|
logger.info(f"[{task_id}] Resolved Tubidy download URL: {mp3_url}")
|
|
|
|
filename = f"{first_result['title']}.mp3"
|
|
# Clean filename
|
|
filename = "".join([c for c in filename if c.isalnum() or c in " .-_()"]).strip()
|
|
if not filename.endswith(".mp3"):
|
|
filename += ".mp3"
|
|
if not filename or filename == ".mp3":
|
|
filename = "tubidy_fallback.mp3"
|
|
|
|
dest_path = os.path.join(temp_dir, filename)
|
|
|
|
# Download file stream
|
|
download_file_stream(mp3_url, dest_path, resolved_proxy_url)
|
|
logger.info(f"[{task_id}] Tubidy fallback download completed.")
|
|
|
|
# Tag metadata
|
|
tag_mp3(dest_path, filename)
|
|
logger.info(f"[{task_id}] Tubidy fallback ID3 tagging complete.")
|
|
download_successful = True
|
|
except Exception as e:
|
|
logger.error(f"[{task_id}] Tubidy fallback download failed: {e}")
|
|
raise Exception(f"All download options (spotDL, yt-dlp, Tubidy) failed. Last error: {e}")
|
|
else:
|
|
logger.error(f"[{task_id}] No Tubidy search results found for query '{search_query}'")
|
|
raise Exception("All download options (spotDL, yt-dlp, Tubidy) failed. No Tubidy search results found.")
|
|
|
|
# Verify files and move to /remote-music/music
|
|
if not download_successful:
|
|
raise Exception("Ingestion finished but no download was successful.")
|
|
|
|
if download_successful and not is_playlist:
|
|
new_moved = move_new_files_to_dest(temp_dir, "/remote-music/music")
|
|
if "files" not in jobs_dict[task_id] or not jobs_dict[task_id]["files"]:
|
|
jobs_dict[task_id]["files"] = []
|
|
jobs_dict[task_id]["files"].extend(new_moved)
|
|
|
|
if "files" not in jobs_dict[task_id] or not jobs_dict[task_id]["files"]:
|
|
logger.info(f"[{task_id}] All tracks in this query were already downloaded (skipped duplicates).")
|
|
jobs_dict[task_id]["files"] = []
|
|
jobs_dict[task_id]["status"] = "completed"
|
|
save_jobs(jobs_dict)
|
|
if chat_id:
|
|
send_telegram_notification(chat_id, f"✅ Successfully ingested request: '{query}'\n\n(All tracks already existed - skipped duplicates)")
|
|
return
|
|
|
|
jobs_dict[task_id]["status"] = "completed"
|
|
save_jobs(jobs_dict)
|
|
logger.info(f"[{task_id}] Music Ingestion task fully completed.")
|
|
|
|
if chat_id:
|
|
files_str = "\n".join(jobs_dict[task_id]["files"])
|
|
send_telegram_notification(chat_id, f"✅ Successfully ingested request: '{query}'\n\nFiles imported to Navidrome:\n{files_str}")
|
|
|
|
except Exception as e:
|
|
logger.error(f"[{task_id}] Ingestion failed: {e}")
|
|
jobs_dict[task_id]["status"] = "failed"
|
|
jobs_dict[task_id]["error"] = str(e)
|
|
save_jobs(jobs_dict)
|
|
if chat_id:
|
|
send_telegram_notification(chat_id, f"❌ Failed to ingest request: '{query}'\nError: {e}")
|
|
|
|
finally:
|
|
# Clean up local emptyDir workspace directory for this task
|
|
if os.path.exists(temp_dir):
|
|
shutil.rmtree(temp_dir)
|
|
|