feat: add Music Ingestion Pipeline (MusicSeerr) with Gluetun proxy and Telegram notifications
Build and Push Docker Images / build (nextgen) (push) Waiting to run
Details
Build and Push Docker Images / build (web) (push) Waiting to run
Details
Build and Push Docker Images / build (worker) (push) Waiting to run
Details
Build and Push Docker Images / deploy (push) Blocked by required conditions
Details
Build and Push Docker Images / build (api) (push) Has been cancelled
Details
Build and Push Docker Images / build (musicseerr) (push) Has been cancelled
Details
Build and Push Docker Images / build (nextgen) (push) Waiting to run
Details
Build and Push Docker Images / build (web) (push) Waiting to run
Details
Build and Push Docker Images / build (worker) (push) Waiting to run
Details
Build and Push Docker Images / deploy (push) Blocked by required conditions
Details
Build and Push Docker Images / build (api) (push) Has been cancelled
Details
Build and Push Docker Images / build (musicseerr) (push) Has been cancelled
Details
This commit is contained in:
parent
ea005a2b29
commit
300eedc5b1
|
|
@ -9,6 +9,7 @@ on:
|
||||||
- 'apps/api/**'
|
- 'apps/api/**'
|
||||||
- 'apps/web/**'
|
- 'apps/web/**'
|
||||||
- 'apps/nextgen/**'
|
- 'apps/nextgen/**'
|
||||||
|
- 'apps/musicseerr/**'
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
build:
|
build:
|
||||||
|
|
@ -16,7 +17,7 @@ jobs:
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
component: [worker, api, web, nextgen]
|
component: [worker, api, web, nextgen, musicseerr]
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,35 @@
|
||||||
|
FROM python:3.11-slim
|
||||||
|
|
||||||
|
# Install system dependencies (ffmpeg is required for audio extraction/spotDL)
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
ffmpeg \
|
||||||
|
curl \
|
||||||
|
git \
|
||||||
|
build-essential \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Set working directory
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
# Install python dependencies directly (keeps container lightweight)
|
||||||
|
RUN pip install --no-cache-dir \
|
||||||
|
fastapi \
|
||||||
|
uvicorn \
|
||||||
|
requests \
|
||||||
|
beautifulsoup4 \
|
||||||
|
mutagen \
|
||||||
|
spotdl \
|
||||||
|
yt-dlp
|
||||||
|
|
||||||
|
# Copy application files
|
||||||
|
COPY main.py tasks.py /app/
|
||||||
|
|
||||||
|
# Initialize directories and set permissions for k8s volumes
|
||||||
|
RUN mkdir -p /tmp/downloads && chmod 777 /tmp/downloads
|
||||||
|
RUN mkdir -p /remote-music && chmod 777 /remote-music
|
||||||
|
|
||||||
|
# Expose FastAPI default port
|
||||||
|
EXPOSE 8000
|
||||||
|
|
||||||
|
# Start FastAPI application
|
||||||
|
CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||||
|
|
@ -0,0 +1,458 @@
|
||||||
|
import os
|
||||||
|
import uuid
|
||||||
|
import logging
|
||||||
|
from fastapi import FastAPI, BackgroundTasks, Request, Form
|
||||||
|
from fastapi.responses import HTMLResponse, JSONResponse
|
||||||
|
from pydantic import BaseModel
|
||||||
|
import requests
|
||||||
|
|
||||||
|
from tasks import run_download_task
|
||||||
|
|
||||||
|
# Configure Logger
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
logger = logging.getLogger("musicseerr-main")
|
||||||
|
|
||||||
|
app = FastAPI(title="MusicSeerr Ingestion Pipeline", version="1.0.0")
|
||||||
|
|
||||||
|
# In-memory jobs database
|
||||||
|
JOBS = {}
|
||||||
|
|
||||||
|
# Read configurations from Environment Variables
|
||||||
|
PROXY_URL = os.getenv("GLUETUN_PROXY_URL", "http://gluetun-svc:8888")
|
||||||
|
TELEGRAM_BOT_TOKEN = os.getenv("TELEGRAM_BOT_TOKEN")
|
||||||
|
|
||||||
|
class SubmitRequest(BaseModel):
|
||||||
|
query: str
|
||||||
|
|
||||||
|
def send_telegram_message(chat_id: int, text: str):
|
||||||
|
"""
|
||||||
|
Sends a message back to Telegram. Outgoing traffic uses the normal cluster network.
|
||||||
|
"""
|
||||||
|
if not TELEGRAM_BOT_TOKEN or not chat_id:
|
||||||
|
return
|
||||||
|
url = f"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendMessage"
|
||||||
|
payload = {"chat_id": chat_id, "text": text}
|
||||||
|
try:
|
||||||
|
requests.post(url, json=payload, timeout=10)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to send Telegram message to chat {chat_id}: {e}")
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/health")
|
||||||
|
def health_check():
|
||||||
|
return {"status": "healthy", "proxy": PROXY_URL}
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/jobs")
|
||||||
|
def get_jobs():
|
||||||
|
"""
|
||||||
|
Returns the list of all jobs, sorted by timestamp.
|
||||||
|
"""
|
||||||
|
return JSONResponse(content=list(JOBS.values()))
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/status/{task_id}")
|
||||||
|
def get_job_status(task_id: str):
|
||||||
|
"""
|
||||||
|
Returns the status of a specific job.
|
||||||
|
"""
|
||||||
|
if task_id not in JOBS:
|
||||||
|
return JSONResponse(status_code=404, content={"error": "Job not found"})
|
||||||
|
return JSONResponse(content=JOBS[task_id])
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/submit")
|
||||||
|
def submit_job(req: SubmitRequest, background_tasks: BackgroundTasks):
|
||||||
|
"""
|
||||||
|
API endpoint to submit download jobs via JSON.
|
||||||
|
"""
|
||||||
|
task_id = str(uuid.uuid4())
|
||||||
|
query = req.query.strip()
|
||||||
|
|
||||||
|
if not query:
|
||||||
|
return JSONResponse(status_code=400, content={"error": "Query cannot be empty"})
|
||||||
|
|
||||||
|
JOBS[task_id] = {
|
||||||
|
"id": task_id,
|
||||||
|
"query": query,
|
||||||
|
"status": "pending",
|
||||||
|
"priority_used": None,
|
||||||
|
"files": [],
|
||||||
|
"error": None
|
||||||
|
}
|
||||||
|
|
||||||
|
background_tasks.add_task(run_download_task, task_id, query, PROXY_URL, JOBS)
|
||||||
|
|
||||||
|
return {"task_id": task_id, "status": "pending"}
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/telegram-webhook")
|
||||||
|
async def telegram_webhook(request: Request, background_tasks: BackgroundTasks):
|
||||||
|
"""
|
||||||
|
Endpoint for Telegram bot webhooks.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
payload = await request.json()
|
||||||
|
logger.info(f"Received Telegram Webhook Payload: {payload}")
|
||||||
|
|
||||||
|
message = payload.get("message", {})
|
||||||
|
chat = message.get("chat", {})
|
||||||
|
chat_id = chat.get("id")
|
||||||
|
text = message.get("text", "").strip()
|
||||||
|
|
||||||
|
if not text or not chat_id:
|
||||||
|
return {"status": "ignored"}
|
||||||
|
|
||||||
|
# Ignore commands like /start unless they have arguments
|
||||||
|
if text.startswith("/start"):
|
||||||
|
parts = text.split(" ", 1)
|
||||||
|
if len(parts) > 1:
|
||||||
|
text = parts[1].strip()
|
||||||
|
else:
|
||||||
|
send_telegram_message(
|
||||||
|
chat_id,
|
||||||
|
"Welcome to MusicSeerr! 🎵\nSend me a song name, Spotify URL, YouTube URL, or Tubidy link, and I'll ingest it into the Navidrome library!"
|
||||||
|
)
|
||||||
|
return {"status": "welcomed"}
|
||||||
|
|
||||||
|
task_id = str(uuid.uuid4())
|
||||||
|
JOBS[task_id] = {
|
||||||
|
"id": task_id,
|
||||||
|
"query": text,
|
||||||
|
"status": "pending",
|
||||||
|
"priority_used": None,
|
||||||
|
"files": [],
|
||||||
|
"error": None
|
||||||
|
}
|
||||||
|
|
||||||
|
# Pass chat_id to the background task (we will update run_download_task to support this)
|
||||||
|
background_tasks.add_task(run_download_task, task_id, text, PROXY_URL, JOBS, chat_id)
|
||||||
|
|
||||||
|
send_telegram_message(chat_id, f"📥 Queued: '{text}'\nTask ID: {task_id}")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error handling Telegram webhook: {e}")
|
||||||
|
|
||||||
|
return {"status": "processed"}
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/", response_class=HTMLResponse)
|
||||||
|
def index_page():
|
||||||
|
"""
|
||||||
|
Serves a highly-styled, responsive dark-mode portal.
|
||||||
|
"""
|
||||||
|
html_content = """
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
|
<title>MusicSeerr - Music Ingestion Portal</title>
|
||||||
|
<link href="https://fonts.googleapis.com/css2?family=Outfit:wght@300;400;600;800&display=swap" rel="stylesheet">
|
||||||
|
<style>
|
||||||
|
:root {
|
||||||
|
--bg-main: #0B0F19;
|
||||||
|
--bg-card: #151D30;
|
||||||
|
--bg-input: #1F2A45;
|
||||||
|
--accent-primary: #8B5CF6;
|
||||||
|
--accent-secondary: #EC4899;
|
||||||
|
--text-main: #F3F4F6;
|
||||||
|
--text-muted: #9CA3AF;
|
||||||
|
--success: #10B981;
|
||||||
|
--warning: #F59E0B;
|
||||||
|
--error: #EF4444;
|
||||||
|
--info: #3B82F6;
|
||||||
|
}
|
||||||
|
* {
|
||||||
|
box-sizing: border-box;
|
||||||
|
margin: 0;
|
||||||
|
padding: 0;
|
||||||
|
font-family: 'Outfit', sans-serif;
|
||||||
|
}
|
||||||
|
body {
|
||||||
|
background: linear-gradient(135deg, var(--bg-main) 0%, #080B13 100%);
|
||||||
|
color: var(--text-main);
|
||||||
|
min-height: 100vh;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
align-items: center;
|
||||||
|
padding: 2rem 1rem;
|
||||||
|
}
|
||||||
|
header {
|
||||||
|
text-align: center;
|
||||||
|
margin-bottom: 2.5rem;
|
||||||
|
max-width: 600px;
|
||||||
|
}
|
||||||
|
header h1 {
|
||||||
|
font-size: 3rem;
|
||||||
|
font-weight: 800;
|
||||||
|
background: linear-gradient(to right, var(--accent-primary), var(--accent-secondary));
|
||||||
|
-webkit-background-clip: text;
|
||||||
|
-webkit-text-fill-color: transparent;
|
||||||
|
margin-bottom: 0.5rem;
|
||||||
|
}
|
||||||
|
header p {
|
||||||
|
color: var(--text-muted);
|
||||||
|
font-size: 1.1rem;
|
||||||
|
}
|
||||||
|
.container {
|
||||||
|
width: 100%;
|
||||||
|
max-width: 800px;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 2rem;
|
||||||
|
}
|
||||||
|
.card {
|
||||||
|
background: var(--bg-card);
|
||||||
|
border: 1px solid rgba(255, 255, 255, 0.05);
|
||||||
|
border-radius: 16px;
|
||||||
|
padding: 2rem;
|
||||||
|
box-shadow: 0 10px 25px -5px rgba(0, 0, 0, 0.3);
|
||||||
|
}
|
||||||
|
form {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 1.2rem;
|
||||||
|
}
|
||||||
|
.form-group {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 0.5rem;
|
||||||
|
}
|
||||||
|
label {
|
||||||
|
font-weight: 600;
|
||||||
|
color: var(--text-main);
|
||||||
|
}
|
||||||
|
input[type="text"] {
|
||||||
|
width: 100%;
|
||||||
|
padding: 1rem;
|
||||||
|
border-radius: 10px;
|
||||||
|
border: 1px solid rgba(255, 255, 255, 0.1);
|
||||||
|
background-color: var(--bg-input);
|
||||||
|
color: var(--text-main);
|
||||||
|
font-size: 1rem;
|
||||||
|
transition: all 0.3s ease;
|
||||||
|
}
|
||||||
|
input[type="text"]:focus {
|
||||||
|
outline: none;
|
||||||
|
border-color: var(--accent-primary);
|
||||||
|
box-shadow: 0 0 0 3px rgba(139, 92, 246, 0.3);
|
||||||
|
}
|
||||||
|
button {
|
||||||
|
padding: 1rem;
|
||||||
|
border: none;
|
||||||
|
border-radius: 10px;
|
||||||
|
background: linear-gradient(to right, var(--accent-primary), var(--accent-secondary));
|
||||||
|
color: white;
|
||||||
|
font-size: 1.1rem;
|
||||||
|
font-weight: 600;
|
||||||
|
cursor: pointer;
|
||||||
|
transition: all 0.3s ease;
|
||||||
|
box-shadow: 0 4px 14px 0 rgba(139, 92, 246, 0.4);
|
||||||
|
}
|
||||||
|
button:hover {
|
||||||
|
transform: translateY(-2px);
|
||||||
|
box-shadow: 0 6px 20px 0 rgba(139, 92, 246, 0.6);
|
||||||
|
}
|
||||||
|
button:active {
|
||||||
|
transform: translateY(0);
|
||||||
|
}
|
||||||
|
.jobs-section h2 {
|
||||||
|
font-size: 1.5rem;
|
||||||
|
font-weight: 600;
|
||||||
|
margin-bottom: 1rem;
|
||||||
|
border-left: 4px solid var(--accent-primary);
|
||||||
|
padding-left: 0.75rem;
|
||||||
|
}
|
||||||
|
.job-list {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 1rem;
|
||||||
|
max-height: 500px;
|
||||||
|
overflow-y: auto;
|
||||||
|
}
|
||||||
|
.job-item {
|
||||||
|
background: var(--bg-input);
|
||||||
|
border-radius: 10px;
|
||||||
|
padding: 1.2rem;
|
||||||
|
display: flex;
|
||||||
|
justify-content: space-between;
|
||||||
|
align-items: center;
|
||||||
|
gap: 1rem;
|
||||||
|
border-left: 4px solid var(--text-muted);
|
||||||
|
transition: all 0.3s ease;
|
||||||
|
}
|
||||||
|
.job-item.pending { border-left-color: var(--info); }
|
||||||
|
.job-item.downloading { border-left-color: var(--warning); }
|
||||||
|
.job-item.completed { border-left-color: var(--success); }
|
||||||
|
.job-item.failed { border-left-color: var(--error); }
|
||||||
|
|
||||||
|
.job-info {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 0.25rem;
|
||||||
|
overflow: hidden;
|
||||||
|
}
|
||||||
|
.job-query {
|
||||||
|
font-weight: 600;
|
||||||
|
font-size: 1rem;
|
||||||
|
white-space: nowrap;
|
||||||
|
overflow: hidden;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
}
|
||||||
|
.job-details {
|
||||||
|
font-size: 0.8rem;
|
||||||
|
color: var(--text-muted);
|
||||||
|
}
|
||||||
|
.badge {
|
||||||
|
padding: 0.25rem 0.75rem;
|
||||||
|
border-radius: 50px;
|
||||||
|
font-size: 0.8rem;
|
||||||
|
font-weight: 600;
|
||||||
|
text-transform: uppercase;
|
||||||
|
}
|
||||||
|
.badge.pending { background: rgba(59, 130, 246, 0.2); color: var(--info); }
|
||||||
|
.badge.downloading { background: rgba(245, 158, 11, 0.2); color: var(--warning); }
|
||||||
|
.badge.completed { background: rgba(16, 185, 129, 0.2); color: var(--success); }
|
||||||
|
.badge.failed { background: rgba(239, 68, 68, 0.2); color: var(--error); }
|
||||||
|
|
||||||
|
.status-container {
|
||||||
|
display: flex;
|
||||||
|
gap: 2rem;
|
||||||
|
margin-top: 1.5rem;
|
||||||
|
font-size: 0.9rem;
|
||||||
|
color: var(--text-muted);
|
||||||
|
justify-content: center;
|
||||||
|
}
|
||||||
|
.status-item {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 0.5rem;
|
||||||
|
}
|
||||||
|
.status-dot {
|
||||||
|
width: 10px;
|
||||||
|
height: 10px;
|
||||||
|
border-radius: 50%;
|
||||||
|
background-color: var(--success);
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<header>
|
||||||
|
<h1>MusicSeerr 🎵</h1>
|
||||||
|
<p>Sovereign Music Ingestion Portal for Navidrome</p>
|
||||||
|
</header>
|
||||||
|
|
||||||
|
<div class="container">
|
||||||
|
<div class="card">
|
||||||
|
<form id="ingestForm">
|
||||||
|
<div class="form-group">
|
||||||
|
<label for="query">Search Query or Media URL</label>
|
||||||
|
<input type="text" id="query" name="query" placeholder="Song Name, Spotify Link, YouTube Link, or Tubidy Link..." required>
|
||||||
|
</div>
|
||||||
|
<button type="submit">Ingest Audio</button>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="card jobs-section">
|
||||||
|
<h2>Active Ingestion Jobs</h2>
|
||||||
|
<div class="job-list" id="jobList">
|
||||||
|
<p style="color: var(--text-muted); text-align: center; padding: 2rem;">No active downloads. Submit a link above!</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="status-container">
|
||||||
|
<div class="status-item">
|
||||||
|
<div class="status-dot"></div>
|
||||||
|
<span>FastAPI Backend Active</span>
|
||||||
|
</div>
|
||||||
|
<div class="status-item">
|
||||||
|
<div class="status-dot"></div>
|
||||||
|
<span>VPN Proxy Bound</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
const form = document.getElementById('ingestForm');
|
||||||
|
const queryInput = document.getElementById('query');
|
||||||
|
const jobList = document.getElementById('jobList');
|
||||||
|
|
||||||
|
form.addEventListener('submit', async (e) => {
|
||||||
|
e.preventDefault();
|
||||||
|
const query = queryInput.value.trim();
|
||||||
|
if (!query) return;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch('/submit', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'application/json'
|
||||||
|
},
|
||||||
|
body: JSON.stringify({ query })
|
||||||
|
});
|
||||||
|
|
||||||
|
if (response.ok) {
|
||||||
|
queryInput.value = '';
|
||||||
|
loadJobs();
|
||||||
|
} else {
|
||||||
|
alert('Failed to submit ingestion job.');
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
console.error('Error submitting job:', err);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
async function loadJobs() {
|
||||||
|
try {
|
||||||
|
const response = await fetch('/jobs');
|
||||||
|
if (response.ok) {
|
||||||
|
const jobs = await response.json();
|
||||||
|
if (jobs.length === 0) {
|
||||||
|
jobList.innerHTML = '<p style="color: var(--text-muted); text-align: center; padding: 2rem;">No active downloads. Submit a link above!</p>';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort by completion / timestamp (newest first)
|
||||||
|
jobs.reverse();
|
||||||
|
|
||||||
|
jobList.innerHTML = jobs.map(job => {
|
||||||
|
let details = '';
|
||||||
|
if (job.status === 'completed' && job.files.length > 0) {
|
||||||
|
details = `Imported: ${job.files.join(', ')}`;
|
||||||
|
} else if (job.status === 'failed') {
|
||||||
|
details = `Error: ${job.error || 'Unknown error occurred'}`;
|
||||||
|
} else if (job.status === 'downloading') {
|
||||||
|
details = `Downloading via ${job.priority_used || 'pipeline'}...`;
|
||||||
|
} else {
|
||||||
|
details = 'Queued in background tasks';
|
||||||
|
}
|
||||||
|
|
||||||
|
return `
|
||||||
|
<div class="job-item ${job.status}">
|
||||||
|
<div class="job-info">
|
||||||
|
<div class="job-query">${escapeHtml(job.query)}</div>
|
||||||
|
<div class="job-details">${escapeHtml(details)}</div>
|
||||||
|
</div>
|
||||||
|
<span class="badge ${job.status}">${job.status}</span>
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
}).join('');
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
console.error('Error loading jobs:', err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function escapeHtml(str) {
|
||||||
|
return str.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Auto-refresh jobs every 5 seconds
|
||||||
|
loadJobs();
|
||||||
|
setInterval(loadJobs, 5000);
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
"""
|
||||||
|
return HTMLResponse(content=html_content)
|
||||||
|
|
@ -0,0 +1,267 @@
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
import requests
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
from urllib.parse import urljoin
|
||||||
|
from mutagen.mp3 import EasyMP3
|
||||||
|
from mutagen.id3 import ID3
|
||||||
|
|
||||||
|
# Configure Logger
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
logger = logging.getLogger("musicseerr-tasks")
|
||||||
|
|
||||||
|
def scrape_tubidy_link(url: str, proxy_url: str) -> str:
|
||||||
|
"""
|
||||||
|
Scrapes a Tubidy page to find the direct MP3 download URL.
|
||||||
|
"""
|
||||||
|
proxies = {"http": proxy_url, "https": proxy_url} if proxy_url else None
|
||||||
|
headers = {
|
||||||
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(f"Fetching Tubidy page: {url} via proxy: {proxy_url}")
|
||||||
|
response = requests.get(url, headers=headers, proxies=proxies, timeout=20)
|
||||||
|
response.raise_for_status()
|
||||||
|
|
||||||
|
soup = BeautifulSoup(response.text, "html.parser")
|
||||||
|
|
||||||
|
# 1. Search for direct mp3 links in audio tags
|
||||||
|
audio_tag = soup.find("audio")
|
||||||
|
if audio_tag:
|
||||||
|
source = audio_tag.find("source")
|
||||||
|
if source and source.get("src"):
|
||||||
|
return urljoin(url, source["src"])
|
||||||
|
if audio_tag.get("src"):
|
||||||
|
return urljoin(url, audio_tag["src"])
|
||||||
|
|
||||||
|
# 2. Search for <a> tags containing .mp3 in href
|
||||||
|
for a in soup.find_all("a", href=True):
|
||||||
|
href = a["href"]
|
||||||
|
if ".mp3" in href.lower():
|
||||||
|
return urljoin(url, href)
|
||||||
|
|
||||||
|
# 3. Follow potential download/mp3 buttons if nested
|
||||||
|
for a in soup.find_all("a", href=True):
|
||||||
|
text = a.text.lower()
|
||||||
|
if "mp3" in text or "download" in text:
|
||||||
|
sub_url = urljoin(url, a["href"])
|
||||||
|
logger.info(f"Following nested Tubidy download link: {sub_url}")
|
||||||
|
try:
|
||||||
|
sub_resp = requests.get(sub_url, headers=headers, proxies=proxies, timeout=15)
|
||||||
|
if sub_resp.status_code == 200:
|
||||||
|
sub_soup = BeautifulSoup(sub_resp.text, "html.parser")
|
||||||
|
for sub_a in sub_soup.find_all("a", href=True):
|
||||||
|
sub_href = sub_a["href"]
|
||||||
|
if ".mp3" in sub_href.lower():
|
||||||
|
return urljoin(sub_url, sub_href)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Failed to fetch nested link {sub_url}: {e}")
|
||||||
|
|
||||||
|
# 4. Fallback: search raw text for mp3 URLs
|
||||||
|
mp3_matches = re.findall(r'https?://[^\s"\'>]+\.mp3', response.text)
|
||||||
|
if mp3_matches:
|
||||||
|
return mp3_matches[0]
|
||||||
|
|
||||||
|
raise ValueError("Could not find any MP3 download link on the Tubidy page.")
|
||||||
|
|
||||||
|
|
||||||
|
def download_file_stream(url: str, dest_path: str, proxy_url: str):
|
||||||
|
"""
|
||||||
|
Downloads a file in chunks using requests through the proxy.
|
||||||
|
"""
|
||||||
|
proxies = {"http": proxy_url, "https": proxy_url} if proxy_url else None
|
||||||
|
headers = {
|
||||||
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(f"Streaming download from {url} to {dest_path}")
|
||||||
|
response = requests.get(url, headers=headers, proxies=proxies, stream=True, timeout=60)
|
||||||
|
response.raise_for_status()
|
||||||
|
|
||||||
|
with open(dest_path, "wb") as f:
|
||||||
|
for chunk in response.iter_content(chunk_size=8192):
|
||||||
|
if chunk:
|
||||||
|
f.write(chunk)
|
||||||
|
|
||||||
|
|
||||||
|
def tag_mp3(file_path: str, filename: str):
|
||||||
|
"""
|
||||||
|
Parses the filename to extract artist/title and injects ID3 tags using Mutagen.
|
||||||
|
"""
|
||||||
|
base_name = os.path.splitext(os.path.basename(file_path))[0]
|
||||||
|
artist = "Unknown Artist"
|
||||||
|
title = base_name
|
||||||
|
|
||||||
|
# Try parsing "Artist - Title" format
|
||||||
|
if " - " in base_name:
|
||||||
|
parts = base_name.split(" - ", 1)
|
||||||
|
artist = parts[0].strip()
|
||||||
|
title = parts[1].strip()
|
||||||
|
|
||||||
|
logger.info(f"Applying tags - Title: '{title}', Artist: '{artist}' to {file_path}")
|
||||||
|
|
||||||
|
# Initialize ID3 tags if they do not exist
|
||||||
|
try:
|
||||||
|
audio = EasyMP3(file_path)
|
||||||
|
except Exception:
|
||||||
|
id3 = ID3()
|
||||||
|
id3.save(file_path)
|
||||||
|
audio = EasyMP3(file_path)
|
||||||
|
|
||||||
|
audio["title"] = title
|
||||||
|
audio["artist"] = artist
|
||||||
|
audio["album"] = "Tubidy Ingestion"
|
||||||
|
audio.save()
|
||||||
|
|
||||||
|
|
||||||
|
def send_telegram_notification(chat_id: int, text: str):
|
||||||
|
"""
|
||||||
|
Sends a message back to Telegram.
|
||||||
|
"""
|
||||||
|
token = os.getenv("TELEGRAM_BOT_TOKEN")
|
||||||
|
if not token or not chat_id:
|
||||||
|
return
|
||||||
|
url = f"https://api.telegram.org/bot{token}/sendMessage"
|
||||||
|
try:
|
||||||
|
requests.post(url, json={"chat_id": chat_id, "text": text}, timeout=10)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to send Telegram notification: {e}")
|
||||||
|
|
||||||
|
|
||||||
|
def run_download_task(task_id: str, query: str, proxy_url: str, jobs_dict: dict, chat_id: int = None):
|
||||||
|
"""
|
||||||
|
Core download manager task that runs in the background.
|
||||||
|
"""
|
||||||
|
jobs_dict[task_id]["status"] = "downloading"
|
||||||
|
|
||||||
|
temp_dir = f"/tmp/downloads/{task_id}"
|
||||||
|
os.makedirs(temp_dir, exist_ok=True)
|
||||||
|
|
||||||
|
try:
|
||||||
|
is_tubidy = "tubidy" in query.lower()
|
||||||
|
is_spotify = "spotify.com" in query.lower()
|
||||||
|
is_youtube = "youtube.com" in query.lower() or "youtu.be" in query.lower()
|
||||||
|
|
||||||
|
# Priority 1: spotDL (text search or Spotify URL)
|
||||||
|
if not is_tubidy and not is_youtube:
|
||||||
|
jobs_dict[task_id]["priority_used"] = "spotDL"
|
||||||
|
logger.info(f"[{task_id}] Attempting Priority 1: spotDL for '{query}'")
|
||||||
|
|
||||||
|
env = os.environ.copy()
|
||||||
|
if proxy_url:
|
||||||
|
env["http_proxy"] = proxy_url
|
||||||
|
env["https_proxy"] = proxy_url
|
||||||
|
|
||||||
|
cmd = ["spotdl", "download", query]
|
||||||
|
if proxy_url:
|
||||||
|
cmd += ["--proxy", proxy_url]
|
||||||
|
|
||||||
|
# Run spotDL in the temp directory
|
||||||
|
result = subprocess.run(cmd, cwd=temp_dir, env=env, capture_output=True, text=True)
|
||||||
|
if result.returncode == 0:
|
||||||
|
logger.info(f"[{task_id}] spotDL download succeeded.")
|
||||||
|
else:
|
||||||
|
logger.warning(f"[{task_id}] spotDL failed: {result.stderr or result.stdout}. Falling back to yt-dlp.")
|
||||||
|
# Clear state for P2 fallback
|
||||||
|
is_spotify = False
|
||||||
|
|
||||||
|
# Priority 2: yt-dlp (direct YouTube or fallback for failed spotDL)
|
||||||
|
if not is_tubidy and not is_spotify:
|
||||||
|
jobs_dict[task_id]["priority_used"] = "yt-dlp"
|
||||||
|
logger.info(f"[{task_id}] Attempting Priority 2: yt-dlp for '{query}'")
|
||||||
|
|
||||||
|
env = os.environ.copy()
|
||||||
|
if proxy_url:
|
||||||
|
env["http_proxy"] = proxy_url
|
||||||
|
env["https_proxy"] = proxy_url
|
||||||
|
|
||||||
|
# If not a link, treat as search query
|
||||||
|
search_query = query
|
||||||
|
if not query.startswith("http"):
|
||||||
|
search_query = f"ytsearch1:{query}"
|
||||||
|
|
||||||
|
cmd = [
|
||||||
|
"yt-dlp",
|
||||||
|
"-x",
|
||||||
|
"--audio-format", "mp3",
|
||||||
|
"--embed-metadata",
|
||||||
|
"--yes-playlist",
|
||||||
|
"--output", "%(title)s.%(ext)s"
|
||||||
|
]
|
||||||
|
if proxy_url:
|
||||||
|
cmd += ["--proxy", proxy_url]
|
||||||
|
cmd.append(search_query)
|
||||||
|
|
||||||
|
result = subprocess.run(cmd, cwd=temp_dir, env=env, capture_output=True, text=True)
|
||||||
|
if result.returncode == 0:
|
||||||
|
logger.info(f"[{task_id}] yt-dlp download succeeded.")
|
||||||
|
else:
|
||||||
|
logger.error(f"[{task_id}] yt-dlp failed: {result.stderr or result.stdout}")
|
||||||
|
raise Exception(f"yt-dlp failed to download: {result.stderr}")
|
||||||
|
|
||||||
|
# Priority 3: Custom Tubidy Scraper
|
||||||
|
if is_tubidy:
|
||||||
|
jobs_dict[task_id]["priority_used"] = "Tubidy"
|
||||||
|
logger.info(f"[{task_id}] Attempting Priority 3: Tubidy Scraper for '{query}'")
|
||||||
|
|
||||||
|
# Scrape MP3 URL
|
||||||
|
mp3_url = scrape_tubidy_link(query, proxy_url)
|
||||||
|
logger.info(f"[{task_id}] Scraped Tubidy MP3 URL: {mp3_url}")
|
||||||
|
|
||||||
|
# Formulate filename
|
||||||
|
filename = query.split("/")[-1]
|
||||||
|
if not filename or ".html" in filename or "?" in filename:
|
||||||
|
filename = "tubidy_download"
|
||||||
|
filename = filename.replace(".html", "").replace(".php", "").strip()
|
||||||
|
if not filename.endswith(".mp3"):
|
||||||
|
filename += ".mp3"
|
||||||
|
|
||||||
|
dest_path = os.path.join(temp_dir, filename)
|
||||||
|
|
||||||
|
# Download file stream
|
||||||
|
download_file_stream(mp3_url, dest_path, proxy_url)
|
||||||
|
logger.info(f"[{task_id}] Tubidy download completed.")
|
||||||
|
|
||||||
|
# Tag metadata
|
||||||
|
tag_mp3(dest_path, filename)
|
||||||
|
logger.info(f"[{task_id}] Tubidy ID3 tagging complete.")
|
||||||
|
|
||||||
|
# Verify files and move to /remote-music
|
||||||
|
downloaded_files = os.listdir(temp_dir)
|
||||||
|
if not downloaded_files:
|
||||||
|
raise Exception("Ingestion finished but no files were found in local temp folder.")
|
||||||
|
|
||||||
|
os.makedirs("/remote-music", exist_ok=True)
|
||||||
|
moved_files = []
|
||||||
|
|
||||||
|
for file_name in downloaded_files:
|
||||||
|
src_file = os.path.join(temp_dir, file_name)
|
||||||
|
dest_file = os.path.join("/remote-music", file_name)
|
||||||
|
|
||||||
|
logger.info(f"[{task_id}] Moving '{file_name}' to remote Seedbox directory")
|
||||||
|
shutil.move(src_file, dest_file)
|
||||||
|
moved_files.append(file_name)
|
||||||
|
|
||||||
|
jobs_dict[task_id]["files"] = moved_files
|
||||||
|
jobs_dict[task_id]["status"] = "completed"
|
||||||
|
logger.info(f"[{task_id}] Music Ingestion task fully completed.")
|
||||||
|
|
||||||
|
if chat_id:
|
||||||
|
files_str = "\n".join(moved_files)
|
||||||
|
send_telegram_notification(chat_id, f"✅ Successfully ingested request: '{query}'\n\nFiles imported to Navidrome:\n{files_str}")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"[{task_id}] Ingestion failed: {e}")
|
||||||
|
jobs_dict[task_id]["status"] = "failed"
|
||||||
|
jobs_dict[task_id]["error"] = str(e)
|
||||||
|
if chat_id:
|
||||||
|
send_telegram_notification(chat_id, f"❌ Failed to ingest request: '{query}'\nError: {e}")
|
||||||
|
|
||||||
|
finally:
|
||||||
|
# Clean up local emptyDir workspace directory for this task
|
||||||
|
if os.path.exists(temp_dir):
|
||||||
|
shutil.rmtree(temp_dir)
|
||||||
|
|
||||||
|
|
@ -10,3 +10,5 @@ resources:
|
||||||
- ruva.yaml
|
- ruva.yaml
|
||||||
- rufaro.yaml
|
- rufaro.yaml
|
||||||
- media-suite.yaml
|
- media-suite.yaml
|
||||||
|
- musicseerr.yaml
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,134 @@
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: gluetun
|
||||||
|
namespace: family-apps
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: gluetun
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: gluetun
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: gluetun
|
||||||
|
image: qmcgaw/gluetun:latest
|
||||||
|
securityContext:
|
||||||
|
capabilities:
|
||||||
|
add:
|
||||||
|
- NET_ADMIN
|
||||||
|
env:
|
||||||
|
# User configures VPN provider here
|
||||||
|
- name: VPN_SERVICE_PROVIDER
|
||||||
|
value: "custom"
|
||||||
|
- name: VPN_TYPE
|
||||||
|
value: "wireguard"
|
||||||
|
- name: HTTPPROXY
|
||||||
|
value: "on"
|
||||||
|
- name: HTTPPROXY_LOG
|
||||||
|
value: "off"
|
||||||
|
# Template environment variables for custom VPN credentials
|
||||||
|
# - name: WIREGUARD_PRIVATE_KEY
|
||||||
|
# value: "YOUR_KEY"
|
||||||
|
# - name: WIREGUARD_ADDRESSES
|
||||||
|
# value: "10.0.0.2/32"
|
||||||
|
ports:
|
||||||
|
- name: proxy
|
||||||
|
containerPort: 8888
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: gluetun-svc
|
||||||
|
namespace: family-apps
|
||||||
|
spec:
|
||||||
|
ports:
|
||||||
|
- port: 8888
|
||||||
|
targetPort: 8888
|
||||||
|
name: proxy
|
||||||
|
selector:
|
||||||
|
app: gluetun
|
||||||
|
---
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: musicseerr
|
||||||
|
namespace: family-apps
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: musicseerr
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: musicseerr
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: musicseerr
|
||||||
|
image: frankchine/geocrop-musicseerr:latest
|
||||||
|
imagePullPolicy: Always
|
||||||
|
env:
|
||||||
|
- name: GLUETUN_PROXY_URL
|
||||||
|
value: "http://gluetun-svc:8888"
|
||||||
|
- name: TELEGRAM_BOT_TOKEN
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: musicseerr-secrets
|
||||||
|
key: telegram-bot-token
|
||||||
|
optional: true
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
containerPort: 8000
|
||||||
|
volumeMounts:
|
||||||
|
- name: downloads
|
||||||
|
mountPath: /tmp/downloads
|
||||||
|
- name: remote-music
|
||||||
|
mountPath: /remote-music
|
||||||
|
volumes:
|
||||||
|
- name: downloads
|
||||||
|
emptyDir: {}
|
||||||
|
- name: remote-music
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: rclone-pvc
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: musicseerr
|
||||||
|
namespace: family-apps
|
||||||
|
spec:
|
||||||
|
ports:
|
||||||
|
- port: 8000
|
||||||
|
targetPort: 8000
|
||||||
|
name: http
|
||||||
|
selector:
|
||||||
|
app: musicseerr
|
||||||
|
---
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: Ingress
|
||||||
|
metadata:
|
||||||
|
name: musicseerr-ingress
|
||||||
|
namespace: family-apps
|
||||||
|
annotations:
|
||||||
|
cert-manager.io/cluster-issuer: "letsencrypt-prod"
|
||||||
|
spec:
|
||||||
|
ingressClassName: nginx
|
||||||
|
tls:
|
||||||
|
- hosts:
|
||||||
|
- musicseerr.techarvest.co.zw
|
||||||
|
secretName: musicseerr-tls
|
||||||
|
rules:
|
||||||
|
- host: musicseerr.techarvest.co.zw
|
||||||
|
http:
|
||||||
|
paths:
|
||||||
|
- path: /
|
||||||
|
pathType: Prefix
|
||||||
|
backend:
|
||||||
|
service:
|
||||||
|
name: musicseerr
|
||||||
|
port:
|
||||||
|
number: 8000
|
||||||
Loading…
Reference in New Issue