fix(musicseer): implement sequential playlist scraping via embeds, add Deno to Dockerfile for yt-dlp, and configure Always imagePullPolicy

apps/api/main.py

@@ -245,9 +245,14 @@ async def create_inference_job(job_req: InferenceJobRequest, current_user: dict
         cached["cached"] = True
         return cached
     
+    # Transform payload to match worker expectations
+    payload = job_req.model_dump()
+    payload['radius_m'] = int(payload.pop('radius_km') * 1000)
+    payload['model'] = payload.pop('model_name')
+
     job = task_queue.enqueue(
-        'worker.run_inference', 
-        job_req.model_dump(),
+        'worker.run_job', 
+        payload,
         job_timeout='25m',
         result_ttl=86400,
         failure_ttl=86400

apps/musicseerr/Dockerfile

@@ -8,6 +8,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     build-essential \
     && rm -rf /var/lib/apt/lists/*
 
+# Install Deno for yt-dlp JavaScript decryption support
+COPY --from=denoland/deno:bin /deno /usr/local/bin/deno
+
 # Set working directory
 WORKDIR /app
 

apps/musicseerr/tasks.py

@@ -4,6 +4,7 @@ import subprocess
 import logging
 import re
 import requests
+import json
 from bs4 import BeautifulSoup
 from urllib.parse import urljoin
 from mutagen.mp3 import EasyMP3
@@ -229,38 +230,79 @@ def send_telegram_notification(chat_id: int, text: str):
 
 def extract_spotify_track_title(url: str, proxy_url: str) -> str:
     """
-    Fetches the Spotify track page and extracts the track name and artists from the title tag.
+    Scrapes a Spotify track embed page and returns the track search string ("Artist - Title").
     """
+    match = re.search(r'track/([a-zA-Z0-9]+)', url)
+    if not match:
+        logger.warning(f"Could not extract track ID from URL: {url}")
+        return ""
+    track_id = match.group(1)
+    embed_url = f"https://open.spotify.com/embed/track/{track_id}"
+    
     proxies = {"http": proxy_url, "https": proxy_url} if proxy_url else None
     headers = {
         "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
     }
-    logger.info(f"Extracting track title from Spotify page: {url}")
+    
+    logger.info(f"Scraping Spotify track via embed URL: {embed_url}")
     try:
-        response = requests.get(url, headers=headers, proxies=proxies, timeout=15)
+        response = requests.get(embed_url, headers=headers, proxies=proxies, timeout=15)
         if response.status_code == 200:
-            soup = BeautifulSoup(response.text, "html.parser")
-            title_tag = soup.find("title")
-            if title_tag:
-                title_text = title_tag.text
-                # Format is typically "Song Name - song and lyrics by Artist1, Artist2 | Spotify"
-                # Or "Song Name - song by Artist | Spotify"
-                if " - song " in title_text:
-                    parts = title_text.split(" - song ", 1)
-                    song_name = parts[0].strip()
-                    artist_part = parts[1].split(" by ", 1)
-                    if len(artist_part) > 1:
-                        artists = artist_part[1].split(" | Spotify", 1)[0].strip()
-                        return f"{artists} - {song_name}"
-                    return song_name
-                elif " | Spotify" in title_text:
-                    return title_text.split(" | Spotify")[0].strip()
-                return title_text.strip()
+            next_data_match = re.search(r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>', response.text)
+            if next_data_match:
+                data = json.loads(next_data_match.group(1))
+                entity = data['props']['pageProps']['state']['data']['entity']
+                name = entity.get('name', '').strip()
+                artists = [a.get('name', '') for a in entity.get('artists', [])]
+                artists = [a for a in artists if a]
+                if name and artists:
+                    return f"{', '.join(artists)} - {name}"
+                elif name:
+                    return name
     except Exception as e:
         logger.warning(f"Failed to extract Spotify track title: {e}")
     return ""
 
 
+def scrape_spotify_playlist(url: str, proxy_url: str) -> list:
+    """
+    Scrapes a Spotify playlist embed page and returns a list of track search strings ("Artist - Title").
+    """
+    match = re.search(r'playlist/([a-zA-Z0-9]+)', url)
+    if not match:
+        logger.warning(f"Could not extract playlist ID from URL: {url}")
+        return []
+    playlist_id = match.group(1)
+    embed_url = f"https://open.spotify.com/embed/playlist/{playlist_id}"
+    
+    proxies = {"http": proxy_url, "https": proxy_url} if proxy_url else None
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+    }
+    
+    logger.info(f"Scraping Spotify playlist via embed URL: {embed_url}")
+    try:
+        response = requests.get(embed_url, headers=headers, proxies=proxies, timeout=15)
+        if response.status_code == 200:
+            next_data_match = re.search(r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>', response.text)
+            if next_data_match:
+                data = json.loads(next_data_match.group(1))
+                entity = data['props']['pageProps']['state']['data']['entity']
+                tracks = entity.get('trackList', [])
+                track_queries = []
+                for t in tracks:
+                    title = t.get('title', '').strip()
+                    subtitle = t.get('subtitle', '').replace('\xa0', ' ').strip()
+                    if title and subtitle:
+                        track_queries.append(f"{subtitle} - {title}")
+                    elif title:
+                        track_queries.append(title)
+                return track_queries
+    except Exception as e:
+        logger.warning(f"Failed to scrape Spotify playlist: {e}")
+    return []
+
+
 def symlink_existing_tracks(temp_dir: str):
     """
     Finds all audio files recursively in /remote-music/music, extracts artist and title,
@@ -353,7 +395,95 @@ def run_download_task(task_id: str, query: str, proxy_url: str, jobs_dict: dict,
             resolved_title = extract_spotify_track_title(query, resolved_proxy_url)
             if resolved_title:
                 logger.info(f"[{task_id}] Pre-resolved Spotify URL to text title: '{resolved_title}'")
+                query = resolved_title
 
+        is_playlist = "spotify.com/playlist/" in query.lower()
+
+        if is_playlist:
+            jobs_dict[task_id]["priority_used"] = "playlist-scraper"
+            logger.info(f"[{task_id}] Spotify playlist detected: '{query}'")
+            track_queries = scrape_spotify_playlist(query, resolved_proxy_url)
+            if not track_queries:
+                raise Exception("Could not scrape any tracks from the Spotify playlist embed page.")
+                
+            logger.info(f"[{task_id}] Successfully scraped {len(track_queries)} tracks from playlist.")
+            download_count = 0
+            
+            for idx, track_query in enumerate(track_queries):
+                logger.info(f"[{task_id}] [Track {idx+1}/{len(track_queries)}] Downloading '{track_query}'")
+                track_success = False
+                
+                # 1. spotDL
+                env = os.environ.copy()
+                if resolved_proxy_url:
+                    env["http_proxy"] = resolved_proxy_url
+                    env["https_proxy"] = resolved_proxy_url
+                    
+                cmd = ["spotdl", "download", track_query]
+                if resolved_proxy_url:
+                    cmd += ["--proxy", resolved_proxy_url]
+                    
+                logger.info(f"[{task_id}] [Track {idx+1}] Trying spotDL...")
+                result = subprocess.run(cmd, cwd=temp_dir, env=env, capture_output=True, text=True)
+                if result.returncode == 0:
+                    logger.info(f"[{task_id}] [Track {idx+1}] spotDL succeeded.")
+                    track_success = True
+                else:
+                    logger.warning(f"[{task_id}] [Track {idx+1}] spotDL failed: {result.stderr or result.stdout}. Trying yt-dlp...")
+                    
+                # 2. yt-dlp
+                if not track_success:
+                    search_query = f"ytsearch1:{track_query}"
+                    cmd_ytdlp = [
+                        "yt-dlp",
+                        "-x",
+                        "--audio-format", "mp3",
+                        "--embed-metadata",
+                        "--yes-playlist",
+                        "--output", "%(title)s.%(ext)s"
+                    ]
+                    if resolved_proxy_url:
+                        cmd_ytdlp += ["--proxy", resolved_proxy_url]
+                    cmd_ytdlp.append(search_query)
+                    
+                    logger.info(f"[{task_id}] [Track {idx+1}] Trying yt-dlp...")
+                    result_ytdlp = subprocess.run(cmd_ytdlp, cwd=temp_dir, env=env, capture_output=True, text=True)
+                    if result_ytdlp.returncode == 0:
+                        logger.info(f"[{task_id}] [Track {idx+1}] yt-dlp succeeded.")
+                        track_success = True
+                    else:
+                        logger.warning(f"[{task_id}] [Track {idx+1}] yt-dlp failed. Trying Tubidy...")
+                        
+                # 3. Tubidy
+                if not track_success:
+                    try:
+                        logger.info(f"[{task_id}] [Track {idx+1}] Trying Tubidy...")
+                        search_results = search_tubidy(track_query, resolved_proxy_url)
+                        if search_results:
+                            first_result = search_results[0]
+                            mp3_url = get_tubidy_download_link(first_result["id"], resolved_proxy_url)
+                            filename = f"{first_result['title']}.mp3"
+                            filename = "".join([c for c in filename if c.isalnum() or c in " .-_()"]).strip()
+                            if not filename.endswith(".mp3"):
+                                filename += ".mp3"
+                            dest_path = os.path.join(temp_dir, filename)
+                            download_file_stream(mp3_url, dest_path, resolved_proxy_url)
+                            tag_mp3(dest_path, filename)
+                            track_success = True
+                            logger.info(f"[{task_id}] [Track {idx+1}] Tubidy succeeded.")
+                    except Exception as e:
+                        logger.error(f"[{task_id}] [Track {idx+1}] Tubidy failed: {e}")
+                        
+                if track_success:
+                    download_count += 1
+                    
+            if download_count > 0:
+                download_successful = True
+                logger.info(f"[{task_id}] Playlist download finished. Successfully downloaded {download_count} tracks.")
+            else:
+                raise Exception("Failed to download any tracks from the playlist.")
+
+        else:
             is_tubidy = "tubidy" in query.lower()
             is_spotify = "spotify.com" in query.lower()
             is_youtube = "youtube.com" in query.lower() or "youtu.be" in query.lower()

k8s/base/supabase-basket.yaml

@@ -1810,7 +1810,7 @@ spec:
           imagePullPolicy: IfNotPresent
           env:
             - name: API_EXTERNAL_URL
-              value: "http://supabase.local"
+              value: "https://basket.techarvest.co.zw"
             - name: DB_DRIVER
               value: "postgres"
             - name: DB_SSL
@@ -1824,7 +1824,9 @@ spec:
             - name: GOTRUE_DISABLE_SIGNUP
               value: "false"
             - name: GOTRUE_EXTERNAL_ANONYMOUS_USERS_ENABLED
-              value: "false"
+              value: "true"
+            - name: GOTRUE_EXTERNAL_ANONYMOUS_ENABLED
+              value: "true"
             - name: GOTRUE_EXTERNAL_EMAIL_ENABLED
               value: "true"
             - name: GOTRUE_EXTERNAL_PHONE_ENABLED
@@ -1848,7 +1850,7 @@ spec:
             - name: GOTRUE_MAILER_URLPATHS_RECOVERY
               value: "/auth/v1/verify"
             - name: GOTRUE_SITE_URL
-              value: "http://supabase.local"
+              value: "https://basket.techarvest.co.zw"
             - name: GOTRUE_SMS_AUTOCONFIRM
               value: "false"
             - name: GOTRUE_SMTP_ADMIN_EMAIL

k8s/family-apps/musicseerr.yaml

@@ -54,7 +54,7 @@ spec:
       containers:
       - name: musicseerr
         image: frankchine/geocrop-musicseerr:latest
-        imagePullPolicy: IfNotPresent
+        imagePullPolicy: Always
         env:
         - name: GLUETUN_PROXY_URL
           value: "http://gluetun-svc:8888"

k8s/nextgen/manual-deployments.yaml

@@ -123,3 +123,44 @@ spec:
                 name: next-gen-dev
                 port:
                   number: 80
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: next-gen-main
+  namespace: nextgen
+spec:
+  selector:
+    app: next-gen-main
+  ports:
+    - name: http
+      protocol: TCP
+      port: 80
+      targetPort: http
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: next-gen-main
+  namespace: nextgen
+  annotations:
+    cert-manager.io/cluster-issuer: "letsencrypt-prod"
+    nginx.ingress.kubernetes.io/ssl-redirect: "false"
+    nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
+spec:
+  ingressClassName: nginx
+  tls:
+    - hosts:
+        - next-gen.techarvest.co.zw
+      secretName: next-gen-main-tls
+  rules:
+    - host: next-gen.techarvest.co.zw
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: next-gen-main
+                port:
+                  number: 80

terraform/main.tf

@@ -63,6 +63,24 @@ resource "kubernetes_namespace" "portainer" {
   }
 }
 
+resource "kubernetes_namespace" "argo" {
+  metadata {
+    name = "argo"
+  }
+}
+
+resource "kubernetes_namespace" "kubernetes_dashboard" {
+  metadata {
+    name = "kubernetes-dashboard"
+  }
+}
+
+resource "kubernetes_namespace" "dressup" {
+  metadata {
+    name = "dressup"
+  }
+}
+
 # ==========================================
 # PORTAINER (kubectl deployed)
 # Port installed via kubectl manifest at:
@@ -150,3 +168,25 @@ resource "kubernetes_namespace" "portainer" {
 #   - argo-server
 #   - workflow-controller
 # ==========================================
+
+# ==========================================
+# PRIORITY CLASSES
+# ==========================================
+
+resource "kubernetes_priority_class_v1" "dressup_priority" {
+  metadata {
+    name = "dressup-priority"
+  }
+  value          = 800000
+  global_default = false
+  description    = "High priority scheduling for dressup app workloads during spikes."
+}
+
+resource "kubernetes_priority_class_v1" "inference_worker_priority" {
+  metadata {
+    name = "inference-worker-priority"
+  }
+  value          = 900000
+  global_default = false
+  description    = "High priority scheduling for GeoCrop inference and worker pods."
+}

terraform/terraform.tfstate

@@ -1,10 +1,45 @@
 {
   "version": 4,
   "terraform_version": "1.14.9",
-  "serial": 19,
+  "serial": 25,
   "lineage": "80e41663-9b90-f349-cc6c-be6879179605",
   "outputs": {},
   "resources": [
+    {
+      "mode": "managed",
+      "type": "kubernetes_namespace",
+      "name": "argo",
+      "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]",
+      "instances": [
+        {
+          "schema_version": 0,
+          "attributes": {
+            "id": "argo",
+            "metadata": [
+              {
+                "annotations": {},
+                "generate_name": "",
+                "generation": 0,
+                "labels": {},
+                "name": "argo",
+                "resource_version": "3086761",
+                "uid": "2c5f8fa2-cdb8-4ff8-9c00-f1f5f589a9f1"
+              }
+            ],
+            "timeouts": null,
+            "wait_for_default_service_account": null
+          },
+          "sensitive_attributes": [],
+          "identity_schema_version": 1,
+          "identity": {
+            "api_version": "v1",
+            "kind": "Namespace",
+            "name": "argo"
+          },
+          "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiZGVsZXRlIjozMDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjAifQ=="
+        }
+      ]
+    },
     {
       "mode": "managed",
       "type": "kubernetes_namespace",
@@ -110,6 +145,41 @@
         }
       ]
     },
+    {
+      "mode": "managed",
+      "type": "kubernetes_namespace",
+      "name": "dressup",
+      "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]",
+      "instances": [
+        {
+          "schema_version": 0,
+          "attributes": {
+            "id": "dressup",
+            "metadata": [
+              {
+                "annotations": {},
+                "generate_name": "",
+                "generation": 0,
+                "labels": {},
+                "name": "dressup",
+                "resource_version": "1817654",
+                "uid": "93ccf1db-35f5-47ba-89c8-d5a186a4f9e6"
+              }
+            ],
+            "timeouts": null,
+            "wait_for_default_service_account": null
+          },
+          "sensitive_attributes": [],
+          "identity_schema_version": 1,
+          "identity": {
+            "api_version": "v1",
+            "kind": "Namespace",
+            "name": "dressup"
+          },
+          "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiZGVsZXRlIjozMDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjAifQ=="
+        }
+      ]
+    },
     {
       "mode": "managed",
       "type": "kubernetes_namespace",
@@ -180,6 +250,41 @@
         }
       ]
     },
+    {
+      "mode": "managed",
+      "type": "kubernetes_namespace",
+      "name": "kubernetes_dashboard",
+      "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]",
+      "instances": [
+        {
+          "schema_version": 0,
+          "attributes": {
+            "id": "kubernetes-dashboard",
+            "metadata": [
+              {
+                "annotations": {},
+                "generate_name": "",
+                "generation": 0,
+                "labels": {},
+                "name": "kubernetes-dashboard",
+                "resource_version": "3212",
+                "uid": "05cc388b-3829-43dd-ab0f-b3acf5170423"
+              }
+            ],
+            "timeouts": null,
+            "wait_for_default_service_account": null
+          },
+          "sensitive_attributes": [],
+          "identity_schema_version": 1,
+          "identity": {
+            "api_version": "v1",
+            "kind": "Namespace",
+            "name": "kubernetes-dashboard"
+          },
+          "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiZGVsZXRlIjozMDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjAifQ=="
+        }
+      ]
+    },
     {
       "mode": "managed",
       "type": "kubernetes_namespace",
@@ -284,6 +389,70 @@
           "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiZGVsZXRlIjozMDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjAifQ=="
         }
       ]
+    },
+    {
+      "mode": "managed",
+      "type": "kubernetes_priority_class_v1",
+      "name": "dressup_priority",
+      "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]",
+      "instances": [
+        {
+          "schema_version": 0,
+          "attributes": {
+            "description": "High priority scheduling for dressup app workloads during spikes.",
+            "global_default": false,
+            "id": "dressup-priority",
+            "metadata": [
+              {
+                "annotations": null,
+                "generate_name": "",
+                "generation": 1,
+                "labels": null,
+                "name": "dressup-priority",
+                "resource_version": "4396811",
+                "uid": "b4d9b507-3608-4ce4-8f53-646cca98bbe7"
+              }
+            ],
+            "preemption_policy": "PreemptLowerPriority",
+            "value": 800000
+          },
+          "sensitive_attributes": [],
+          "identity_schema_version": 0,
+          "private": "bnVsbA=="
+        }
+      ]
+    },
+    {
+      "mode": "managed",
+      "type": "kubernetes_priority_class_v1",
+      "name": "inference_worker_priority",
+      "provider": "provider[\"registry.terraform.io/hashicorp/kubernetes\"]",
+      "instances": [
+        {
+          "schema_version": 0,
+          "attributes": {
+            "description": "High priority scheduling for GeoCrop inference and worker pods.",
+            "global_default": false,
+            "id": "inference-worker-priority",
+            "metadata": [
+              {
+                "annotations": null,
+                "generate_name": "",
+                "generation": 1,
+                "labels": null,
+                "name": "inference-worker-priority",
+                "resource_version": "4396810",
+                "uid": "78d9337d-f1a5-496d-b63b-068141b1f3e3"
+              }
+            ],
+            "preemption_policy": "PreemptLowerPriority",
+            "value": 900000
+          },
+          "sensitive_attributes": [],
+          "identity_schema_version": 0,
+          "private": "bnVsbA=="
+        }
+      ]
     }
   ],
   "check_results": null