fix(musicseerr): implement proxy fallback for Spotify playlist scraping
Build and Push Docker Images / deploy (push) Blocked by required conditions
Details
Build and Push Docker Images / build (musicseerr) (push) Failing after 3m9s
Details
Build and Push Docker Images / build (nextgen) (push) Failing after 10s
Details
Build and Push Docker Images / build (api) (push) Failing after 5m34s
Details
Build and Push Docker Images / build (web) (push) Successful in 7m26s
Details
Build and Push Docker Images / build (worker) (push) Has been cancelled
Details
Build and Push Docker Images / deploy (push) Blocked by required conditions
Details
Build and Push Docker Images / build (musicseerr) (push) Failing after 3m9s
Details
Build and Push Docker Images / build (nextgen) (push) Failing after 10s
Details
Build and Push Docker Images / build (api) (push) Failing after 5m34s
Details
Build and Push Docker Images / build (web) (push) Successful in 7m26s
Details
Build and Push Docker Images / build (worker) (push) Has been cancelled
Details
This commit is contained in:
parent
d91a4abee4
commit
4164bde79b
|
|
@ -275,31 +275,35 @@ def scrape_spotify_playlist(url: str, proxy_url: str) -> list:
|
||||||
playlist_id = match.group(1)
|
playlist_id = match.group(1)
|
||||||
embed_url = f"https://open.spotify.com/embed/playlist/{playlist_id}"
|
embed_url = f"https://open.spotify.com/embed/playlist/{playlist_id}"
|
||||||
|
|
||||||
proxies = {"http": proxy_url, "https": proxy_url} if proxy_url else None
|
|
||||||
headers = {
|
headers = {
|
||||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||||
}
|
}
|
||||||
|
# Try with proxy first, then without proxy
|
||||||
logger.info(f"Scraping Spotify playlist via embed URL: {embed_url}")
|
for use_proxy in [True, False]:
|
||||||
try:
|
current_proxies = {"http": proxy_url, "https": proxy_url} if (use_proxy and proxy_url) else None
|
||||||
response = requests.get(embed_url, headers=headers, proxies=proxies, timeout=15)
|
proxy_desc = f"via proxy {proxy_url}" if current_proxies else "directly"
|
||||||
if response.status_code == 200:
|
logger.info(f"Scraping Spotify playlist via embed URL: {embed_url} ({proxy_desc})")
|
||||||
next_data_match = re.search(r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>', response.text)
|
try:
|
||||||
if next_data_match:
|
response = requests.get(embed_url, headers=headers, proxies=current_proxies, timeout=15)
|
||||||
data = json.loads(next_data_match.group(1))
|
if response.status_code == 200:
|
||||||
entity = data['props']['pageProps']['state']['data']['entity']
|
next_data_match = re.search(r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>', response.text)
|
||||||
tracks = entity.get('tracks', entity.get('trackList', []))
|
if next_data_match:
|
||||||
track_queries = []
|
data = json.loads(next_data_match.group(1))
|
||||||
for t in tracks:
|
entity = data['props']['pageProps']['state']['data']['entity']
|
||||||
title = t.get('title', '').strip()
|
tracks = entity.get('tracks', entity.get('trackList', []))
|
||||||
subtitle = t.get('subtitle', '').replace('\xa0', ' ').strip()
|
track_queries = []
|
||||||
if title and subtitle:
|
for t in tracks:
|
||||||
track_queries.append(f"{subtitle} - {title}")
|
title = t.get('title', '').strip()
|
||||||
elif title:
|
subtitle = t.get('subtitle', '').replace('\xa0', ' ').strip()
|
||||||
track_queries.append(title)
|
if title and subtitle:
|
||||||
return track_queries
|
track_queries.append(f"{subtitle} - {title}")
|
||||||
except Exception as e:
|
elif title:
|
||||||
logger.warning(f"Failed to scrape Spotify playlist: {e}")
|
track_queries.append(title)
|
||||||
|
return track_queries
|
||||||
|
logger.warning(f"Failed to scrape Spotify playlist {proxy_desc}: HTTP {response.status_code}")
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Failed to scrape Spotify playlist {proxy_desc}: {e}")
|
||||||
|
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue