From 44e00b9793acb3e5dd239412b671b2bc4362a9b3 Mon Sep 17 00:00:00 2001 From: moritsbw-cpu Date: Tue, 17 Nov 2026 11:32:25 +0000 Subject: [PATCH] Spotify Web API collector: DR artist -> Spotify artist, releases since 2018 Co-Authored-By: Claude Opus 5.5 --- src/main.rs | 11 ++- src/spotify_api.rs | 186 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 195 insertions(+), 2 deletions(-) create mode 100644 src/spotify_api.rs diff --git a/src/main.rs b/src/main.rs index 53d84c5..6bdef14 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,6 @@ //! airplay: collects what Danish radio stations play, for modelling when artists get played again. //! -//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive|oldpages] [--backfill] +//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|spotify-api|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive|oldpages] [--backfill] //! airplay export nmf-denmark [--out data/nmf_denmark.csv] //! airplay stats [--db data/airplay.db] //! airplay check [--db data/airplay.db] gaps, duplicates, DR cross-check, chart completeness @@ -26,6 +26,7 @@ mod oldpages; mod playlists; mod rotations; mod spotify; +mod spotify_api; mod web; use anyhow::Result; @@ -62,7 +63,7 @@ fn main() -> Result<()> { rotations::store_all(&db, &dir, args.iter().any(|a| a == "--reparse")) } _ => { - eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive|oldpages] [--backfill] + eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|spotify-api|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive|oldpages] [--backfill] //! airplay export nmf-denmark [--out data/nmf_denmark.csv]\n airplay stats [--db FILE]\n airplay check [--db FILE]\n airplay model panel|backtest|rank\n airplay serve [--addr HOST:PORT] [--prefix /airplay]"); std::process::exit(2); } @@ -132,6 +133,12 @@ fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> { let r = spotify::collect(db, &kworb_http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 400 }), &mut log); step("spotify chart history (kworb)", r.map(|(l, p)| format!("{l} songs listed, {p} histories read")), &mut log); } + if wants("spotify-api") { + // Spotify Web API: artists' release dates (needs /root/.spotify-credentials). + let lim = arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 500 }); + let r = spotify_api::collect(db, lim, if backfill { 0 } else { 1200 }, &mut log); + step("spotify releases (web api)", r.map(|n| format!("{n} artists looked up")), &mut log); + } if wants("playlists") { let r = playlists::snapshot(db, &http, &mut log); step("spotify playlists (New Music Friday DK, Top 50 DK)", r.map(|n| format!("{n} changed and saved")), &mut log); diff --git a/src/spotify_api.rs b/src/spotify_api.rs new file mode 100644 index 0000000..2585b5a --- /dev/null +++ b/src/spotify_api.rs @@ -0,0 +1,186 @@ +//! Release dates from Spotify's Web API (client credentials; no user login). For every DR artist, +//! today's pool first and then the archive-only artists (2019–24 pages): +//! +//! the Spotify artist: the link on the DR artist page (nmf_artist_lookups), else a search +//! (GET /v1/search?type=artist) accepted only on an exact name match +//! their releases: GET /v1/artists/{id}/albums?include_groups=album,single,appears_on +//! (album_group says which; appears_on singles are features on someone else's single) +//! +//! Credentials: SPOTIFY_CLIENT_ID / SPOTIFY_CLIENT_SECRET from the environment or from +//! /root/.spotify-credentials (KEY=value lines). Rate limits: a pause between requests, and +//! Retry-After on 429. + +use crate::db::Db; +use anyhow::{anyhow, bail, Result}; +use chrono::Utc; +use rusqlite::params; +use serde_json::Value; +use std::time::{Duration, Instant}; + +pub fn create_tables(db: &Db) -> Result<()> { + db.conn.execute_batch( + "CREATE TABLE IF NOT EXISTS spotify_artist_map ( + dr_artist_id TEXT PRIMARY KEY, spotify_artist_id TEXT, -- NULL = no match found + source TEXT, -- 'dr page' | 'search' + spotify_name TEXT, fetched_at TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS spotify_releases ( + spotify_artist_id TEXT NOT NULL, album_id TEXT NOT NULL, name TEXT, + album_type TEXT, album_group TEXT, release_date TEXT, date_precision TEXT, total_tracks INTEGER, + PRIMARY KEY (spotify_artist_id, album_id) + ); + CREATE TABLE IF NOT EXISTS spotify_release_lookups ( + spotify_artist_id TEXT PRIMARY KEY, releases INTEGER, fetched_at TEXT NOT NULL + );", + )?; + Ok(()) +} + +fn credentials() -> Result<(String, String)> { + let mut id = std::env::var("SPOTIFY_CLIENT_ID").ok(); + let mut secret = std::env::var("SPOTIFY_CLIENT_SECRET").ok(); + if let Ok(f) = std::fs::read_to_string("/root/.spotify-credentials") { + for l in f.lines() { + if let Some((k, v)) = l.split_once('=') { + let v = v.trim().trim_matches('"').to_string(); + match k.trim() { "SPOTIFY_CLIENT_ID" if id.is_none() => id = Some(v), "SPOTIFY_CLIENT_SECRET" if secret.is_none() => secret = Some(v), _ => {} } + } + } + } + match (id, secret) { (Some(i), Some(s)) if !i.is_empty() && !s.is_empty() => Ok((i, s)), _ => bail!("no Spotify credentials (SPOTIFY_CLIENT_ID / SPOTIFY_CLIENT_SECRET)") } +} + +pub struct Api { client: reqwest::blocking::Client, id: String, secret: String, token: String, expires: Instant, delay: Duration } + +impl Api { + pub fn new(delay_ms: u64) -> Result { + let (id, secret) = credentials()?; + let client = reqwest::blocking::Client::builder() + .user_agent("AirplayPredictor/0.1 (research)").timeout(Duration::from_secs(30)).build()?; + let mut api = Api { client, id, secret, token: String::new(), expires: Instant::now(), delay: Duration::from_millis(delay_ms) }; + api.refresh()?; + Ok(api) + } + + fn refresh(&mut self) -> Result<()> { + let r = self.client.post("https://accounts.spotify.com/api/token") + .basic_auth(&self.id, Some(&self.secret)).form(&[("grant_type", "client_credentials")]).send()?; + let status = r.status(); + let v: Value = r.json().unwrap_or(Value::Null); + let Some(t) = v["access_token"].as_str() else { bail!("Spotify token: {status} {}", v["error_description"].as_str().or(v["error"].as_str()).unwrap_or("")) }; + self.token = t.to_string(); + self.expires = Instant::now() + Duration::from_secs(v["expires_in"].as_u64().unwrap_or(3600).saturating_sub(120)); + Ok(()) + } + + /// GET a Web API URL; None on 404. + pub fn get(&mut self, url: &str) -> Result> { + let mut last = anyhow!("no attempt"); + for attempt in 0..6 { + if Instant::now() >= self.expires { self.refresh()?; } + std::thread::sleep(self.delay); + let r = match self.client.get(url).bearer_auth(&self.token).send() { Ok(r) => r, Err(e) => { last = anyhow!("{url}: {e}"); std::thread::sleep(Duration::from_secs(5 << attempt)); continue; } }; + match r.status().as_u16() { + 200 => return Ok(Some(r.json()?)), + 404 => return Ok(None), + 401 => { self.refresh()?; last = anyhow!("401 from {url}"); } + 429 => { + let wait = r.headers().get("retry-after").and_then(|v| v.to_str().ok()).and_then(|s| s.parse::().ok()).unwrap_or(30); + if wait > 3600 { bail!("Spotify rate limit: retry after {wait} s"); } + eprintln!("spotify: rate limited, waiting {wait} s"); + std::thread::sleep(Duration::from_secs(wait + 1)); + last = anyhow!("429 from {url}"); + } + s if (400..500).contains(&s) => bail!("{s} from {url}: {}", r.text().unwrap_or_default().chars().take(200).collect::()), + s => { last = anyhow!("{s} from {url}: {}", r.text().unwrap_or_default().chars().take(200).collect::()); std::thread::sleep(Duration::from_secs(5 << attempt)); } + } + } + Err(last) + } +} + +fn key(s: &str) -> String { s.to_lowercase().chars().filter(|c| c.is_alphanumeric()).collect() } + +fn urlencode(s: &str) -> String { + s.bytes().map(|b| match b { b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => (b as char).to_string(), _ => format!("%{b:02X}") }).collect() +} + +/// Map DR artists to Spotify artists, then read each one's releases. Returns artists done. +pub fn collect(db: &Db, limit: usize, refresh: usize, log: &mut Vec) -> Result { + create_tables(db)?; + let mut api = Api::new(120)?; + // Today's pool first (with the DR page's Spotify link where there is one), then archive-only artists. + let todo: Vec<(String, String, Option)> = db.conn.prepare( + "SELECT a.artist_id, a.name, l.spotify_artist_id, 0 FROM dr_artists a LEFT JOIN nmf_artist_lookups l ON l.dr_artist_id = a.artist_id + WHERE a.artist_id NOT IN (SELECT dr_artist_id FROM spotify_artist_map) + UNION ALL + SELECT m.artist_id, m.name, NULL, 1 FROM old_artist_meta m + WHERE m.name IS NOT NULL AND m.artist_id NOT IN (SELECT artist_id FROM dr_artists) AND m.artist_id NOT IN (SELECT dr_artist_id FROM spotify_artist_map) + ORDER BY 4, 1")? + .query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?.collect::>()?; + eprintln!("spotify: {} artists to look up", todo.len()); + let mut done = 0; + for (dr_id, name, linked) in todo.into_iter().take(limit) { + let (sid, source, sname) = if let Some(s) = linked { (Some(s), "dr page", None) } else { + let q = format!("https://api.spotify.com/v1/search?type=artist&limit=10&q={}", urlencode(&name)); + let v = match api.get(&q) { Ok(v) => v.unwrap_or(Value::Null), Err(e) => { log.push(format!("spotify search '{name}': {e}")); if e.to_string().contains("rate limit") { break; } continue; } }; + let items = v["artists"]["items"].as_array().cloned().unwrap_or_default(); + // Exact name only; among those the first (Spotify ranks by relevance and popularity). + let best = items.iter().find(|i| key(i["name"].as_str().unwrap_or_default()) == key(&name) && !key(&name).is_empty()); + (best.and_then(|b| b["id"].as_str()).map(str::to_string), "search", best.and_then(|b| b["name"].as_str()).map(str::to_string)) + }; + if let Some(s) = &sid { + let have: i64 = db.conn.query_row("SELECT count(*) FROM spotify_release_lookups WHERE spotify_artist_id = ?1", [s], |r| r.get(0))?; + if have == 0 { + match releases(&mut api, db, s) { + Ok(n) => { db.conn.execute("INSERT OR REPLACE INTO spotify_release_lookups VALUES (?1, ?2, ?3)", params![s, n as i64, Utc::now().to_rfc3339()])?; } + Err(e) => { log.push(format!("spotify releases {s}: {e}")); if e.to_string().contains("rate limit") { break; } continue; } + } + } + } + db.conn.execute("INSERT OR REPLACE INTO spotify_artist_map VALUES (?1, ?2, ?3, ?4, ?5)", params![dr_id, sid, source, sname, Utc::now().to_rfc3339()])?; + done += 1; + if done % 250 == 0 { eprintln!("spotify: {done} artists"); } + } + // Then catch new releases: re-read today's pool artists last read 7+ days ago, oldest first. + let stale: Vec = db.conn.prepare( + "SELECT DISTINCT l.spotify_artist_id FROM spotify_release_lookups l JOIN spotify_artist_map m ON m.spotify_artist_id = l.spotify_artist_id + JOIN dr_artists a ON a.artist_id = m.dr_artist_id WHERE l.fetched_at < datetime('now', '-7 days') ORDER BY l.fetched_at")? + .query_map([], |r| r.get(0))?.collect::>()?; + for s in stale.into_iter().take(refresh) { + match releases(&mut api, db, &s) { + Ok(n) => { db.conn.execute("INSERT OR REPLACE INTO spotify_release_lookups VALUES (?1, ?2, ?3)", params![s, n as i64, Utc::now().to_rfc3339()])?; } + Err(e) => { log.push(format!("spotify releases {s}: {e}")); break; } + } + } + Ok(done) +} + +/// The artist's releases since 2018, one release type at a time, newest first (new apps get 10 +/// per page, so paging stops at the first release before 2018). +fn releases(api: &mut Api, db: &Db, sid: &str) -> Result { + let mut n = 0; + let tx = db.conn.unchecked_transaction()?; + for group in ["single", "album", "appears_on"] { + let mut url = Some(format!("https://api.spotify.com/v1/artists/{sid}/albums?include_groups={group}&limit=10")); + let mut pages = 0; + while let Some(u) = url.take() { + let Some(v) = api.get(&u)? else { break }; + let mut old = false; + for a in v["items"].as_array().into_iter().flatten() { + let date = a["release_date"].as_str().unwrap_or_default(); + if date < "2018" { old = true; continue; } + // Features on compilations are noise; keep singles the artist is on. + if group == "appears_on" && a["album_type"].as_str() != Some("single") { continue; } + tx.execute("INSERT OR REPLACE INTO spotify_releases VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", + params![sid, a["id"].as_str(), a["name"].as_str(), a["album_type"].as_str(), group, date, + a["release_date_precision"].as_str(), a["total_tracks"].as_i64()])?; + n += 1; + } + pages += 1; + url = v["next"].as_str().filter(|_| !old && pages < 30).map(str::to_string); + } + } + tx.commit()?; + Ok(n) +}