Spotify Web API collector: DR artist -> Spotify artist, releases since 2018

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-17 11:32:25 +00:00
co-authored by Claude Opus 5.5
parent 7e2764ce8c
commit 44e00b9793
2 changed files with 195 additions and 2 deletions
+9 -2
View File
@@ -1,6 +1,6 @@
//! airplay: collects what Danish radio stations play, for modelling when artists get played again. //! airplay: collects what Danish radio stations play, for modelling when artists get played again.
//! //!
//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive|oldpages] [--backfill] //! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|spotify-api|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive|oldpages] [--backfill]
//! airplay export nmf-denmark [--out data/nmf_denmark.csv] //! airplay export nmf-denmark [--out data/nmf_denmark.csv]
//! airplay stats [--db data/airplay.db] //! airplay stats [--db data/airplay.db]
//! airplay check [--db data/airplay.db] gaps, duplicates, DR cross-check, chart completeness //! airplay check [--db data/airplay.db] gaps, duplicates, DR cross-check, chart completeness
@@ -26,6 +26,7 @@ mod oldpages;
mod playlists; mod playlists;
mod rotations; mod rotations;
mod spotify; mod spotify;
mod spotify_api;
mod web; mod web;
use anyhow::Result; use anyhow::Result;
@@ -62,7 +63,7 @@ fn main() -> Result<()> {
rotations::store_all(&db, &dir, args.iter().any(|a| a == "--reparse")) rotations::store_all(&db, &dir, args.iter().any(|a| a == "--reparse"))
} }
_ => { _ => {
eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive|oldpages] [--backfill] eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|spotify-api|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive|oldpages] [--backfill]
//! airplay export nmf-denmark [--out data/nmf_denmark.csv]\n airplay stats [--db FILE]\n airplay check [--db FILE]\n airplay model panel|backtest|rank\n airplay serve [--addr HOST:PORT] [--prefix /airplay]"); //! airplay export nmf-denmark [--out data/nmf_denmark.csv]\n airplay stats [--db FILE]\n airplay check [--db FILE]\n airplay model panel|backtest|rank\n airplay serve [--addr HOST:PORT] [--prefix /airplay]");
std::process::exit(2); std::process::exit(2);
} }
@@ -132,6 +133,12 @@ fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> {
let r = spotify::collect(db, &kworb_http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 400 }), &mut log); let r = spotify::collect(db, &kworb_http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 400 }), &mut log);
step("spotify chart history (kworb)", r.map(|(l, p)| format!("{l} songs listed, {p} histories read")), &mut log); step("spotify chart history (kworb)", r.map(|(l, p)| format!("{l} songs listed, {p} histories read")), &mut log);
} }
if wants("spotify-api") {
// Spotify Web API: artists' release dates (needs /root/.spotify-credentials).
let lim = arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 500 });
let r = spotify_api::collect(db, lim, if backfill { 0 } else { 1200 }, &mut log);
step("spotify releases (web api)", r.map(|n| format!("{n} artists looked up")), &mut log);
}
if wants("playlists") { if wants("playlists") {
let r = playlists::snapshot(db, &http, &mut log); let r = playlists::snapshot(db, &http, &mut log);
step("spotify playlists (New Music Friday DK, Top 50 DK)", r.map(|n| format!("{n} changed and saved")), &mut log); step("spotify playlists (New Music Friday DK, Top 50 DK)", r.map(|n| format!("{n} changed and saved")), &mut log);
+186
View File
@@ -0,0 +1,186 @@
//! Release dates from Spotify's Web API (client credentials; no user login). For every DR artist,
//! today's pool first and then the archive-only artists (2019–24 pages):
//!
//! the Spotify artist: the link on the DR artist page (nmf_artist_lookups), else a search
//! (GET /v1/search?type=artist) accepted only on an exact name match
//! their releases: GET /v1/artists/{id}/albums?include_groups=album,single,appears_on
//! (album_group says which; appears_on singles are features on someone else's single)
//!
//! Credentials: SPOTIFY_CLIENT_ID / SPOTIFY_CLIENT_SECRET from the environment or from
//! /root/.spotify-credentials (KEY=value lines). Rate limits: a pause between requests, and
//! Retry-After on 429.
use crate::db::Db;
use anyhow::{anyhow, bail, Result};
use chrono::Utc;
use rusqlite::params;
use serde_json::Value;
use std::time::{Duration, Instant};
pub fn create_tables(db: &Db) -> Result<()> {
db.conn.execute_batch(
"CREATE TABLE IF NOT EXISTS spotify_artist_map (
dr_artist_id TEXT PRIMARY KEY, spotify_artist_id TEXT, -- NULL = no match found
source TEXT, -- 'dr page' | 'search'
spotify_name TEXT, fetched_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS spotify_releases (
spotify_artist_id TEXT NOT NULL, album_id TEXT NOT NULL, name TEXT,
album_type TEXT, album_group TEXT, release_date TEXT, date_precision TEXT, total_tracks INTEGER,
PRIMARY KEY (spotify_artist_id, album_id)
);
CREATE TABLE IF NOT EXISTS spotify_release_lookups (
spotify_artist_id TEXT PRIMARY KEY, releases INTEGER, fetched_at TEXT NOT NULL
);",
)?;
Ok(())
}
fn credentials() -> Result<(String, String)> {
let mut id = std::env::var("SPOTIFY_CLIENT_ID").ok();
let mut secret = std::env::var("SPOTIFY_CLIENT_SECRET").ok();
if let Ok(f) = std::fs::read_to_string("/root/.spotify-credentials") {
for l in f.lines() {
if let Some((k, v)) = l.split_once('=') {
let v = v.trim().trim_matches('"').to_string();
match k.trim() { "SPOTIFY_CLIENT_ID" if id.is_none() => id = Some(v), "SPOTIFY_CLIENT_SECRET" if secret.is_none() => secret = Some(v), _ => {} }
}
}
}
match (id, secret) { (Some(i), Some(s)) if !i.is_empty() && !s.is_empty() => Ok((i, s)), _ => bail!("no Spotify credentials (SPOTIFY_CLIENT_ID / SPOTIFY_CLIENT_SECRET)") }
}
pub struct Api { client: reqwest::blocking::Client, id: String, secret: String, token: String, expires: Instant, delay: Duration }
impl Api {
pub fn new(delay_ms: u64) -> Result<Api> {
let (id, secret) = credentials()?;
let client = reqwest::blocking::Client::builder()
.user_agent("AirplayPredictor/0.1 (research)").timeout(Duration::from_secs(30)).build()?;
let mut api = Api { client, id, secret, token: String::new(), expires: Instant::now(), delay: Duration::from_millis(delay_ms) };
api.refresh()?;
Ok(api)
}
fn refresh(&mut self) -> Result<()> {
let r = self.client.post("https://accounts.spotify.com/api/token")
.basic_auth(&self.id, Some(&self.secret)).form(&[("grant_type", "client_credentials")]).send()?;
let status = r.status();
let v: Value = r.json().unwrap_or(Value::Null);
let Some(t) = v["access_token"].as_str() else { bail!("Spotify token: {status} {}", v["error_description"].as_str().or(v["error"].as_str()).unwrap_or("")) };
self.token = t.to_string();
self.expires = Instant::now() + Duration::from_secs(v["expires_in"].as_u64().unwrap_or(3600).saturating_sub(120));
Ok(())
}
/// GET a Web API URL; None on 404.
pub fn get(&mut self, url: &str) -> Result<Option<Value>> {
let mut last = anyhow!("no attempt");
for attempt in 0..6 {
if Instant::now() >= self.expires { self.refresh()?; }
std::thread::sleep(self.delay);
let r = match self.client.get(url).bearer_auth(&self.token).send() { Ok(r) => r, Err(e) => { last = anyhow!("{url}: {e}"); std::thread::sleep(Duration::from_secs(5 << attempt)); continue; } };
match r.status().as_u16() {
200 => return Ok(Some(r.json()?)),
404 => return Ok(None),
401 => { self.refresh()?; last = anyhow!("401 from {url}"); }
429 => {
let wait = r.headers().get("retry-after").and_then(|v| v.to_str().ok()).and_then(|s| s.parse::<u64>().ok()).unwrap_or(30);
if wait > 3600 { bail!("Spotify rate limit: retry after {wait} s"); }
eprintln!("spotify: rate limited, waiting {wait} s");
std::thread::sleep(Duration::from_secs(wait + 1));
last = anyhow!("429 from {url}");
}
s if (400..500).contains(&s) => bail!("{s} from {url}: {}", r.text().unwrap_or_default().chars().take(200).collect::<String>()),
s => { last = anyhow!("{s} from {url}: {}", r.text().unwrap_or_default().chars().take(200).collect::<String>()); std::thread::sleep(Duration::from_secs(5 << attempt)); }
}
}
Err(last)
}
}
fn key(s: &str) -> String { s.to_lowercase().chars().filter(|c| c.is_alphanumeric()).collect() }
fn urlencode(s: &str) -> String {
s.bytes().map(|b| match b { b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => (b as char).to_string(), _ => format!("%{b:02X}") }).collect()
}
/// Map DR artists to Spotify artists, then read each one's releases. Returns artists done.
pub fn collect(db: &Db, limit: usize, refresh: usize, log: &mut Vec<String>) -> Result<usize> {
create_tables(db)?;
let mut api = Api::new(120)?;
// Today's pool first (with the DR page's Spotify link where there is one), then archive-only artists.
let todo: Vec<(String, String, Option<String>)> = db.conn.prepare(
"SELECT a.artist_id, a.name, l.spotify_artist_id, 0 FROM dr_artists a LEFT JOIN nmf_artist_lookups l ON l.dr_artist_id = a.artist_id
WHERE a.artist_id NOT IN (SELECT dr_artist_id FROM spotify_artist_map)
UNION ALL
SELECT m.artist_id, m.name, NULL, 1 FROM old_artist_meta m
WHERE m.name IS NOT NULL AND m.artist_id NOT IN (SELECT artist_id FROM dr_artists) AND m.artist_id NOT IN (SELECT dr_artist_id FROM spotify_artist_map)
ORDER BY 4, 1")?
.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?.collect::<Result<_, _>>()?;
eprintln!("spotify: {} artists to look up", todo.len());
let mut done = 0;
for (dr_id, name, linked) in todo.into_iter().take(limit) {
let (sid, source, sname) = if let Some(s) = linked { (Some(s), "dr page", None) } else {
let q = format!("https://api.spotify.com/v1/search?type=artist&limit=10&q={}", urlencode(&name));
let v = match api.get(&q) { Ok(v) => v.unwrap_or(Value::Null), Err(e) => { log.push(format!("spotify search '{name}': {e}")); if e.to_string().contains("rate limit") { break; } continue; } };
let items = v["artists"]["items"].as_array().cloned().unwrap_or_default();
// Exact name only; among those the first (Spotify ranks by relevance and popularity).
let best = items.iter().find(|i| key(i["name"].as_str().unwrap_or_default()) == key(&name) && !key(&name).is_empty());
(best.and_then(|b| b["id"].as_str()).map(str::to_string), "search", best.and_then(|b| b["name"].as_str()).map(str::to_string))
};
if let Some(s) = &sid {
let have: i64 = db.conn.query_row("SELECT count(*) FROM spotify_release_lookups WHERE spotify_artist_id = ?1", [s], |r| r.get(0))?;
if have == 0 {
match releases(&mut api, db, s) {
Ok(n) => { db.conn.execute("INSERT OR REPLACE INTO spotify_release_lookups VALUES (?1, ?2, ?3)", params![s, n as i64, Utc::now().to_rfc3339()])?; }
Err(e) => { log.push(format!("spotify releases {s}: {e}")); if e.to_string().contains("rate limit") { break; } continue; }
}
}
}
db.conn.execute("INSERT OR REPLACE INTO spotify_artist_map VALUES (?1, ?2, ?3, ?4, ?5)", params![dr_id, sid, source, sname, Utc::now().to_rfc3339()])?;
done += 1;
if done % 250 == 0 { eprintln!("spotify: {done} artists"); }
}
// Then catch new releases: re-read today's pool artists last read 7+ days ago, oldest first.
let stale: Vec<String> = db.conn.prepare(
"SELECT DISTINCT l.spotify_artist_id FROM spotify_release_lookups l JOIN spotify_artist_map m ON m.spotify_artist_id = l.spotify_artist_id
JOIN dr_artists a ON a.artist_id = m.dr_artist_id WHERE l.fetched_at < datetime('now', '-7 days') ORDER BY l.fetched_at")?
.query_map([], |r| r.get(0))?.collect::<Result<_, _>>()?;
for s in stale.into_iter().take(refresh) {
match releases(&mut api, db, &s) {
Ok(n) => { db.conn.execute("INSERT OR REPLACE INTO spotify_release_lookups VALUES (?1, ?2, ?3)", params![s, n as i64, Utc::now().to_rfc3339()])?; }
Err(e) => { log.push(format!("spotify releases {s}: {e}")); break; }
}
}
Ok(done)
}
/// The artist's releases since 2018, one release type at a time, newest first (new apps get 10
/// per page, so paging stops at the first release before 2018).
fn releases(api: &mut Api, db: &Db, sid: &str) -> Result<usize> {
let mut n = 0;
let tx = db.conn.unchecked_transaction()?;
for group in ["single", "album", "appears_on"] {
let mut url = Some(format!("https://api.spotify.com/v1/artists/{sid}/albums?include_groups={group}&limit=10"));
let mut pages = 0;
while let Some(u) = url.take() {
let Some(v) = api.get(&u)? else { break };
let mut old = false;
for a in v["items"].as_array().into_iter().flatten() {
let date = a["release_date"].as_str().unwrap_or_default();
if date < "2018" { old = true; continue; }
// Features on compilations are noise; keep singles the artist is on.
if group == "appears_on" && a["album_type"].as_str() != Some("single") { continue; }
tx.execute("INSERT OR REPLACE INTO spotify_releases VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
params![sid, a["id"].as_str(), a["name"].as_str(), a["album_type"].as_str(), group, date,
a["release_date_precision"].as_str(), a["total_tracks"].as_i64()])?;
n += 1;
}
pages += 1;
url = v["next"].as_str().filter(|_| !old && pages < 30).map(str::to_string);
}
}
tx.commit()?;
Ok(n)
}