Spotify Web API collector: DR artist -> Spotify artist, releases since 2018
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+9
-2
@@ -1,6 +1,6 @@
|
||||
//! airplay: collects what Danish radio stations play, for modelling when artists get played again.
|
||||
//!
|
||||
//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive|oldpages] [--backfill]
|
||||
//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|spotify-api|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive|oldpages] [--backfill]
|
||||
//! airplay export nmf-denmark [--out data/nmf_denmark.csv]
|
||||
//! airplay stats [--db data/airplay.db]
|
||||
//! airplay check [--db data/airplay.db] gaps, duplicates, DR cross-check, chart completeness
|
||||
@@ -26,6 +26,7 @@ mod oldpages;
|
||||
mod playlists;
|
||||
mod rotations;
|
||||
mod spotify;
|
||||
mod spotify_api;
|
||||
mod web;
|
||||
|
||||
use anyhow::Result;
|
||||
@@ -62,7 +63,7 @@ fn main() -> Result<()> {
|
||||
rotations::store_all(&db, &dir, args.iter().any(|a| a == "--reparse"))
|
||||
}
|
||||
_ => {
|
||||
eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive|oldpages] [--backfill]
|
||||
eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|spotify-api|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive|oldpages] [--backfill]
|
||||
//! airplay export nmf-denmark [--out data/nmf_denmark.csv]\n airplay stats [--db FILE]\n airplay check [--db FILE]\n airplay model panel|backtest|rank\n airplay serve [--addr HOST:PORT] [--prefix /airplay]");
|
||||
std::process::exit(2);
|
||||
}
|
||||
@@ -132,6 +133,12 @@ fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> {
|
||||
let r = spotify::collect(db, &kworb_http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 400 }), &mut log);
|
||||
step("spotify chart history (kworb)", r.map(|(l, p)| format!("{l} songs listed, {p} histories read")), &mut log);
|
||||
}
|
||||
if wants("spotify-api") {
|
||||
// Spotify Web API: artists' release dates (needs /root/.spotify-credentials).
|
||||
let lim = arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 500 });
|
||||
let r = spotify_api::collect(db, lim, if backfill { 0 } else { 1200 }, &mut log);
|
||||
step("spotify releases (web api)", r.map(|n| format!("{n} artists looked up")), &mut log);
|
||||
}
|
||||
if wants("playlists") {
|
||||
let r = playlists::snapshot(db, &http, &mut log);
|
||||
step("spotify playlists (New Music Friday DK, Top 50 DK)", r.map(|n| format!("{n} changed and saved")), &mut log);
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
//! Release dates from Spotify's Web API (client credentials; no user login). For every DR artist,
|
||||
//! today's pool first and then the archive-only artists (2019–24 pages):
|
||||
//!
|
||||
//! the Spotify artist: the link on the DR artist page (nmf_artist_lookups), else a search
|
||||
//! (GET /v1/search?type=artist) accepted only on an exact name match
|
||||
//! their releases: GET /v1/artists/{id}/albums?include_groups=album,single,appears_on
|
||||
//! (album_group says which; appears_on singles are features on someone else's single)
|
||||
//!
|
||||
//! Credentials: SPOTIFY_CLIENT_ID / SPOTIFY_CLIENT_SECRET from the environment or from
|
||||
//! /root/.spotify-credentials (KEY=value lines). Rate limits: a pause between requests, and
|
||||
//! Retry-After on 429.
|
||||
|
||||
use crate::db::Db;
|
||||
use anyhow::{anyhow, bail, Result};
|
||||
use chrono::Utc;
|
||||
use rusqlite::params;
|
||||
use serde_json::Value;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
pub fn create_tables(db: &Db) -> Result<()> {
|
||||
db.conn.execute_batch(
|
||||
"CREATE TABLE IF NOT EXISTS spotify_artist_map (
|
||||
dr_artist_id TEXT PRIMARY KEY, spotify_artist_id TEXT, -- NULL = no match found
|
||||
source TEXT, -- 'dr page' | 'search'
|
||||
spotify_name TEXT, fetched_at TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS spotify_releases (
|
||||
spotify_artist_id TEXT NOT NULL, album_id TEXT NOT NULL, name TEXT,
|
||||
album_type TEXT, album_group TEXT, release_date TEXT, date_precision TEXT, total_tracks INTEGER,
|
||||
PRIMARY KEY (spotify_artist_id, album_id)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS spotify_release_lookups (
|
||||
spotify_artist_id TEXT PRIMARY KEY, releases INTEGER, fetched_at TEXT NOT NULL
|
||||
);",
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn credentials() -> Result<(String, String)> {
|
||||
let mut id = std::env::var("SPOTIFY_CLIENT_ID").ok();
|
||||
let mut secret = std::env::var("SPOTIFY_CLIENT_SECRET").ok();
|
||||
if let Ok(f) = std::fs::read_to_string("/root/.spotify-credentials") {
|
||||
for l in f.lines() {
|
||||
if let Some((k, v)) = l.split_once('=') {
|
||||
let v = v.trim().trim_matches('"').to_string();
|
||||
match k.trim() { "SPOTIFY_CLIENT_ID" if id.is_none() => id = Some(v), "SPOTIFY_CLIENT_SECRET" if secret.is_none() => secret = Some(v), _ => {} }
|
||||
}
|
||||
}
|
||||
}
|
||||
match (id, secret) { (Some(i), Some(s)) if !i.is_empty() && !s.is_empty() => Ok((i, s)), _ => bail!("no Spotify credentials (SPOTIFY_CLIENT_ID / SPOTIFY_CLIENT_SECRET)") }
|
||||
}
|
||||
|
||||
pub struct Api { client: reqwest::blocking::Client, id: String, secret: String, token: String, expires: Instant, delay: Duration }
|
||||
|
||||
impl Api {
|
||||
pub fn new(delay_ms: u64) -> Result<Api> {
|
||||
let (id, secret) = credentials()?;
|
||||
let client = reqwest::blocking::Client::builder()
|
||||
.user_agent("AirplayPredictor/0.1 (research)").timeout(Duration::from_secs(30)).build()?;
|
||||
let mut api = Api { client, id, secret, token: String::new(), expires: Instant::now(), delay: Duration::from_millis(delay_ms) };
|
||||
api.refresh()?;
|
||||
Ok(api)
|
||||
}
|
||||
|
||||
fn refresh(&mut self) -> Result<()> {
|
||||
let r = self.client.post("https://accounts.spotify.com/api/token")
|
||||
.basic_auth(&self.id, Some(&self.secret)).form(&[("grant_type", "client_credentials")]).send()?;
|
||||
let status = r.status();
|
||||
let v: Value = r.json().unwrap_or(Value::Null);
|
||||
let Some(t) = v["access_token"].as_str() else { bail!("Spotify token: {status} {}", v["error_description"].as_str().or(v["error"].as_str()).unwrap_or("")) };
|
||||
self.token = t.to_string();
|
||||
self.expires = Instant::now() + Duration::from_secs(v["expires_in"].as_u64().unwrap_or(3600).saturating_sub(120));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// GET a Web API URL; None on 404.
|
||||
pub fn get(&mut self, url: &str) -> Result<Option<Value>> {
|
||||
let mut last = anyhow!("no attempt");
|
||||
for attempt in 0..6 {
|
||||
if Instant::now() >= self.expires { self.refresh()?; }
|
||||
std::thread::sleep(self.delay);
|
||||
let r = match self.client.get(url).bearer_auth(&self.token).send() { Ok(r) => r, Err(e) => { last = anyhow!("{url}: {e}"); std::thread::sleep(Duration::from_secs(5 << attempt)); continue; } };
|
||||
match r.status().as_u16() {
|
||||
200 => return Ok(Some(r.json()?)),
|
||||
404 => return Ok(None),
|
||||
401 => { self.refresh()?; last = anyhow!("401 from {url}"); }
|
||||
429 => {
|
||||
let wait = r.headers().get("retry-after").and_then(|v| v.to_str().ok()).and_then(|s| s.parse::<u64>().ok()).unwrap_or(30);
|
||||
if wait > 3600 { bail!("Spotify rate limit: retry after {wait} s"); }
|
||||
eprintln!("spotify: rate limited, waiting {wait} s");
|
||||
std::thread::sleep(Duration::from_secs(wait + 1));
|
||||
last = anyhow!("429 from {url}");
|
||||
}
|
||||
s if (400..500).contains(&s) => bail!("{s} from {url}: {}", r.text().unwrap_or_default().chars().take(200).collect::<String>()),
|
||||
s => { last = anyhow!("{s} from {url}: {}", r.text().unwrap_or_default().chars().take(200).collect::<String>()); std::thread::sleep(Duration::from_secs(5 << attempt)); }
|
||||
}
|
||||
}
|
||||
Err(last)
|
||||
}
|
||||
}
|
||||
|
||||
fn key(s: &str) -> String { s.to_lowercase().chars().filter(|c| c.is_alphanumeric()).collect() }
|
||||
|
||||
fn urlencode(s: &str) -> String {
|
||||
s.bytes().map(|b| match b { b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => (b as char).to_string(), _ => format!("%{b:02X}") }).collect()
|
||||
}
|
||||
|
||||
/// Map DR artists to Spotify artists, then read each one's releases. Returns artists done.
|
||||
pub fn collect(db: &Db, limit: usize, refresh: usize, log: &mut Vec<String>) -> Result<usize> {
|
||||
create_tables(db)?;
|
||||
let mut api = Api::new(120)?;
|
||||
// Today's pool first (with the DR page's Spotify link where there is one), then archive-only artists.
|
||||
let todo: Vec<(String, String, Option<String>)> = db.conn.prepare(
|
||||
"SELECT a.artist_id, a.name, l.spotify_artist_id, 0 FROM dr_artists a LEFT JOIN nmf_artist_lookups l ON l.dr_artist_id = a.artist_id
|
||||
WHERE a.artist_id NOT IN (SELECT dr_artist_id FROM spotify_artist_map)
|
||||
UNION ALL
|
||||
SELECT m.artist_id, m.name, NULL, 1 FROM old_artist_meta m
|
||||
WHERE m.name IS NOT NULL AND m.artist_id NOT IN (SELECT artist_id FROM dr_artists) AND m.artist_id NOT IN (SELECT dr_artist_id FROM spotify_artist_map)
|
||||
ORDER BY 4, 1")?
|
||||
.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?.collect::<Result<_, _>>()?;
|
||||
eprintln!("spotify: {} artists to look up", todo.len());
|
||||
let mut done = 0;
|
||||
for (dr_id, name, linked) in todo.into_iter().take(limit) {
|
||||
let (sid, source, sname) = if let Some(s) = linked { (Some(s), "dr page", None) } else {
|
||||
let q = format!("https://api.spotify.com/v1/search?type=artist&limit=10&q={}", urlencode(&name));
|
||||
let v = match api.get(&q) { Ok(v) => v.unwrap_or(Value::Null), Err(e) => { log.push(format!("spotify search '{name}': {e}")); if e.to_string().contains("rate limit") { break; } continue; } };
|
||||
let items = v["artists"]["items"].as_array().cloned().unwrap_or_default();
|
||||
// Exact name only; among those the first (Spotify ranks by relevance and popularity).
|
||||
let best = items.iter().find(|i| key(i["name"].as_str().unwrap_or_default()) == key(&name) && !key(&name).is_empty());
|
||||
(best.and_then(|b| b["id"].as_str()).map(str::to_string), "search", best.and_then(|b| b["name"].as_str()).map(str::to_string))
|
||||
};
|
||||
if let Some(s) = &sid {
|
||||
let have: i64 = db.conn.query_row("SELECT count(*) FROM spotify_release_lookups WHERE spotify_artist_id = ?1", [s], |r| r.get(0))?;
|
||||
if have == 0 {
|
||||
match releases(&mut api, db, s) {
|
||||
Ok(n) => { db.conn.execute("INSERT OR REPLACE INTO spotify_release_lookups VALUES (?1, ?2, ?3)", params![s, n as i64, Utc::now().to_rfc3339()])?; }
|
||||
Err(e) => { log.push(format!("spotify releases {s}: {e}")); if e.to_string().contains("rate limit") { break; } continue; }
|
||||
}
|
||||
}
|
||||
}
|
||||
db.conn.execute("INSERT OR REPLACE INTO spotify_artist_map VALUES (?1, ?2, ?3, ?4, ?5)", params![dr_id, sid, source, sname, Utc::now().to_rfc3339()])?;
|
||||
done += 1;
|
||||
if done % 250 == 0 { eprintln!("spotify: {done} artists"); }
|
||||
}
|
||||
// Then catch new releases: re-read today's pool artists last read 7+ days ago, oldest first.
|
||||
let stale: Vec<String> = db.conn.prepare(
|
||||
"SELECT DISTINCT l.spotify_artist_id FROM spotify_release_lookups l JOIN spotify_artist_map m ON m.spotify_artist_id = l.spotify_artist_id
|
||||
JOIN dr_artists a ON a.artist_id = m.dr_artist_id WHERE l.fetched_at < datetime('now', '-7 days') ORDER BY l.fetched_at")?
|
||||
.query_map([], |r| r.get(0))?.collect::<Result<_, _>>()?;
|
||||
for s in stale.into_iter().take(refresh) {
|
||||
match releases(&mut api, db, &s) {
|
||||
Ok(n) => { db.conn.execute("INSERT OR REPLACE INTO spotify_release_lookups VALUES (?1, ?2, ?3)", params![s, n as i64, Utc::now().to_rfc3339()])?; }
|
||||
Err(e) => { log.push(format!("spotify releases {s}: {e}")); break; }
|
||||
}
|
||||
}
|
||||
Ok(done)
|
||||
}
|
||||
|
||||
/// The artist's releases since 2018, one release type at a time, newest first (new apps get 10
|
||||
/// per page, so paging stops at the first release before 2018).
|
||||
fn releases(api: &mut Api, db: &Db, sid: &str) -> Result<usize> {
|
||||
let mut n = 0;
|
||||
let tx = db.conn.unchecked_transaction()?;
|
||||
for group in ["single", "album", "appears_on"] {
|
||||
let mut url = Some(format!("https://api.spotify.com/v1/artists/{sid}/albums?include_groups={group}&limit=10"));
|
||||
let mut pages = 0;
|
||||
while let Some(u) = url.take() {
|
||||
let Some(v) = api.get(&u)? else { break };
|
||||
let mut old = false;
|
||||
for a in v["items"].as_array().into_iter().flatten() {
|
||||
let date = a["release_date"].as_str().unwrap_or_default();
|
||||
if date < "2018" { old = true; continue; }
|
||||
// Features on compilations are noise; keep singles the artist is on.
|
||||
if group == "appears_on" && a["album_type"].as_str() != Some("single") { continue; }
|
||||
tx.execute("INSERT OR REPLACE INTO spotify_releases VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
|
||||
params![sid, a["id"].as_str(), a["name"].as_str(), a["album_type"].as_str(), group, date,
|
||||
a["release_date_precision"].as_str(), a["total_tracks"].as_i64()])?;
|
||||
n += 1;
|
||||
}
|
||||
pages += 1;
|
||||
url = v["next"].as_str().filter(|_| !old && pages < 30).map(str::to_string);
|
||||
}
|
||||
}
|
||||
tx.commit()?;
|
||||
Ok(n)
|
||||
}
|
||||
Reference in New Issue
Block a user