From 46fbec76af24d9712571efc83d53555054172b82 Mon Sep 17 00:00:00 2001 From: moritsbw-cpu Date: Tue, 17 Nov 2026 13:57:15 +0000 Subject: [PATCH] iTunes Search API collector; model release dates merge Deezer, iTunes and Spotify Co-Authored-By: Claude Opus 5.5 --- src/itunes.rs | 124 +++++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 11 +++- src/model/panel.rs | 31 +++++++++++- 3 files changed, 163 insertions(+), 3 deletions(-) create mode 100644 src/itunes.rs diff --git a/src/itunes.rs b/src/itunes.rs new file mode 100644 index 0000000..9c100cb --- /dev/null +++ b/src/itunes.rs @@ -0,0 +1,124 @@ +//! Release dates from Apple's iTunes Search API (free, no key; Danish storefront). For every DR +//! artist, today's pool first and then the archive-only artists (2019–24 pages): +//! +//! GET https://itunes.apple.com/search?term={name}&entity=musicArtist&country=dk&limit=10 +//! accepted only on an exact name match +//! GET https://itunes.apple.com/lookup?id={artist}&entity=album&limit=200&country=dk&sort=recent +//! singles ("… - Single"), EPs ("… - EP") and albums, with release dates +//! +//! Apple asks for about 20 requests a minute; it answers 403/429 when it's too many. + +use crate::db::Db; +use anyhow::{bail, Result}; +use chrono::Utc; +use rusqlite::params; +use serde_json::Value; +use std::time::Duration; + +pub fn create_tables(db: &Db) -> Result<()> { + db.conn.execute_batch( + "CREATE TABLE IF NOT EXISTS itunes_artist_map ( + dr_artist_id TEXT PRIMARY KEY, itunes_artist_id INTEGER, -- NULL = no exact match + itunes_name TEXT, fetched_at TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS itunes_releases ( + itunes_artist_id INTEGER NOT NULL, collection_id INTEGER NOT NULL, name TEXT, + kind TEXT, -- single | ep | album + release_date TEXT, track_count INTEGER, + PRIMARY KEY (itunes_artist_id, collection_id) + ); + CREATE TABLE IF NOT EXISTS itunes_release_lookups ( + itunes_artist_id INTEGER PRIMARY KEY, releases INTEGER, fetched_at TEXT NOT NULL + );", + )?; + Ok(()) +} + +struct Client { http: reqwest::blocking::Client, delay: Duration } + +impl Client { + /// GET JSON, pausing between requests; on 403/429 waits a minute, then two, ... (gives up after 5). + fn get(&self, url: &str) -> Result { + let mut wait = 60; + for _ in 0..6 { + std::thread::sleep(self.delay); + match self.http.get(url).send() { + Ok(r) if r.status().is_success() => return Ok(r.json().unwrap_or(Value::Null)), + Ok(r) if matches!(r.status().as_u16(), 403 | 429 | 503) => { eprintln!("itunes: {} — waiting {wait} s", r.status()); } + Ok(r) => bail!("{} from {url}", r.status()), + Err(e) => { eprintln!("itunes: {e} — waiting {wait} s"); } + } + std::thread::sleep(Duration::from_secs(wait)); + wait *= 2; + } + bail!("itunes: still refused after retries (rate limit)") + } +} + +fn key(s: &str) -> String { s.to_lowercase().chars().filter(|c| c.is_alphanumeric()).collect() } + +fn urlencode(s: &str) -> String { + s.bytes().map(|b| match b { b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => (b as char).to_string(), _ => format!("%{b:02X}") }).collect() +} + +/// Match artists, read their releases; then re-read today's pool artists last read 7+ days ago. +pub fn collect(db: &Db, limit: usize, refresh: usize, log: &mut Vec) -> Result { + create_tables(db)?; + let c = Client { + http: reqwest::blocking::Client::builder().user_agent("AirplayPredictor/0.1 (research)").timeout(Duration::from_secs(30)).build()?, + delay: Duration::from_millis(3100), + }; + let todo: Vec<(String, String)> = db.conn.prepare( + "SELECT artist_id, name, 0 FROM dr_artists WHERE artist_id NOT IN (SELECT dr_artist_id FROM itunes_artist_map) + UNION ALL + SELECT artist_id, name, 1 FROM old_artist_meta + WHERE name IS NOT NULL AND artist_id NOT IN (SELECT artist_id FROM dr_artists) AND artist_id NOT IN (SELECT dr_artist_id FROM itunes_artist_map) + ORDER BY 3, 1")? + .query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?.collect::>()?; + eprintln!("itunes: {} artists to look up", todo.len()); + let mut done = 0; + for (dr_id, name) in todo.into_iter().take(limit) { + if key(&name).is_empty() { continue; } + let v = match c.get(&format!("https://itunes.apple.com/search?term={}&entity=musicArtist&country=dk&limit=10", urlencode(&name))) { + Ok(v) => v, Err(e) => { log.push(format!("itunes search '{name}': {e}")); break; } + }; + let hit = v["results"].as_array().into_iter().flatten().find(|r| key(r["artistName"].as_str().unwrap_or_default()) == key(&name)); + let (iid, iname) = (hit.and_then(|h| h["artistId"].as_i64()), hit.and_then(|h| h["artistName"].as_str()).map(str::to_string)); + if let Some(id) = iid { + let have: i64 = db.conn.query_row("SELECT count(*) FROM itunes_release_lookups WHERE itunes_artist_id = ?1", [id], |r| r.get(0))?; + if have == 0 { + if let Err(e) = releases(&c, db, id) { log.push(format!("itunes releases {id}: {e}")); break; } + } + } + db.conn.execute("INSERT OR REPLACE INTO itunes_artist_map VALUES (?1, ?2, ?3, ?4)", params![dr_id, iid, iname, Utc::now().to_rfc3339()])?; + done += 1; + if done % 200 == 0 { eprintln!("itunes: {done} artists"); } + } + let stale: Vec = db.conn.prepare( + "SELECT DISTINCT l.itunes_artist_id FROM itunes_release_lookups l JOIN itunes_artist_map m ON m.itunes_artist_id = l.itunes_artist_id + JOIN dr_artists a ON a.artist_id = m.dr_artist_id WHERE l.fetched_at < datetime('now', '-7 days') ORDER BY l.fetched_at")? + .query_map([], |r| r.get(0))?.collect::>()?; + for id in stale.into_iter().take(refresh) { + if let Err(e) = releases(&c, db, id) { log.push(format!("itunes releases {id}: {e}")); break; } + } + Ok(done) +} + +fn releases(c: &Client, db: &Db, id: i64) -> Result { + let v = c.get(&format!("https://itunes.apple.com/lookup?id={id}&entity=album&limit=200&country=dk&sort=recent"))?; + let tx = db.conn.unchecked_transaction()?; + let mut n = 0; + for r in v["results"].as_array().into_iter().flatten() { + // The artist's own releases (not compilations they appear on). + if r["wrapperType"].as_str() != Some("collection") || r["artistId"].as_i64() != Some(id) { continue; } + let name = r["collectionName"].as_str().unwrap_or_default(); + let kind = if name.ends_with(" - Single") { "single" } else if name.ends_with(" - EP") { "ep" } else { "album" }; + let date = r["releaseDate"].as_str().map(|d| d.chars().take(10).collect::()); + tx.execute("INSERT OR REPLACE INTO itunes_releases VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + params![id, r["collectionId"].as_i64(), name, kind, date, r["trackCount"].as_i64()])?; + n += 1; + } + tx.execute("INSERT OR REPLACE INTO itunes_release_lookups VALUES (?1, ?2, ?3)", params![id, n as i64, Utc::now().to_rfc3339()])?; + tx.commit()?; + Ok(n) +} diff --git a/src/main.rs b/src/main.rs index e826d07..80e25a5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,6 @@ //! airplay: collects what Danish radio stations play, for modelling when artists get played again. //! -//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|spotify-api|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive|oldpages] [--backfill] +//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|spotify-api|itunes|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive|oldpages] [--backfill] //! airplay export nmf-denmark [--out data/nmf_denmark.csv] //! airplay stats [--db data/airplay.db] //! airplay check [--db data/airplay.db] gaps, duplicates, DR cross-check, chart completeness @@ -27,6 +27,7 @@ mod playlists; mod rotations; mod spotify; mod spotify_api; +mod itunes; mod web; use anyhow::Result; @@ -63,7 +64,7 @@ fn main() -> Result<()> { rotations::store_all(&db, &dir, args.iter().any(|a| a == "--reparse")) } _ => { - eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|spotify-api|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive|oldpages] [--backfill] + eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|spotify-api|itunes|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive|oldpages] [--backfill] //! airplay export nmf-denmark [--out data/nmf_denmark.csv]\n airplay stats [--db FILE]\n airplay check [--db FILE]\n airplay model panel|backtest|rank\n airplay serve [--addr HOST:PORT] [--prefix /airplay]"); std::process::exit(2); } @@ -140,6 +141,12 @@ fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> { let r = spotify_api::collect(db, lim, if backfill { 0 } else { 60 }, &mut log); step("spotify releases (web api)", r.map(|n| format!("{n} artists looked up")), &mut log); } + if wants("itunes") { + // Apple's iTunes Search API: artists' singles, EPs and albums with dates (~20 requests a minute). + let lim = arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 250 }); + let r = itunes::collect(db, lim, if backfill { 0 } else { 300 }, &mut log); + step("itunes releases", r.map(|n| format!("{n} artists looked up")), &mut log); + } if wants("playlists") { let r = playlists::snapshot(db, &http, &mut log); step("spotify playlists (New Music Friday DK, Top 50 DK)", r.map(|n| format!("{n} changed and saved")), &mut log); diff --git a/src/model/panel.rs b/src/model/panel.rs index 6a8a075..b106a23 100644 --- a/src/model/panel.rs +++ b/src/model/panel.rs @@ -304,7 +304,36 @@ pub(crate) fn load_releases(db: &Db, id_of: &HashMap, n: usize) - if let Some(v) = out[a].as_mut() { v.push(d); } } } - for v in out.iter_mut().flatten() { v.sort(); v.dedup(); } + // iTunes (singles, EPs, albums) and Spotify (own releases, singles they're featured on). + let has = |t: &str| -> Result { Ok(db.conn.query_row("SELECT count(*) FROM sqlite_master WHERE name = ?1", [t], |r| r.get::<_, i64>(0))? > 0) }; + let mut extra = |q: &str, matched: &str| -> Result<()> { + for r in db.conn.prepare(matched)?.query_map([], |r| r.get::<_, String>(0))? { + if let Some(&a) = id_of.get(&r?) { out[a].get_or_insert_with(Vec::new); } + } + for r in db.conn.prepare(q)?.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? { + let (id, d) = r?; + if let (Some(&a), Ok(d)) = (id_of.get(&id), NaiveDate::parse_from_str(&d[..d.len().min(10)], "%Y-%m-%d")) { + if let Some(v) = out[a].as_mut() { v.push(d); } + } + } + Ok(()) + }; + if has("itunes_releases")? { + extra("SELECT m.dr_artist_id, r.release_date FROM itunes_artist_map m JOIN itunes_releases r ON r.itunes_artist_id = m.itunes_artist_id WHERE r.release_date > '1900'", + "SELECT m.dr_artist_id FROM itunes_artist_map m JOIN itunes_release_lookups l ON l.itunes_artist_id = m.itunes_artist_id")?; + } + if has("spotify_releases")? { + // Day-precision dates only ("2019" or "2019-05" would land on the 1st). + extra("SELECT m.dr_artist_id, r.release_date FROM spotify_artist_map m JOIN spotify_releases r ON r.spotify_artist_id = m.spotify_artist_id WHERE r.date_precision = 'day'", + "SELECT m.dr_artist_id FROM spotify_artist_map m JOIN spotify_release_lookups l ON l.spotify_artist_id = m.spotify_artist_id")?; + } + // The same release from several sources is a few days apart at most: keep the earliest. + for v in out.iter_mut().flatten() { + v.sort(); + let mut kept: Vec = Vec::with_capacity(v.len()); + for d in v.drain(..) { if kept.last().is_none_or(|l| (d - *l).num_days() > 3) { kept.push(d); } } + *v = kept; + } Ok(out) }