iTunes Search API collector; model release dates merge Deezer, iTunes and Spotify

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-17 13:57:15 +00:00
co-authored by Claude Opus 5.5
parent 045be4a502
commit 46fbec76af
3 changed files with 163 additions and 3 deletions
+124
View File
@@ -0,0 +1,124 @@
//! Release dates from Apple's iTunes Search API (free, no key; Danish storefront). For every DR
//! artist, today's pool first and then the archive-only artists (2019–24 pages):
//!
//! GET https://itunes.apple.com/search?term={name}&entity=musicArtist&country=dk&limit=10
//! accepted only on an exact name match
//! GET https://itunes.apple.com/lookup?id={artist}&entity=album&limit=200&country=dk&sort=recent
//! singles ("… - Single"), EPs ("… - EP") and albums, with release dates
//!
//! Apple asks for about 20 requests a minute; it answers 403/429 when it's too many.
use crate::db::Db;
use anyhow::{bail, Result};
use chrono::Utc;
use rusqlite::params;
use serde_json::Value;
use std::time::Duration;
pub fn create_tables(db: &Db) -> Result<()> {
db.conn.execute_batch(
"CREATE TABLE IF NOT EXISTS itunes_artist_map (
dr_artist_id TEXT PRIMARY KEY, itunes_artist_id INTEGER, -- NULL = no exact match
itunes_name TEXT, fetched_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS itunes_releases (
itunes_artist_id INTEGER NOT NULL, collection_id INTEGER NOT NULL, name TEXT,
kind TEXT, -- single | ep | album
release_date TEXT, track_count INTEGER,
PRIMARY KEY (itunes_artist_id, collection_id)
);
CREATE TABLE IF NOT EXISTS itunes_release_lookups (
itunes_artist_id INTEGER PRIMARY KEY, releases INTEGER, fetched_at TEXT NOT NULL
);",
)?;
Ok(())
}
struct Client { http: reqwest::blocking::Client, delay: Duration }
impl Client {
/// GET JSON, pausing between requests; on 403/429 waits a minute, then two, ... (gives up after 5).
fn get(&self, url: &str) -> Result<Value> {
let mut wait = 60;
for _ in 0..6 {
std::thread::sleep(self.delay);
match self.http.get(url).send() {
Ok(r) if r.status().is_success() => return Ok(r.json().unwrap_or(Value::Null)),
Ok(r) if matches!(r.status().as_u16(), 403 | 429 | 503) => { eprintln!("itunes: {} — waiting {wait} s", r.status()); }
Ok(r) => bail!("{} from {url}", r.status()),
Err(e) => { eprintln!("itunes: {e} — waiting {wait} s"); }
}
std::thread::sleep(Duration::from_secs(wait));
wait *= 2;
}
bail!("itunes: still refused after retries (rate limit)")
}
}
fn key(s: &str) -> String { s.to_lowercase().chars().filter(|c| c.is_alphanumeric()).collect() }
fn urlencode(s: &str) -> String {
s.bytes().map(|b| match b { b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => (b as char).to_string(), _ => format!("%{b:02X}") }).collect()
}
/// Match artists, read their releases; then re-read today's pool artists last read 7+ days ago.
pub fn collect(db: &Db, limit: usize, refresh: usize, log: &mut Vec<String>) -> Result<usize> {
create_tables(db)?;
let c = Client {
http: reqwest::blocking::Client::builder().user_agent("AirplayPredictor/0.1 (research)").timeout(Duration::from_secs(30)).build()?,
delay: Duration::from_millis(3100),
};
let todo: Vec<(String, String)> = db.conn.prepare(
"SELECT artist_id, name, 0 FROM dr_artists WHERE artist_id NOT IN (SELECT dr_artist_id FROM itunes_artist_map)
UNION ALL
SELECT artist_id, name, 1 FROM old_artist_meta
WHERE name IS NOT NULL AND artist_id NOT IN (SELECT artist_id FROM dr_artists) AND artist_id NOT IN (SELECT dr_artist_id FROM itunes_artist_map)
ORDER BY 3, 1")?
.query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?.collect::<Result<_, _>>()?;
eprintln!("itunes: {} artists to look up", todo.len());
let mut done = 0;
for (dr_id, name) in todo.into_iter().take(limit) {
if key(&name).is_empty() { continue; }
let v = match c.get(&format!("https://itunes.apple.com/search?term={}&entity=musicArtist&country=dk&limit=10", urlencode(&name))) {
Ok(v) => v, Err(e) => { log.push(format!("itunes search '{name}': {e}")); break; }
};
let hit = v["results"].as_array().into_iter().flatten().find(|r| key(r["artistName"].as_str().unwrap_or_default()) == key(&name));
let (iid, iname) = (hit.and_then(|h| h["artistId"].as_i64()), hit.and_then(|h| h["artistName"].as_str()).map(str::to_string));
if let Some(id) = iid {
let have: i64 = db.conn.query_row("SELECT count(*) FROM itunes_release_lookups WHERE itunes_artist_id = ?1", [id], |r| r.get(0))?;
if have == 0 {
if let Err(e) = releases(&c, db, id) { log.push(format!("itunes releases {id}: {e}")); break; }
}
}
db.conn.execute("INSERT OR REPLACE INTO itunes_artist_map VALUES (?1, ?2, ?3, ?4)", params![dr_id, iid, iname, Utc::now().to_rfc3339()])?;
done += 1;
if done % 200 == 0 { eprintln!("itunes: {done} artists"); }
}
let stale: Vec<i64> = db.conn.prepare(
"SELECT DISTINCT l.itunes_artist_id FROM itunes_release_lookups l JOIN itunes_artist_map m ON m.itunes_artist_id = l.itunes_artist_id
JOIN dr_artists a ON a.artist_id = m.dr_artist_id WHERE l.fetched_at < datetime('now', '-7 days') ORDER BY l.fetched_at")?
.query_map([], |r| r.get(0))?.collect::<Result<_, _>>()?;
for id in stale.into_iter().take(refresh) {
if let Err(e) = releases(&c, db, id) { log.push(format!("itunes releases {id}: {e}")); break; }
}
Ok(done)
}
fn releases(c: &Client, db: &Db, id: i64) -> Result<usize> {
let v = c.get(&format!("https://itunes.apple.com/lookup?id={id}&entity=album&limit=200&country=dk&sort=recent"))?;
let tx = db.conn.unchecked_transaction()?;
let mut n = 0;
for r in v["results"].as_array().into_iter().flatten() {
// The artist's own releases (not compilations they appear on).
if r["wrapperType"].as_str() != Some("collection") || r["artistId"].as_i64() != Some(id) { continue; }
let name = r["collectionName"].as_str().unwrap_or_default();
let kind = if name.ends_with(" - Single") { "single" } else if name.ends_with(" - EP") { "ep" } else { "album" };
let date = r["releaseDate"].as_str().map(|d| d.chars().take(10).collect::<String>());
tx.execute("INSERT OR REPLACE INTO itunes_releases VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
params![id, r["collectionId"].as_i64(), name, kind, date, r["trackCount"].as_i64()])?;
n += 1;
}
tx.execute("INSERT OR REPLACE INTO itunes_release_lookups VALUES (?1, ?2, ?3)", params![id, n as i64, Utc::now().to_rfc3339()])?;
tx.commit()?;
Ok(n)
}
+9 -2
View File
@@ -1,6 +1,6 @@
//! airplay: collects what Danish radio stations play, for modelling when artists get played again. //! airplay: collects what Danish radio stations play, for modelling when artists get played again.
//! //!
//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|spotify-api|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive|oldpages] [--backfill] //! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|spotify-api|itunes|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive|oldpages] [--backfill]
//! airplay export nmf-denmark [--out data/nmf_denmark.csv] //! airplay export nmf-denmark [--out data/nmf_denmark.csv]
//! airplay stats [--db data/airplay.db] //! airplay stats [--db data/airplay.db]
//! airplay check [--db data/airplay.db] gaps, duplicates, DR cross-check, chart completeness //! airplay check [--db data/airplay.db] gaps, duplicates, DR cross-check, chart completeness
@@ -27,6 +27,7 @@ mod playlists;
mod rotations; mod rotations;
mod spotify; mod spotify;
mod spotify_api; mod spotify_api;
mod itunes;
mod web; mod web;
use anyhow::Result; use anyhow::Result;
@@ -63,7 +64,7 @@ fn main() -> Result<()> {
rotations::store_all(&db, &dir, args.iter().any(|a| a == "--reparse")) rotations::store_all(&db, &dir, args.iter().any(|a| a == "--reparse"))
} }
_ => { _ => {
eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|spotify-api|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive|oldpages] [--backfill] eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|spotify-api|itunes|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive|oldpages] [--backfill]
//! airplay export nmf-denmark [--out data/nmf_denmark.csv]\n airplay stats [--db FILE]\n airplay check [--db FILE]\n airplay model panel|backtest|rank\n airplay serve [--addr HOST:PORT] [--prefix /airplay]"); //! airplay export nmf-denmark [--out data/nmf_denmark.csv]\n airplay stats [--db FILE]\n airplay check [--db FILE]\n airplay model panel|backtest|rank\n airplay serve [--addr HOST:PORT] [--prefix /airplay]");
std::process::exit(2); std::process::exit(2);
} }
@@ -140,6 +141,12 @@ fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> {
let r = spotify_api::collect(db, lim, if backfill { 0 } else { 60 }, &mut log); let r = spotify_api::collect(db, lim, if backfill { 0 } else { 60 }, &mut log);
step("spotify releases (web api)", r.map(|n| format!("{n} artists looked up")), &mut log); step("spotify releases (web api)", r.map(|n| format!("{n} artists looked up")), &mut log);
} }
if wants("itunes") {
// Apple's iTunes Search API: artists' singles, EPs and albums with dates (~20 requests a minute).
let lim = arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 250 });
let r = itunes::collect(db, lim, if backfill { 0 } else { 300 }, &mut log);
step("itunes releases", r.map(|n| format!("{n} artists looked up")), &mut log);
}
if wants("playlists") { if wants("playlists") {
let r = playlists::snapshot(db, &http, &mut log); let r = playlists::snapshot(db, &http, &mut log);
step("spotify playlists (New Music Friday DK, Top 50 DK)", r.map(|n| format!("{n} changed and saved")), &mut log); step("spotify playlists (New Music Friday DK, Top 50 DK)", r.map(|n| format!("{n} changed and saved")), &mut log);
+30 -1
View File
@@ -304,7 +304,36 @@ pub(crate) fn load_releases(db: &Db, id_of: &HashMap<String, usize>, n: usize) -
if let Some(v) = out[a].as_mut() { v.push(d); } if let Some(v) = out[a].as_mut() { v.push(d); }
} }
} }
for v in out.iter_mut().flatten() { v.sort(); v.dedup(); } // iTunes (singles, EPs, albums) and Spotify (own releases, singles they're featured on).
let has = |t: &str| -> Result<bool> { Ok(db.conn.query_row("SELECT count(*) FROM sqlite_master WHERE name = ?1", [t], |r| r.get::<_, i64>(0))? > 0) };
let mut extra = |q: &str, matched: &str| -> Result<()> {
for r in db.conn.prepare(matched)?.query_map([], |r| r.get::<_, String>(0))? {
if let Some(&a) = id_of.get(&r?) { out[a].get_or_insert_with(Vec::new); }
}
for r in db.conn.prepare(q)?.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? {
let (id, d) = r?;
if let (Some(&a), Ok(d)) = (id_of.get(&id), NaiveDate::parse_from_str(&d[..d.len().min(10)], "%Y-%m-%d")) {
if let Some(v) = out[a].as_mut() { v.push(d); }
}
}
Ok(())
};
if has("itunes_releases")? {
extra("SELECT m.dr_artist_id, r.release_date FROM itunes_artist_map m JOIN itunes_releases r ON r.itunes_artist_id = m.itunes_artist_id WHERE r.release_date > '1900'",
"SELECT m.dr_artist_id FROM itunes_artist_map m JOIN itunes_release_lookups l ON l.itunes_artist_id = m.itunes_artist_id")?;
}
if has("spotify_releases")? {
// Day-precision dates only ("2019" or "2019-05" would land on the 1st).
extra("SELECT m.dr_artist_id, r.release_date FROM spotify_artist_map m JOIN spotify_releases r ON r.spotify_artist_id = m.spotify_artist_id WHERE r.date_precision = 'day'",
"SELECT m.dr_artist_id FROM spotify_artist_map m JOIN spotify_release_lookups l ON l.spotify_artist_id = m.spotify_artist_id")?;
}
// The same release from several sources is a few days apart at most: keep the earliest.
for v in out.iter_mut().flatten() {
v.sort();
let mut kept: Vec<NaiveDate> = Vec::with_capacity(v.len());
for d in v.drain(..) { if kept.last().is_none_or(|l| (d - *l).num_days() > 3) { kept.push(d); } }
*v = kept;
}
Ok(out) Ok(out)
} }