iTunes Search API collector; model release dates merge Deezer, iTunes and Spotify
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+124
@@ -0,0 +1,124 @@
|
|||||||
|
//! Release dates from Apple's iTunes Search API (free, no key; Danish storefront). For every DR
|
||||||
|
//! artist, today's pool first and then the archive-only artists (2019–24 pages):
|
||||||
|
//!
|
||||||
|
//! GET https://itunes.apple.com/search?term={name}&entity=musicArtist&country=dk&limit=10
|
||||||
|
//! accepted only on an exact name match
|
||||||
|
//! GET https://itunes.apple.com/lookup?id={artist}&entity=album&limit=200&country=dk&sort=recent
|
||||||
|
//! singles ("… - Single"), EPs ("… - EP") and albums, with release dates
|
||||||
|
//!
|
||||||
|
//! Apple asks for about 20 requests a minute; it answers 403/429 when it's too many.
|
||||||
|
|
||||||
|
use crate::db::Db;
|
||||||
|
use anyhow::{bail, Result};
|
||||||
|
use chrono::Utc;
|
||||||
|
use rusqlite::params;
|
||||||
|
use serde_json::Value;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
pub fn create_tables(db: &Db) -> Result<()> {
|
||||||
|
db.conn.execute_batch(
|
||||||
|
"CREATE TABLE IF NOT EXISTS itunes_artist_map (
|
||||||
|
dr_artist_id TEXT PRIMARY KEY, itunes_artist_id INTEGER, -- NULL = no exact match
|
||||||
|
itunes_name TEXT, fetched_at TEXT NOT NULL
|
||||||
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS itunes_releases (
|
||||||
|
itunes_artist_id INTEGER NOT NULL, collection_id INTEGER NOT NULL, name TEXT,
|
||||||
|
kind TEXT, -- single | ep | album
|
||||||
|
release_date TEXT, track_count INTEGER,
|
||||||
|
PRIMARY KEY (itunes_artist_id, collection_id)
|
||||||
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS itunes_release_lookups (
|
||||||
|
itunes_artist_id INTEGER PRIMARY KEY, releases INTEGER, fetched_at TEXT NOT NULL
|
||||||
|
);",
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Client { http: reqwest::blocking::Client, delay: Duration }
|
||||||
|
|
||||||
|
impl Client {
|
||||||
|
/// GET JSON, pausing between requests; on 403/429 waits a minute, then two, ... (gives up after 5).
|
||||||
|
fn get(&self, url: &str) -> Result<Value> {
|
||||||
|
let mut wait = 60;
|
||||||
|
for _ in 0..6 {
|
||||||
|
std::thread::sleep(self.delay);
|
||||||
|
match self.http.get(url).send() {
|
||||||
|
Ok(r) if r.status().is_success() => return Ok(r.json().unwrap_or(Value::Null)),
|
||||||
|
Ok(r) if matches!(r.status().as_u16(), 403 | 429 | 503) => { eprintln!("itunes: {} — waiting {wait} s", r.status()); }
|
||||||
|
Ok(r) => bail!("{} from {url}", r.status()),
|
||||||
|
Err(e) => { eprintln!("itunes: {e} — waiting {wait} s"); }
|
||||||
|
}
|
||||||
|
std::thread::sleep(Duration::from_secs(wait));
|
||||||
|
wait *= 2;
|
||||||
|
}
|
||||||
|
bail!("itunes: still refused after retries (rate limit)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn key(s: &str) -> String { s.to_lowercase().chars().filter(|c| c.is_alphanumeric()).collect() }
|
||||||
|
|
||||||
|
fn urlencode(s: &str) -> String {
|
||||||
|
s.bytes().map(|b| match b { b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => (b as char).to_string(), _ => format!("%{b:02X}") }).collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Match artists, read their releases; then re-read today's pool artists last read 7+ days ago.
|
||||||
|
pub fn collect(db: &Db, limit: usize, refresh: usize, log: &mut Vec<String>) -> Result<usize> {
|
||||||
|
create_tables(db)?;
|
||||||
|
let c = Client {
|
||||||
|
http: reqwest::blocking::Client::builder().user_agent("AirplayPredictor/0.1 (research)").timeout(Duration::from_secs(30)).build()?,
|
||||||
|
delay: Duration::from_millis(3100),
|
||||||
|
};
|
||||||
|
let todo: Vec<(String, String)> = db.conn.prepare(
|
||||||
|
"SELECT artist_id, name, 0 FROM dr_artists WHERE artist_id NOT IN (SELECT dr_artist_id FROM itunes_artist_map)
|
||||||
|
UNION ALL
|
||||||
|
SELECT artist_id, name, 1 FROM old_artist_meta
|
||||||
|
WHERE name IS NOT NULL AND artist_id NOT IN (SELECT artist_id FROM dr_artists) AND artist_id NOT IN (SELECT dr_artist_id FROM itunes_artist_map)
|
||||||
|
ORDER BY 3, 1")?
|
||||||
|
.query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?.collect::<Result<_, _>>()?;
|
||||||
|
eprintln!("itunes: {} artists to look up", todo.len());
|
||||||
|
let mut done = 0;
|
||||||
|
for (dr_id, name) in todo.into_iter().take(limit) {
|
||||||
|
if key(&name).is_empty() { continue; }
|
||||||
|
let v = match c.get(&format!("https://itunes.apple.com/search?term={}&entity=musicArtist&country=dk&limit=10", urlencode(&name))) {
|
||||||
|
Ok(v) => v, Err(e) => { log.push(format!("itunes search '{name}': {e}")); break; }
|
||||||
|
};
|
||||||
|
let hit = v["results"].as_array().into_iter().flatten().find(|r| key(r["artistName"].as_str().unwrap_or_default()) == key(&name));
|
||||||
|
let (iid, iname) = (hit.and_then(|h| h["artistId"].as_i64()), hit.and_then(|h| h["artistName"].as_str()).map(str::to_string));
|
||||||
|
if let Some(id) = iid {
|
||||||
|
let have: i64 = db.conn.query_row("SELECT count(*) FROM itunes_release_lookups WHERE itunes_artist_id = ?1", [id], |r| r.get(0))?;
|
||||||
|
if have == 0 {
|
||||||
|
if let Err(e) = releases(&c, db, id) { log.push(format!("itunes releases {id}: {e}")); break; }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
db.conn.execute("INSERT OR REPLACE INTO itunes_artist_map VALUES (?1, ?2, ?3, ?4)", params![dr_id, iid, iname, Utc::now().to_rfc3339()])?;
|
||||||
|
done += 1;
|
||||||
|
if done % 200 == 0 { eprintln!("itunes: {done} artists"); }
|
||||||
|
}
|
||||||
|
let stale: Vec<i64> = db.conn.prepare(
|
||||||
|
"SELECT DISTINCT l.itunes_artist_id FROM itunes_release_lookups l JOIN itunes_artist_map m ON m.itunes_artist_id = l.itunes_artist_id
|
||||||
|
JOIN dr_artists a ON a.artist_id = m.dr_artist_id WHERE l.fetched_at < datetime('now', '-7 days') ORDER BY l.fetched_at")?
|
||||||
|
.query_map([], |r| r.get(0))?.collect::<Result<_, _>>()?;
|
||||||
|
for id in stale.into_iter().take(refresh) {
|
||||||
|
if let Err(e) = releases(&c, db, id) { log.push(format!("itunes releases {id}: {e}")); break; }
|
||||||
|
}
|
||||||
|
Ok(done)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn releases(c: &Client, db: &Db, id: i64) -> Result<usize> {
|
||||||
|
let v = c.get(&format!("https://itunes.apple.com/lookup?id={id}&entity=album&limit=200&country=dk&sort=recent"))?;
|
||||||
|
let tx = db.conn.unchecked_transaction()?;
|
||||||
|
let mut n = 0;
|
||||||
|
for r in v["results"].as_array().into_iter().flatten() {
|
||||||
|
// The artist's own releases (not compilations they appear on).
|
||||||
|
if r["wrapperType"].as_str() != Some("collection") || r["artistId"].as_i64() != Some(id) { continue; }
|
||||||
|
let name = r["collectionName"].as_str().unwrap_or_default();
|
||||||
|
let kind = if name.ends_with(" - Single") { "single" } else if name.ends_with(" - EP") { "ep" } else { "album" };
|
||||||
|
let date = r["releaseDate"].as_str().map(|d| d.chars().take(10).collect::<String>());
|
||||||
|
tx.execute("INSERT OR REPLACE INTO itunes_releases VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
|
||||||
|
params![id, r["collectionId"].as_i64(), name, kind, date, r["trackCount"].as_i64()])?;
|
||||||
|
n += 1;
|
||||||
|
}
|
||||||
|
tx.execute("INSERT OR REPLACE INTO itunes_release_lookups VALUES (?1, ?2, ?3)", params![id, n as i64, Utc::now().to_rfc3339()])?;
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(n)
|
||||||
|
}
|
||||||
+9
-2
@@ -1,6 +1,6 @@
|
|||||||
//! airplay: collects what Danish radio stations play, for modelling when artists get played again.
|
//! airplay: collects what Danish radio stations play, for modelling when artists get played again.
|
||||||
//!
|
//!
|
||||||
//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|spotify-api|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive|oldpages] [--backfill]
|
//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|spotify-api|itunes|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive|oldpages] [--backfill]
|
||||||
//! airplay export nmf-denmark [--out data/nmf_denmark.csv]
|
//! airplay export nmf-denmark [--out data/nmf_denmark.csv]
|
||||||
//! airplay stats [--db data/airplay.db]
|
//! airplay stats [--db data/airplay.db]
|
||||||
//! airplay check [--db data/airplay.db] gaps, duplicates, DR cross-check, chart completeness
|
//! airplay check [--db data/airplay.db] gaps, duplicates, DR cross-check, chart completeness
|
||||||
@@ -27,6 +27,7 @@ mod playlists;
|
|||||||
mod rotations;
|
mod rotations;
|
||||||
mod spotify;
|
mod spotify;
|
||||||
mod spotify_api;
|
mod spotify_api;
|
||||||
|
mod itunes;
|
||||||
mod web;
|
mod web;
|
||||||
|
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
@@ -63,7 +64,7 @@ fn main() -> Result<()> {
|
|||||||
rotations::store_all(&db, &dir, args.iter().any(|a| a == "--reparse"))
|
rotations::store_all(&db, &dir, args.iter().any(|a| a == "--reparse"))
|
||||||
}
|
}
|
||||||
_ => {
|
_ => {
|
||||||
eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|spotify-api|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive|oldpages] [--backfill]
|
eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|spotify-api|itunes|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive|oldpages] [--backfill]
|
||||||
//! airplay export nmf-denmark [--out data/nmf_denmark.csv]\n airplay stats [--db FILE]\n airplay check [--db FILE]\n airplay model panel|backtest|rank\n airplay serve [--addr HOST:PORT] [--prefix /airplay]");
|
//! airplay export nmf-denmark [--out data/nmf_denmark.csv]\n airplay stats [--db FILE]\n airplay check [--db FILE]\n airplay model panel|backtest|rank\n airplay serve [--addr HOST:PORT] [--prefix /airplay]");
|
||||||
std::process::exit(2);
|
std::process::exit(2);
|
||||||
}
|
}
|
||||||
@@ -140,6 +141,12 @@ fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> {
|
|||||||
let r = spotify_api::collect(db, lim, if backfill { 0 } else { 60 }, &mut log);
|
let r = spotify_api::collect(db, lim, if backfill { 0 } else { 60 }, &mut log);
|
||||||
step("spotify releases (web api)", r.map(|n| format!("{n} artists looked up")), &mut log);
|
step("spotify releases (web api)", r.map(|n| format!("{n} artists looked up")), &mut log);
|
||||||
}
|
}
|
||||||
|
if wants("itunes") {
|
||||||
|
// Apple's iTunes Search API: artists' singles, EPs and albums with dates (~20 requests a minute).
|
||||||
|
let lim = arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 250 });
|
||||||
|
let r = itunes::collect(db, lim, if backfill { 0 } else { 300 }, &mut log);
|
||||||
|
step("itunes releases", r.map(|n| format!("{n} artists looked up")), &mut log);
|
||||||
|
}
|
||||||
if wants("playlists") {
|
if wants("playlists") {
|
||||||
let r = playlists::snapshot(db, &http, &mut log);
|
let r = playlists::snapshot(db, &http, &mut log);
|
||||||
step("spotify playlists (New Music Friday DK, Top 50 DK)", r.map(|n| format!("{n} changed and saved")), &mut log);
|
step("spotify playlists (New Music Friday DK, Top 50 DK)", r.map(|n| format!("{n} changed and saved")), &mut log);
|
||||||
|
|||||||
+30
-1
@@ -304,7 +304,36 @@ pub(crate) fn load_releases(db: &Db, id_of: &HashMap<String, usize>, n: usize) -
|
|||||||
if let Some(v) = out[a].as_mut() { v.push(d); }
|
if let Some(v) = out[a].as_mut() { v.push(d); }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for v in out.iter_mut().flatten() { v.sort(); v.dedup(); }
|
// iTunes (singles, EPs, albums) and Spotify (own releases, singles they're featured on).
|
||||||
|
let has = |t: &str| -> Result<bool> { Ok(db.conn.query_row("SELECT count(*) FROM sqlite_master WHERE name = ?1", [t], |r| r.get::<_, i64>(0))? > 0) };
|
||||||
|
let mut extra = |q: &str, matched: &str| -> Result<()> {
|
||||||
|
for r in db.conn.prepare(matched)?.query_map([], |r| r.get::<_, String>(0))? {
|
||||||
|
if let Some(&a) = id_of.get(&r?) { out[a].get_or_insert_with(Vec::new); }
|
||||||
|
}
|
||||||
|
for r in db.conn.prepare(q)?.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? {
|
||||||
|
let (id, d) = r?;
|
||||||
|
if let (Some(&a), Ok(d)) = (id_of.get(&id), NaiveDate::parse_from_str(&d[..d.len().min(10)], "%Y-%m-%d")) {
|
||||||
|
if let Some(v) = out[a].as_mut() { v.push(d); }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
};
|
||||||
|
if has("itunes_releases")? {
|
||||||
|
extra("SELECT m.dr_artist_id, r.release_date FROM itunes_artist_map m JOIN itunes_releases r ON r.itunes_artist_id = m.itunes_artist_id WHERE r.release_date > '1900'",
|
||||||
|
"SELECT m.dr_artist_id FROM itunes_artist_map m JOIN itunes_release_lookups l ON l.itunes_artist_id = m.itunes_artist_id")?;
|
||||||
|
}
|
||||||
|
if has("spotify_releases")? {
|
||||||
|
// Day-precision dates only ("2019" or "2019-05" would land on the 1st).
|
||||||
|
extra("SELECT m.dr_artist_id, r.release_date FROM spotify_artist_map m JOIN spotify_releases r ON r.spotify_artist_id = m.spotify_artist_id WHERE r.date_precision = 'day'",
|
||||||
|
"SELECT m.dr_artist_id FROM spotify_artist_map m JOIN spotify_release_lookups l ON l.spotify_artist_id = m.spotify_artist_id")?;
|
||||||
|
}
|
||||||
|
// The same release from several sources is a few days apart at most: keep the earliest.
|
||||||
|
for v in out.iter_mut().flatten() {
|
||||||
|
v.sort();
|
||||||
|
let mut kept: Vec<NaiveDate> = Vec::with_capacity(v.len());
|
||||||
|
for d in v.drain(..) { if kept.last().is_none_or(|l| (d - *l).num_days() > 3) { kept.push(d); } }
|
||||||
|
*v = kept;
|
||||||
|
}
|
||||||
Ok(out)
|
Ok(out)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user