diff --git a/src/check.rs b/src/check.rs new file mode 100644 index 0000000..6144cab --- /dev/null +++ b/src/check.rs @@ -0,0 +1,127 @@ +//! `airplay check`: how complete and consistent the collected data is. +//! +//! - gaps: hours in which a station has no plays (a failed fetch, or the station off air) +//! - cut short: plays followed by the next song before half of their length (snippets, teasers, +//! DJ segues; checked by hand on P3: different songs, not duplicates) +//! - DR cross-check: plays we counted per artist, channel and week against the weekly totals on +//! DR's artist pages (two independent DR sources), for weeks we cover completely +//! - Hitlisten: weeks missing, and weeks without exactly 20 rows + +use crate::{db::Db, hitlisten}; +use anyhow::Result; +use chrono::{DateTime, Datelike, Duration, NaiveDate, Utc}; +use chrono_tz::Europe::Copenhagen; +use std::collections::{BTreeMap, HashMap}; + +fn parse(t: &str) -> Option> { + DateTime::parse_from_rfc3339(t).ok().map(|d| d.with_timezone(&Utc)) +} + +pub fn run(db: &Db) -> Result<()> { + gaps_and_overlaps(db)?; + dr_cross_check(db)?; + hitlisten_check(db)?; + Ok(()) +} + +fn gaps_and_overlaps(db: &Db) -> Result<()> { + println!("== Gaps and short plays per station (gap = an hour with no plays; cut short = next song started before half of this one)"); + println!("{:<12} {:>7} {:>10} {:>9} longest gap", "station", "plays", "empty hrs", "cut short"); + let stations: Vec = db.conn.prepare("SELECT DISTINCT station FROM plays ORDER BY station")? + .query_map([], |r| r.get(0))?.collect::>()?; + for station in stations { + let mut st = db.conn.prepare("SELECT played_at, coalesce(duration_ms, 0) FROM plays WHERE station = ?1 ORDER BY played_at")?; + let rows: Vec<(DateTime, i64)> = st.query_map([&station], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?)))? + .filter_map(|r| r.ok()).filter_map(|(t, d)| Some((parse(&t)?, d))).collect(); + if rows.len() < 2 { continue; } + let mut empty_hours = 0; + let mut longest = (Duration::zero(), rows[0].0); + let mut overlaps = 0; + for w in rows.windows(2) { + let gap = w[1].0 - w[0].0; + if gap > Duration::hours(1) { empty_hours += (gap.num_minutes() / 60) as i64; } + if gap > longest.0 { longest = (gap, w[0].0); } + // The next song started before half of this one had played. + if w[0].1 > 0 && gap < Duration::milliseconds(w[0].1 / 2) { overlaps += 1; } + } + println!("{:<12} {:>7} {:>10} {:>9} {} min after {}", station, rows.len(), empty_hours, overlaps, + longest.0.num_minutes(), longest.1.with_timezone(&Copenhagen).format("%Y-%m-%d %H:%M")); + } + Ok(()) +} + +fn monday(d: NaiveDate) -> NaiveDate { + d - Duration::days(d.weekday().num_days_from_monday() as i64) +} + +fn dr_cross_check(db: &Db) -> Result<()> { + println!("\n== DR: our plays vs the weekly totals on DR's artist pages"); + // Weeks (Danish time, Monday-Sunday) every DR channel covers from start to end. + let (first, last): (Option, Option) = db.conn.query_row( + "SELECT max(f), min(l) FROM (SELECT min(played_at) f, max(played_at) l FROM plays WHERE source = 'dr' GROUP BY station)", + [], |r| Ok((r.get(0)?, r.get(1)?)))?; + let (Some(first), Some(last)) = (first.as_deref().and_then(parse), last.as_deref().and_then(parse)) else { + println!("no DR plays yet"); + return Ok(()); + }; + let start = monday(first.with_timezone(&Copenhagen).date_naive()) + Duration::days(7); + let end = monday(last.with_timezone(&Copenhagen).date_naive()); // weeks before this are complete + if start >= end { + println!("no complete Monday-Sunday week collected yet (data from {} to {}); available after {}", + first.with_timezone(&Copenhagen).format("%Y-%m-%d"), last.with_timezone(&Copenhagen).format("%Y-%m-%d"), + start + Duration::days(7)); + return Ok(()); + } + let mut ours: HashMap<(String, String, NaiveDate), i64> = HashMap::new(); + let mut st = db.conn.prepare("SELECT station, played_at, artist_ids FROM plays WHERE source = 'dr'")?; + for r in st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?)))? { + let (station, t, ids) = r?; + let Some(t) = parse(&t) else { continue }; + let week = monday(t.with_timezone(&Copenhagen).date_naive()); + if week < start || week >= end { continue; } + for id in ids.split(',').filter(|s| !s.is_empty()) { + *ours.entry((id.to_string(), station.clone(), week)).or_default() += 1; + } + } + let mut theirs = db.conn.prepare("SELECT artist_id, channel, week_start, plays FROM dr_artist_weeks WHERE week_start >= ?1 AND week_start < ?2")?; + let mut compared = 0; + let mut exact = 0; + let mut diffs: BTreeMap = BTreeMap::new(); + for r in theirs.query_map([start.to_string(), end.to_string()], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, i64>(3)?)))? { + let (id, channel, week, plays) = r?; + let Ok(week) = NaiveDate::parse_from_str(&week, "%Y-%m-%d") else { continue }; + let Some(&n) = ours.get(&(id, channel, week)) else { continue }; + compared += 1; + if n == plays { exact += 1; } + *diffs.entry((n - plays).clamp(-5, 5)).or_default() += 1; + } + println!("weeks {start} to {}: {compared} artist-channel-weeks compared, {exact} identical ({:.1}%)", + end - Duration::days(1), if compared > 0 { 100.0 * exact as f64 / compared as f64 } else { 0.0 }); + println!("difference (ours - DR's), clamped to ±5: {diffs:?}"); + Ok(()) +} + +fn hitlisten_check(db: &Db) -> Result<()> { + println!("\n== Hitlisten Airplay Top-20"); + hitlisten::create_tables(db)?; + let weeks: Vec<(i64, i64, i64)> = db.conn.prepare("SELECT year, week, count(*) FROM hitlisten_airplay GROUP BY year, week ORDER BY year, week")? + .query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?.collect::>()?; + if weeks.is_empty() { println!("nothing collected"); return Ok(()); } + let odd: Vec = weeks.iter().filter(|w| w.2 != 20).map(|w| format!("{}-W{:02} ({} rows)", w.0, w.1, w.2)).collect(); + // Missing weeks between the first and the last (ISO weeks; some years have 53). + let have: std::collections::HashSet<(i64, i64)> = weeks.iter().map(|w| (w.0, w.1)).collect(); + let mut missing = Vec::new(); + let (fy, fw, ly, lw) = (weeks[0].0, weeks[0].1, weeks.last().unwrap().0, weeks.last().unwrap().1); + for y in fy..=ly { + let last_week = NaiveDate::from_ymd_opt(y as i32, 12, 28).map(|d| d.iso_week().week() as i64).unwrap_or(52); + for w in 1..=last_week { + if (y == fy && w < fw) || (y == ly && w > lw) { continue; } + if !have.contains(&(y, w)) { missing.push(format!("{y}-W{w:02}")); } + } + } + println!("{} weeks from {fy}-W{fw:02} to {ly}-W{lw:02}; {} missing; {} without 20 rows", + weeks.len(), missing.len(), odd.len()); + if !missing.is_empty() { println!("missing: {}", missing.iter().take(40).cloned().collect::>().join(", ")); } + if !odd.is_empty() { println!("odd: {}", odd.iter().take(20).cloned().collect::>().join(", ")); } + Ok(()) +} diff --git a/src/db.rs b/src/db.rs index 57bc095..0702ecc 100644 --- a/src/db.rs +++ b/src/db.rs @@ -22,6 +22,8 @@ pub struct Play<'a> { impl Db { pub fn open(path: &str) -> Result { let conn = Connection::open(path)?; + // Several collectors may write at once (the timer, a backfill); wait instead of failing. + conn.busy_timeout(std::time::Duration::from_secs(120))?; conn.execute_batch( "PRAGMA journal_mode=WAL; CREATE TABLE IF NOT EXISTS plays ( diff --git a/src/deezer.rs b/src/deezer.rs new file mode 100644 index 0000000..5af17b6 --- /dev/null +++ b/src/deezer.rs @@ -0,0 +1,165 @@ +//! Song metadata from Deezer's public API (no key): ISRC, release date, label, release type, +//! genres, contributors (featured artists) and popularity, for every distinct artist + title in +//! the plays and the charts. +//! +//! GET https://api.deezer.com/search?q={artist} {title} up to 25 candidate tracks +//! GET https://api.deezer.com/track/{id} isrc, release_date, contributors[], rank +//! GET https://api.deezer.com/album/{id} label, upc, record_type, genres, release_date +//! +//! Matching is deliberately simple here: the raw search answers are kept (deezer_searches.raw), +//! so the cleaning step can pick matches again without fetching. Rate limit: 50 requests / 5 s. + +use crate::{db::Db, http::Http}; +use anyhow::Result; +use chrono::Utc; +use rusqlite::{params, OptionalExtension}; +use serde_json::Value; + +const BASE: &str = "https://api.deezer.com"; + +pub fn create_tables(db: &Db) -> Result<()> { + db.conn.execute_batch( + "CREATE TABLE IF NOT EXISTS deezer_searches ( + artist TEXT NOT NULL, title TEXT NOT NULL, -- exactly as the station or chart wrote them + query TEXT NOT NULL, results INTEGER NOT NULL, + best_track_id INTEGER, best_score REAL, -- the simple matcher's pick (redo in cleaning) + fetched_at TEXT NOT NULL, raw TEXT NOT NULL, + PRIMARY KEY (artist, title) + ); + CREATE TABLE IF NOT EXISTS deezer_tracks ( + track_id INTEGER PRIMARY KEY, isrc TEXT, title TEXT, artist TEXT, + contributors TEXT, -- JSON [{name, role}] + release_date TEXT, duration_s INTEGER, rank INTEGER, explicit INTEGER, + album_id INTEGER, fetched_at TEXT NOT NULL, raw TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS deezer_tracks_isrc ON deezer_tracks (isrc); + CREATE TABLE IF NOT EXISTS deezer_albums ( + album_id INTEGER PRIMARY KEY, title TEXT, upc TEXT, label TEXT, release_date TEXT, + record_type TEXT, genres TEXT, nb_tracks INTEGER, fetched_at TEXT NOT NULL, raw TEXT NOT NULL + );", + )?; + Ok(()) +} + +/// Lower-case letters and digits only, for comparing names. +fn key(s: &str) -> String { + s.to_lowercase().chars().filter(|c| c.is_alphanumeric()).collect() +} + +/// The first artist ("Shakira & Burna Boy", "X ft. Y", "A, B" -> the first name) and the title +/// without version tags, for the search query only; the stored names stay as written. +fn query_parts(artist: &str, title: &str) -> (String, String) { + let lower = artist.to_lowercase(); + let mut cut = artist.len(); + for sep in [" feat.", " feat ", " ft.", " ft ", " featuring ", " & ", ", ", " x ", " og ", " [+] ", " med "] { + if let Some(i) = lower.find(sep) { cut = cut.min(i); } + } + let first = artist[..cut].trim().to_string(); + let title = title.split(" (").next().unwrap_or(title).split(" - ").next().unwrap_or(title).trim().to_string(); + (first, title) +} + +/// 0..1: how well a candidate matches (title equal or contained, artist contained). +fn score(candidate: &Value, artist: &str, title: &str) -> f64 { + let (ct, ca) = (key(candidate["title"].as_str().unwrap_or_default()), key(candidate["artist"]["name"].as_str().unwrap_or_default())); + let (qt, qa) = (key(title), key(artist)); + let t = if ct == qt { 1.0 } else if !qt.is_empty() && (ct.contains(&qt) || qt.contains(&ct)) { 0.7 } else { 0.0 }; + let a = if ca == qa { 1.0 } else if !ca.is_empty() && (qa.contains(&ca) || ca.contains(&qa)) { 0.8 } else { 0.0 }; + 0.6 * t + 0.4 * a +} + +fn fetch_json(http: &Http, url: &str) -> Result> { + Ok(match http.get(url)? { + Some(b) => { + let v: Value = serde_json::from_str(&b)?; + if v.get("error").is_some() { anyhow::bail!("deezer: {}", v["error"]); } + Some(v) + } + None => None, + }) +} + +/// Looks up every artist + title not searched yet (`limit` per run). +pub fn collect(db: &Db, http: &Http, limit: usize, log: &mut Vec) -> Result { + create_tables(db)?; + crate::hitlisten::create_tables(db)?; + let todo: Vec<(String, String)> = db.conn.prepare( + "SELECT artist, title FROM (SELECT artist, title FROM plays UNION SELECT artist, title FROM hitlisten_airplay) s + WHERE NOT EXISTS (SELECT 1 FROM deezer_searches d WHERE d.artist = s.artist AND d.title = s.title) + AND artist != '' AND title != ''")? + .query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?.collect::>()?; + let now = Utc::now().to_rfc3339(); + let mut done = 0; + for (artist, title) in todo.into_iter().take(limit) { + let (qa, qt) = query_parts(&artist, &title); + let query = format!("{qa} {qt}"); + let url = format!("{BASE}/search?q={}&limit=10", urlencode(&query)); + let found = match fetch_json(http, &url) { + Ok(v) => v.unwrap_or(Value::Null), + Err(e) => { log.push(format!("deezer search '{query}': {e}")); continue; } + }; + let hits = found["data"].as_array().cloned().unwrap_or_default(); + let best = hits.iter().map(|h| (score(h, &qa, &qt), h)).max_by(|a, b| a.0.total_cmp(&b.0)); + let (best_id, best_score) = match best { + Some((s, h)) if s >= 0.6 => (h["id"].as_i64(), Some(s)), + Some((s, _)) => (None, Some(s)), + None => (None, None), + }; + if let Some(id) = best_id { + if let Err(e) = track(db, http, id, &now) { log.push(format!("deezer track {id}: {e}")); } + } + // Every other strong match is another release of the same song (album, compilation, + // reissue): store those releases too, so cleaning can take the earliest release date. + for h in hits.iter().filter(|h| score(h, &qa, &qt) >= 0.9).take(6) { + if let Some(album_id) = h["album"]["id"].as_i64() { + if let Err(e) = album(db, http, album_id, &now) { log.push(format!("deezer album {album_id}: {e}")); } + } + } + db.conn.execute("INSERT OR REPLACE INTO deezer_searches VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", + params![artist, title, query, hits.len() as i64, best_id, best_score, now, found.to_string()])?; + done += 1; + } + Ok(done) +} + +/// Stores a track and its album (each fetched once). +fn track(db: &Db, http: &Http, id: i64, now: &str) -> Result<()> { + let have: Option = db.conn.query_row("SELECT album_id FROM deezer_tracks WHERE track_id = ?1", [id], |r| r.get(0)).optional()?; + let album_id = match have { + Some(a) => a, + None => { + let Some(t) = fetch_json(http, &format!("{BASE}/track/{id}"))? else { return Ok(()) }; + let contributors: Vec = t["contributors"].as_array().cloned().unwrap_or_default().iter() + .map(|c| serde_json::json!({"name": c["name"], "role": c["role"], "id": c["id"]})).collect(); + db.conn.execute("INSERT OR REPLACE INTO deezer_tracks VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)", + params![id, t["isrc"].as_str(), t["title"].as_str(), t["artist"]["name"].as_str(), + Value::Array(contributors).to_string(), t["release_date"].as_str(), t["duration"].as_i64(), + t["rank"].as_i64(), t["explicit_lyrics"].as_bool().map(|b| b as i64), + t["album"]["id"].as_i64(), now, t.to_string()])?; + t["album"]["id"].as_i64().unwrap_or(0) + } + }; + if album_id != 0 { album(db, http, album_id, now)?; } + Ok(()) +} + +/// Stores a release once. +fn album(db: &Db, http: &Http, album_id: i64, now: &str) -> Result<()> { + let known: i64 = db.conn.query_row("SELECT count(*) FROM deezer_albums WHERE album_id = ?1", [album_id], |r| r.get(0))?; + if known > 0 { return Ok(()); } + if let Some(a) = fetch_json(http, &format!("{BASE}/album/{album_id}"))? { + let genres: Vec<&str> = a["genres"]["data"].as_array().map(|g| g.iter().filter_map(|x| x["name"].as_str()).collect()).unwrap_or_default(); + db.conn.execute("INSERT OR REPLACE INTO deezer_albums VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)", + params![album_id, a["title"].as_str(), a["upc"].as_str(), a["label"].as_str(), a["release_date"].as_str(), + a["record_type"].as_str(), genres.join(", "), a["nb_tracks"].as_i64(), now, a.to_string()])?; + } + Ok(()) +} + +fn urlencode(s: &str) -> String { + s.bytes().map(|b| match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => (b as char).to_string(), + b' ' => "%20".into(), + _ => format!("%{b:02X}"), + }).collect() +} diff --git a/src/main.rs b/src/main.rs index 3f0c17c..97f7e9b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,16 +1,20 @@ //! airplay: collects what Danish radio stations play, for modelling when artists get played again. //! -//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten] [--backfill] +//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|musicbrainz] [--backfill] //! airplay stats [--db data/airplay.db] +//! airplay check [--db data/airplay.db] gaps, duplicates, DR cross-check, chart completeness //! //! --backfill fetches all history at once (every DR artist page, the whole Hitlisten archive); //! without it a run takes a bounded slice, which is what the daily timer does. mod bauer; +mod check; mod db; +mod deezer; mod dr; mod hitlisten; mod http; +mod musicbrainz; use anyhow::Result; use chrono::Utc; @@ -30,8 +34,9 @@ fn main() -> Result<()> { match args.get(1).map(String::as_str) { Some("collect") => collect(&db, &args, &db_path), Some("stats") => stats(&db), + Some("check") => check::run(&db), _ => { - eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten] [--backfill]\n airplay stats [--db FILE]"); + eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|musicbrainz] [--backfill]\n airplay stats [--db FILE]\n airplay check [--db FILE]"); std::process::exit(2); } } @@ -75,6 +80,18 @@ fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> { Err(e) => step("bauer stations", Err(e), &mut log), } } + if wants("deezer") { + // Deezer allows 50 requests per 5 seconds; its own client keeps well under that. + let deezer_http = http::Http::new(150)?; + let r = deezer::collect(db, &deezer_http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 3000 }), &mut log); + step("deezer song metadata", r.map(|n| format!("{n} songs looked up")), &mut log); + } + if wants("musicbrainz") { + // MusicBrainz allows one request per second. + let mb_http = http::Http::new(1100)?; + let r = musicbrainz::collect(db, &mb_http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 2000 }), &mut log); + step("musicbrainz release dates", r.map(|n| format!("{n} ISRCs looked up")), &mut log); + } if wants("hitlisten") { let r = hitlisten::collect(db, &http, if backfill { usize::MAX } else { 60 }, &mut log); step("hitlisten airplay chart", r.map(|n| format!("{n} weeks fetched")), &mut log); @@ -108,5 +125,16 @@ fn stats(db: &db::Db) -> Result<()> { "SELECT count(DISTINCT year * 100 + week), min(printf('%d-W%02d', year, week)), max(printf('%d-W%02d', year, week)) FROM hitlisten_airplay", [], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?; println!("Hitlisten airplay chart: {n} weeks ({} to {})", first.unwrap_or_default(), last.unwrap_or_default()); + deezer::create_tables(db)?; + let (searched, matched, tracks, albums, labels): (i64, i64, i64, i64, i64) = db.conn.query_row( + "SELECT (SELECT count(*) FROM deezer_searches), (SELECT count(*) FROM deezer_searches WHERE best_track_id IS NOT NULL), + (SELECT count(*) FROM deezer_tracks), (SELECT count(*) FROM deezer_albums), + (SELECT count(*) FROM deezer_albums WHERE coalesce(label, '') != '')", [], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)))?; + musicbrainz::create_tables(db)?; + let (mb, mb_known): (i64, i64) = db.conn.query_row( + "SELECT count(*), count(*) FILTER (WHERE recordings > 0) FROM mb_isrc", [], |r| Ok((r.get(0)?, r.get(1)?)))?; + println!("MusicBrainz: {mb} ISRCs looked up, {mb_known} known (original release dates)"); + println!("Deezer: {searched} songs looked up, {matched} matched; {tracks} tracks, {albums} releases ({labels} with a label)"); Ok(()) } diff --git a/src/musicbrainz.rs b/src/musicbrainz.rs new file mode 100644 index 0000000..916ec71 --- /dev/null +++ b/src/musicbrainz.rs @@ -0,0 +1,52 @@ +//! Original release dates from MusicBrainz, by ISRC: a recording's `first-release-date` is its +//! earliest known release, where Deezer often gives a reissue or compilation instead (2Pac's +//! "Changes" as 2009). MusicBrainz misses many new and Danish releases; for those Deezer's +//! date is the original anyway. Cleaning takes the earliest date any source gives. +//! +//! GET https://musicbrainz.org/ws/2/isrc/{isrc}?fmt=json&inc=artist-credits +//! +//! MusicBrainz allows one request per second and asks for an identifying User-Agent. + +use crate::{db::Db, http::Http}; +use anyhow::Result; +use chrono::Utc; +use rusqlite::params; +use serde_json::Value; + +pub fn create_tables(db: &Db) -> Result<()> { + db.conn.execute_batch( + "CREATE TABLE IF NOT EXISTS mb_isrc ( + isrc TEXT PRIMARY KEY, + recordings INTEGER NOT NULL, -- 0: MusicBrainz doesn't know this ISRC + first_release_date TEXT, -- earliest over the recordings with this ISRC + recording_ids TEXT, + fetched_at TEXT NOT NULL, raw TEXT NOT NULL + );", + )?; + Ok(()) +} + +pub fn collect(db: &Db, http: &Http, limit: usize, log: &mut Vec) -> Result { + create_tables(db)?; + crate::deezer::create_tables(db)?; + let todo: Vec = db.conn.prepare( + "SELECT DISTINCT isrc FROM deezer_tracks WHERE coalesce(isrc, '') != '' + AND isrc NOT IN (SELECT isrc FROM mb_isrc)")? + .query_map([], |r| r.get(0))?.collect::>()?; + let mut done = 0; + for isrc in todo.into_iter().take(limit) { + let body = match http.get(&format!("https://musicbrainz.org/ws/2/isrc/{isrc}?fmt=json&inc=artist-credits")) { + Ok(b) => b, + Err(e) => { log.push(format!("musicbrainz {isrc}: {e}")); continue; } + }; + // 404 = MusicBrainz has no recording with this ISRC: stored as 0 recordings. + let v: Value = body.as_deref().map(serde_json::from_str).transpose()?.unwrap_or(Value::Null); + let recs = v["recordings"].as_array().cloned().unwrap_or_default(); + let first = recs.iter().filter_map(|r| r["first-release-date"].as_str()).filter(|d| !d.is_empty()).min().map(str::to_string); + let ids: Vec<&str> = recs.iter().filter_map(|r| r["id"].as_str()).collect(); + db.conn.execute("INSERT OR REPLACE INTO mb_isrc VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + params![isrc, recs.len() as i64, first, ids.join(","), Utc::now().to_rfc3339(), v.to_string()])?; + done += 1; + } + Ok(done) +}