Song metadata: Deezer (ISRC, release date, label, type, genres, contributors) and MusicBrainz original release dates; data audit (airplay check)

Raw Deezer answers are kept so matches can be redone during cleaning. Every strong match's
release is stored, so the earliest release date can be taken (Deezer often returns reissues).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-15 01:14:11 +00:00
co-authored by Claude Opus 5.5
parent 6e8b258dcf
commit ccae36e11a
5 changed files with 376 additions and 2 deletions
+30 -2
View File
@@ -1,16 +1,20 @@
//! airplay: collects what Danish radio stations play, for modelling when artists get played again.
//!
//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten] [--backfill]
//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|musicbrainz] [--backfill]
//! airplay stats [--db data/airplay.db]
//! airplay check [--db data/airplay.db] gaps, duplicates, DR cross-check, chart completeness
//!
//! --backfill fetches all history at once (every DR artist page, the whole Hitlisten archive);
//! without it a run takes a bounded slice, which is what the daily timer does.
mod bauer;
mod check;
mod db;
mod deezer;
mod dr;
mod hitlisten;
mod http;
mod musicbrainz;
use anyhow::Result;
use chrono::Utc;
@@ -30,8 +34,9 @@ fn main() -> Result<()> {
match args.get(1).map(String::as_str) {
Some("collect") => collect(&db, &args, &db_path),
Some("stats") => stats(&db),
Some("check") => check::run(&db),
_ => {
eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten] [--backfill]\n airplay stats [--db FILE]");
eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|musicbrainz] [--backfill]\n airplay stats [--db FILE]\n airplay check [--db FILE]");
std::process::exit(2);
}
}
@@ -75,6 +80,18 @@ fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> {
Err(e) => step("bauer stations", Err(e), &mut log),
}
}
if wants("deezer") {
// Deezer allows 50 requests per 5 seconds; its own client keeps well under that.
let deezer_http = http::Http::new(150)?;
let r = deezer::collect(db, &deezer_http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 3000 }), &mut log);
step("deezer song metadata", r.map(|n| format!("{n} songs looked up")), &mut log);
}
if wants("musicbrainz") {
// MusicBrainz allows one request per second.
let mb_http = http::Http::new(1100)?;
let r = musicbrainz::collect(db, &mb_http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 2000 }), &mut log);
step("musicbrainz release dates", r.map(|n| format!("{n} ISRCs looked up")), &mut log);
}
if wants("hitlisten") {
let r = hitlisten::collect(db, &http, if backfill { usize::MAX } else { 60 }, &mut log);
step("hitlisten airplay chart", r.map(|n| format!("{n} weeks fetched")), &mut log);
@@ -108,5 +125,16 @@ fn stats(db: &db::Db) -> Result<()> {
"SELECT count(DISTINCT year * 100 + week), min(printf('%d-W%02d', year, week)), max(printf('%d-W%02d', year, week)) FROM hitlisten_airplay",
[], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?;
println!("Hitlisten airplay chart: {n} weeks ({} to {})", first.unwrap_or_default(), last.unwrap_or_default());
deezer::create_tables(db)?;
let (searched, matched, tracks, albums, labels): (i64, i64, i64, i64, i64) = db.conn.query_row(
"SELECT (SELECT count(*) FROM deezer_searches), (SELECT count(*) FROM deezer_searches WHERE best_track_id IS NOT NULL),
(SELECT count(*) FROM deezer_tracks), (SELECT count(*) FROM deezer_albums),
(SELECT count(*) FROM deezer_albums WHERE coalesce(label, '') != '')", [],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)))?;
musicbrainz::create_tables(db)?;
let (mb, mb_known): (i64, i64) = db.conn.query_row(
"SELECT count(*), count(*) FILTER (WHERE recordings > 0) FROM mb_isrc", [], |r| Ok((r.get(0)?, r.get(1)?)))?;
println!("MusicBrainz: {mb} ISRCs looked up, {mb_known} known (original release dates)");
println!("Deezer: {searched} songs looked up, {matched} matched; {tracks} tracks, {albums} releases ({labels} with a label)");
Ok(())
}