Song metadata: Deezer (ISRC, release date, label, type, genres, contributors) and MusicBrainz original release dates; data audit (airplay check)
Raw Deezer answers are kept so matches can be redone during cleaning. Every strong match's release is stored, so the earliest release date can be taken (Deezer often returns reissues). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+127
@@ -0,0 +1,127 @@
|
|||||||
|
//! `airplay check`: how complete and consistent the collected data is.
|
||||||
|
//!
|
||||||
|
//! - gaps: hours in which a station has no plays (a failed fetch, or the station off air)
|
||||||
|
//! - cut short: plays followed by the next song before half of their length (snippets, teasers,
|
||||||
|
//! DJ segues; checked by hand on P3: different songs, not duplicates)
|
||||||
|
//! - DR cross-check: plays we counted per artist, channel and week against the weekly totals on
|
||||||
|
//! DR's artist pages (two independent DR sources), for weeks we cover completely
|
||||||
|
//! - Hitlisten: weeks missing, and weeks without exactly 20 rows
|
||||||
|
|
||||||
|
use crate::{db::Db, hitlisten};
|
||||||
|
use anyhow::Result;
|
||||||
|
use chrono::{DateTime, Datelike, Duration, NaiveDate, Utc};
|
||||||
|
use chrono_tz::Europe::Copenhagen;
|
||||||
|
use std::collections::{BTreeMap, HashMap};
|
||||||
|
|
||||||
|
fn parse(t: &str) -> Option<DateTime<Utc>> {
|
||||||
|
DateTime::parse_from_rfc3339(t).ok().map(|d| d.with_timezone(&Utc))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn run(db: &Db) -> Result<()> {
|
||||||
|
gaps_and_overlaps(db)?;
|
||||||
|
dr_cross_check(db)?;
|
||||||
|
hitlisten_check(db)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn gaps_and_overlaps(db: &Db) -> Result<()> {
|
||||||
|
println!("== Gaps and short plays per station (gap = an hour with no plays; cut short = next song started before half of this one)");
|
||||||
|
println!("{:<12} {:>7} {:>10} {:>9} longest gap", "station", "plays", "empty hrs", "cut short");
|
||||||
|
let stations: Vec<String> = db.conn.prepare("SELECT DISTINCT station FROM plays ORDER BY station")?
|
||||||
|
.query_map([], |r| r.get(0))?.collect::<Result<_, _>>()?;
|
||||||
|
for station in stations {
|
||||||
|
let mut st = db.conn.prepare("SELECT played_at, coalesce(duration_ms, 0) FROM plays WHERE station = ?1 ORDER BY played_at")?;
|
||||||
|
let rows: Vec<(DateTime<Utc>, i64)> = st.query_map([&station], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?)))?
|
||||||
|
.filter_map(|r| r.ok()).filter_map(|(t, d)| Some((parse(&t)?, d))).collect();
|
||||||
|
if rows.len() < 2 { continue; }
|
||||||
|
let mut empty_hours = 0;
|
||||||
|
let mut longest = (Duration::zero(), rows[0].0);
|
||||||
|
let mut overlaps = 0;
|
||||||
|
for w in rows.windows(2) {
|
||||||
|
let gap = w[1].0 - w[0].0;
|
||||||
|
if gap > Duration::hours(1) { empty_hours += (gap.num_minutes() / 60) as i64; }
|
||||||
|
if gap > longest.0 { longest = (gap, w[0].0); }
|
||||||
|
// The next song started before half of this one had played.
|
||||||
|
if w[0].1 > 0 && gap < Duration::milliseconds(w[0].1 / 2) { overlaps += 1; }
|
||||||
|
}
|
||||||
|
println!("{:<12} {:>7} {:>10} {:>9} {} min after {}", station, rows.len(), empty_hours, overlaps,
|
||||||
|
longest.0.num_minutes(), longest.1.with_timezone(&Copenhagen).format("%Y-%m-%d %H:%M"));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn monday(d: NaiveDate) -> NaiveDate {
|
||||||
|
d - Duration::days(d.weekday().num_days_from_monday() as i64)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn dr_cross_check(db: &Db) -> Result<()> {
|
||||||
|
println!("\n== DR: our plays vs the weekly totals on DR's artist pages");
|
||||||
|
// Weeks (Danish time, Monday-Sunday) every DR channel covers from start to end.
|
||||||
|
let (first, last): (Option<String>, Option<String>) = db.conn.query_row(
|
||||||
|
"SELECT max(f), min(l) FROM (SELECT min(played_at) f, max(played_at) l FROM plays WHERE source = 'dr' GROUP BY station)",
|
||||||
|
[], |r| Ok((r.get(0)?, r.get(1)?)))?;
|
||||||
|
let (Some(first), Some(last)) = (first.as_deref().and_then(parse), last.as_deref().and_then(parse)) else {
|
||||||
|
println!("no DR plays yet");
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
|
let start = monday(first.with_timezone(&Copenhagen).date_naive()) + Duration::days(7);
|
||||||
|
let end = monday(last.with_timezone(&Copenhagen).date_naive()); // weeks before this are complete
|
||||||
|
if start >= end {
|
||||||
|
println!("no complete Monday-Sunday week collected yet (data from {} to {}); available after {}",
|
||||||
|
first.with_timezone(&Copenhagen).format("%Y-%m-%d"), last.with_timezone(&Copenhagen).format("%Y-%m-%d"),
|
||||||
|
start + Duration::days(7));
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let mut ours: HashMap<(String, String, NaiveDate), i64> = HashMap::new();
|
||||||
|
let mut st = db.conn.prepare("SELECT station, played_at, artist_ids FROM plays WHERE source = 'dr'")?;
|
||||||
|
for r in st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?)))? {
|
||||||
|
let (station, t, ids) = r?;
|
||||||
|
let Some(t) = parse(&t) else { continue };
|
||||||
|
let week = monday(t.with_timezone(&Copenhagen).date_naive());
|
||||||
|
if week < start || week >= end { continue; }
|
||||||
|
for id in ids.split(',').filter(|s| !s.is_empty()) {
|
||||||
|
*ours.entry((id.to_string(), station.clone(), week)).or_default() += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mut theirs = db.conn.prepare("SELECT artist_id, channel, week_start, plays FROM dr_artist_weeks WHERE week_start >= ?1 AND week_start < ?2")?;
|
||||||
|
let mut compared = 0;
|
||||||
|
let mut exact = 0;
|
||||||
|
let mut diffs: BTreeMap<i64, i64> = BTreeMap::new();
|
||||||
|
for r in theirs.query_map([start.to_string(), end.to_string()], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, i64>(3)?)))? {
|
||||||
|
let (id, channel, week, plays) = r?;
|
||||||
|
let Ok(week) = NaiveDate::parse_from_str(&week, "%Y-%m-%d") else { continue };
|
||||||
|
let Some(&n) = ours.get(&(id, channel, week)) else { continue };
|
||||||
|
compared += 1;
|
||||||
|
if n == plays { exact += 1; }
|
||||||
|
*diffs.entry((n - plays).clamp(-5, 5)).or_default() += 1;
|
||||||
|
}
|
||||||
|
println!("weeks {start} to {}: {compared} artist-channel-weeks compared, {exact} identical ({:.1}%)",
|
||||||
|
end - Duration::days(1), if compared > 0 { 100.0 * exact as f64 / compared as f64 } else { 0.0 });
|
||||||
|
println!("difference (ours - DR's), clamped to ±5: {diffs:?}");
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hitlisten_check(db: &Db) -> Result<()> {
|
||||||
|
println!("\n== Hitlisten Airplay Top-20");
|
||||||
|
hitlisten::create_tables(db)?;
|
||||||
|
let weeks: Vec<(i64, i64, i64)> = db.conn.prepare("SELECT year, week, count(*) FROM hitlisten_airplay GROUP BY year, week ORDER BY year, week")?
|
||||||
|
.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?.collect::<Result<_, _>>()?;
|
||||||
|
if weeks.is_empty() { println!("nothing collected"); return Ok(()); }
|
||||||
|
let odd: Vec<String> = weeks.iter().filter(|w| w.2 != 20).map(|w| format!("{}-W{:02} ({} rows)", w.0, w.1, w.2)).collect();
|
||||||
|
// Missing weeks between the first and the last (ISO weeks; some years have 53).
|
||||||
|
let have: std::collections::HashSet<(i64, i64)> = weeks.iter().map(|w| (w.0, w.1)).collect();
|
||||||
|
let mut missing = Vec::new();
|
||||||
|
let (fy, fw, ly, lw) = (weeks[0].0, weeks[0].1, weeks.last().unwrap().0, weeks.last().unwrap().1);
|
||||||
|
for y in fy..=ly {
|
||||||
|
let last_week = NaiveDate::from_ymd_opt(y as i32, 12, 28).map(|d| d.iso_week().week() as i64).unwrap_or(52);
|
||||||
|
for w in 1..=last_week {
|
||||||
|
if (y == fy && w < fw) || (y == ly && w > lw) { continue; }
|
||||||
|
if !have.contains(&(y, w)) { missing.push(format!("{y}-W{w:02}")); }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
println!("{} weeks from {fy}-W{fw:02} to {ly}-W{lw:02}; {} missing; {} without 20 rows",
|
||||||
|
weeks.len(), missing.len(), odd.len());
|
||||||
|
if !missing.is_empty() { println!("missing: {}", missing.iter().take(40).cloned().collect::<Vec<_>>().join(", ")); }
|
||||||
|
if !odd.is_empty() { println!("odd: {}", odd.iter().take(20).cloned().collect::<Vec<_>>().join(", ")); }
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
@@ -22,6 +22,8 @@ pub struct Play<'a> {
|
|||||||
impl Db {
|
impl Db {
|
||||||
pub fn open(path: &str) -> Result<Db> {
|
pub fn open(path: &str) -> Result<Db> {
|
||||||
let conn = Connection::open(path)?;
|
let conn = Connection::open(path)?;
|
||||||
|
// Several collectors may write at once (the timer, a backfill); wait instead of failing.
|
||||||
|
conn.busy_timeout(std::time::Duration::from_secs(120))?;
|
||||||
conn.execute_batch(
|
conn.execute_batch(
|
||||||
"PRAGMA journal_mode=WAL;
|
"PRAGMA journal_mode=WAL;
|
||||||
CREATE TABLE IF NOT EXISTS plays (
|
CREATE TABLE IF NOT EXISTS plays (
|
||||||
|
|||||||
+165
@@ -0,0 +1,165 @@
|
|||||||
|
//! Song metadata from Deezer's public API (no key): ISRC, release date, label, release type,
|
||||||
|
//! genres, contributors (featured artists) and popularity, for every distinct artist + title in
|
||||||
|
//! the plays and the charts.
|
||||||
|
//!
|
||||||
|
//! GET https://api.deezer.com/search?q={artist} {title} up to 25 candidate tracks
|
||||||
|
//! GET https://api.deezer.com/track/{id} isrc, release_date, contributors[], rank
|
||||||
|
//! GET https://api.deezer.com/album/{id} label, upc, record_type, genres, release_date
|
||||||
|
//!
|
||||||
|
//! Matching is deliberately simple here: the raw search answers are kept (deezer_searches.raw),
|
||||||
|
//! so the cleaning step can pick matches again without fetching. Rate limit: 50 requests / 5 s.
|
||||||
|
|
||||||
|
use crate::{db::Db, http::Http};
|
||||||
|
use anyhow::Result;
|
||||||
|
use chrono::Utc;
|
||||||
|
use rusqlite::{params, OptionalExtension};
|
||||||
|
use serde_json::Value;
|
||||||
|
|
||||||
|
const BASE: &str = "https://api.deezer.com";
|
||||||
|
|
||||||
|
pub fn create_tables(db: &Db) -> Result<()> {
|
||||||
|
db.conn.execute_batch(
|
||||||
|
"CREATE TABLE IF NOT EXISTS deezer_searches (
|
||||||
|
artist TEXT NOT NULL, title TEXT NOT NULL, -- exactly as the station or chart wrote them
|
||||||
|
query TEXT NOT NULL, results INTEGER NOT NULL,
|
||||||
|
best_track_id INTEGER, best_score REAL, -- the simple matcher's pick (redo in cleaning)
|
||||||
|
fetched_at TEXT NOT NULL, raw TEXT NOT NULL,
|
||||||
|
PRIMARY KEY (artist, title)
|
||||||
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS deezer_tracks (
|
||||||
|
track_id INTEGER PRIMARY KEY, isrc TEXT, title TEXT, artist TEXT,
|
||||||
|
contributors TEXT, -- JSON [{name, role}]
|
||||||
|
release_date TEXT, duration_s INTEGER, rank INTEGER, explicit INTEGER,
|
||||||
|
album_id INTEGER, fetched_at TEXT NOT NULL, raw TEXT NOT NULL
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS deezer_tracks_isrc ON deezer_tracks (isrc);
|
||||||
|
CREATE TABLE IF NOT EXISTS deezer_albums (
|
||||||
|
album_id INTEGER PRIMARY KEY, title TEXT, upc TEXT, label TEXT, release_date TEXT,
|
||||||
|
record_type TEXT, genres TEXT, nb_tracks INTEGER, fetched_at TEXT NOT NULL, raw TEXT NOT NULL
|
||||||
|
);",
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Lower-case letters and digits only, for comparing names.
|
||||||
|
fn key(s: &str) -> String {
|
||||||
|
s.to_lowercase().chars().filter(|c| c.is_alphanumeric()).collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The first artist ("Shakira & Burna Boy", "X ft. Y", "A, B" -> the first name) and the title
|
||||||
|
/// without version tags, for the search query only; the stored names stay as written.
|
||||||
|
fn query_parts(artist: &str, title: &str) -> (String, String) {
|
||||||
|
let lower = artist.to_lowercase();
|
||||||
|
let mut cut = artist.len();
|
||||||
|
for sep in [" feat.", " feat ", " ft.", " ft ", " featuring ", " & ", ", ", " x ", " og ", " [+] ", " med "] {
|
||||||
|
if let Some(i) = lower.find(sep) { cut = cut.min(i); }
|
||||||
|
}
|
||||||
|
let first = artist[..cut].trim().to_string();
|
||||||
|
let title = title.split(" (").next().unwrap_or(title).split(" - ").next().unwrap_or(title).trim().to_string();
|
||||||
|
(first, title)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 0..1: how well a candidate matches (title equal or contained, artist contained).
|
||||||
|
fn score(candidate: &Value, artist: &str, title: &str) -> f64 {
|
||||||
|
let (ct, ca) = (key(candidate["title"].as_str().unwrap_or_default()), key(candidate["artist"]["name"].as_str().unwrap_or_default()));
|
||||||
|
let (qt, qa) = (key(title), key(artist));
|
||||||
|
let t = if ct == qt { 1.0 } else if !qt.is_empty() && (ct.contains(&qt) || qt.contains(&ct)) { 0.7 } else { 0.0 };
|
||||||
|
let a = if ca == qa { 1.0 } else if !ca.is_empty() && (qa.contains(&ca) || ca.contains(&qa)) { 0.8 } else { 0.0 };
|
||||||
|
0.6 * t + 0.4 * a
|
||||||
|
}
|
||||||
|
|
||||||
|
fn fetch_json(http: &Http, url: &str) -> Result<Option<Value>> {
|
||||||
|
Ok(match http.get(url)? {
|
||||||
|
Some(b) => {
|
||||||
|
let v: Value = serde_json::from_str(&b)?;
|
||||||
|
if v.get("error").is_some() { anyhow::bail!("deezer: {}", v["error"]); }
|
||||||
|
Some(v)
|
||||||
|
}
|
||||||
|
None => None,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Looks up every artist + title not searched yet (`limit` per run).
|
||||||
|
pub fn collect(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) -> Result<usize> {
|
||||||
|
create_tables(db)?;
|
||||||
|
crate::hitlisten::create_tables(db)?;
|
||||||
|
let todo: Vec<(String, String)> = db.conn.prepare(
|
||||||
|
"SELECT artist, title FROM (SELECT artist, title FROM plays UNION SELECT artist, title FROM hitlisten_airplay) s
|
||||||
|
WHERE NOT EXISTS (SELECT 1 FROM deezer_searches d WHERE d.artist = s.artist AND d.title = s.title)
|
||||||
|
AND artist != '' AND title != ''")?
|
||||||
|
.query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?.collect::<Result<_, _>>()?;
|
||||||
|
let now = Utc::now().to_rfc3339();
|
||||||
|
let mut done = 0;
|
||||||
|
for (artist, title) in todo.into_iter().take(limit) {
|
||||||
|
let (qa, qt) = query_parts(&artist, &title);
|
||||||
|
let query = format!("{qa} {qt}");
|
||||||
|
let url = format!("{BASE}/search?q={}&limit=10", urlencode(&query));
|
||||||
|
let found = match fetch_json(http, &url) {
|
||||||
|
Ok(v) => v.unwrap_or(Value::Null),
|
||||||
|
Err(e) => { log.push(format!("deezer search '{query}': {e}")); continue; }
|
||||||
|
};
|
||||||
|
let hits = found["data"].as_array().cloned().unwrap_or_default();
|
||||||
|
let best = hits.iter().map(|h| (score(h, &qa, &qt), h)).max_by(|a, b| a.0.total_cmp(&b.0));
|
||||||
|
let (best_id, best_score) = match best {
|
||||||
|
Some((s, h)) if s >= 0.6 => (h["id"].as_i64(), Some(s)),
|
||||||
|
Some((s, _)) => (None, Some(s)),
|
||||||
|
None => (None, None),
|
||||||
|
};
|
||||||
|
if let Some(id) = best_id {
|
||||||
|
if let Err(e) = track(db, http, id, &now) { log.push(format!("deezer track {id}: {e}")); }
|
||||||
|
}
|
||||||
|
// Every other strong match is another release of the same song (album, compilation,
|
||||||
|
// reissue): store those releases too, so cleaning can take the earliest release date.
|
||||||
|
for h in hits.iter().filter(|h| score(h, &qa, &qt) >= 0.9).take(6) {
|
||||||
|
if let Some(album_id) = h["album"]["id"].as_i64() {
|
||||||
|
if let Err(e) = album(db, http, album_id, &now) { log.push(format!("deezer album {album_id}: {e}")); }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
db.conn.execute("INSERT OR REPLACE INTO deezer_searches VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
|
||||||
|
params![artist, title, query, hits.len() as i64, best_id, best_score, now, found.to_string()])?;
|
||||||
|
done += 1;
|
||||||
|
}
|
||||||
|
Ok(done)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Stores a track and its album (each fetched once).
|
||||||
|
fn track(db: &Db, http: &Http, id: i64, now: &str) -> Result<()> {
|
||||||
|
let have: Option<i64> = db.conn.query_row("SELECT album_id FROM deezer_tracks WHERE track_id = ?1", [id], |r| r.get(0)).optional()?;
|
||||||
|
let album_id = match have {
|
||||||
|
Some(a) => a,
|
||||||
|
None => {
|
||||||
|
let Some(t) = fetch_json(http, &format!("{BASE}/track/{id}"))? else { return Ok(()) };
|
||||||
|
let contributors: Vec<Value> = t["contributors"].as_array().cloned().unwrap_or_default().iter()
|
||||||
|
.map(|c| serde_json::json!({"name": c["name"], "role": c["role"], "id": c["id"]})).collect();
|
||||||
|
db.conn.execute("INSERT OR REPLACE INTO deezer_tracks VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)",
|
||||||
|
params![id, t["isrc"].as_str(), t["title"].as_str(), t["artist"]["name"].as_str(),
|
||||||
|
Value::Array(contributors).to_string(), t["release_date"].as_str(), t["duration"].as_i64(),
|
||||||
|
t["rank"].as_i64(), t["explicit_lyrics"].as_bool().map(|b| b as i64),
|
||||||
|
t["album"]["id"].as_i64(), now, t.to_string()])?;
|
||||||
|
t["album"]["id"].as_i64().unwrap_or(0)
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if album_id != 0 { album(db, http, album_id, now)?; }
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Stores a release once.
|
||||||
|
fn album(db: &Db, http: &Http, album_id: i64, now: &str) -> Result<()> {
|
||||||
|
let known: i64 = db.conn.query_row("SELECT count(*) FROM deezer_albums WHERE album_id = ?1", [album_id], |r| r.get(0))?;
|
||||||
|
if known > 0 { return Ok(()); }
|
||||||
|
if let Some(a) = fetch_json(http, &format!("{BASE}/album/{album_id}"))? {
|
||||||
|
let genres: Vec<&str> = a["genres"]["data"].as_array().map(|g| g.iter().filter_map(|x| x["name"].as_str()).collect()).unwrap_or_default();
|
||||||
|
db.conn.execute("INSERT OR REPLACE INTO deezer_albums VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)",
|
||||||
|
params![album_id, a["title"].as_str(), a["upc"].as_str(), a["label"].as_str(), a["release_date"].as_str(),
|
||||||
|
a["record_type"].as_str(), genres.join(", "), a["nb_tracks"].as_i64(), now, a.to_string()])?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn urlencode(s: &str) -> String {
|
||||||
|
s.bytes().map(|b| match b {
|
||||||
|
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => (b as char).to_string(),
|
||||||
|
b' ' => "%20".into(),
|
||||||
|
_ => format!("%{b:02X}"),
|
||||||
|
}).collect()
|
||||||
|
}
|
||||||
+30
-2
@@ -1,16 +1,20 @@
|
|||||||
//! airplay: collects what Danish radio stations play, for modelling when artists get played again.
|
//! airplay: collects what Danish radio stations play, for modelling when artists get played again.
|
||||||
//!
|
//!
|
||||||
//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten] [--backfill]
|
//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|musicbrainz] [--backfill]
|
||||||
//! airplay stats [--db data/airplay.db]
|
//! airplay stats [--db data/airplay.db]
|
||||||
|
//! airplay check [--db data/airplay.db] gaps, duplicates, DR cross-check, chart completeness
|
||||||
//!
|
//!
|
||||||
//! --backfill fetches all history at once (every DR artist page, the whole Hitlisten archive);
|
//! --backfill fetches all history at once (every DR artist page, the whole Hitlisten archive);
|
||||||
//! without it a run takes a bounded slice, which is what the daily timer does.
|
//! without it a run takes a bounded slice, which is what the daily timer does.
|
||||||
|
|
||||||
mod bauer;
|
mod bauer;
|
||||||
|
mod check;
|
||||||
mod db;
|
mod db;
|
||||||
|
mod deezer;
|
||||||
mod dr;
|
mod dr;
|
||||||
mod hitlisten;
|
mod hitlisten;
|
||||||
mod http;
|
mod http;
|
||||||
|
mod musicbrainz;
|
||||||
|
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use chrono::Utc;
|
use chrono::Utc;
|
||||||
@@ -30,8 +34,9 @@ fn main() -> Result<()> {
|
|||||||
match args.get(1).map(String::as_str) {
|
match args.get(1).map(String::as_str) {
|
||||||
Some("collect") => collect(&db, &args, &db_path),
|
Some("collect") => collect(&db, &args, &db_path),
|
||||||
Some("stats") => stats(&db),
|
Some("stats") => stats(&db),
|
||||||
|
Some("check") => check::run(&db),
|
||||||
_ => {
|
_ => {
|
||||||
eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten] [--backfill]\n airplay stats [--db FILE]");
|
eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|musicbrainz] [--backfill]\n airplay stats [--db FILE]\n airplay check [--db FILE]");
|
||||||
std::process::exit(2);
|
std::process::exit(2);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -75,6 +80,18 @@ fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> {
|
|||||||
Err(e) => step("bauer stations", Err(e), &mut log),
|
Err(e) => step("bauer stations", Err(e), &mut log),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if wants("deezer") {
|
||||||
|
// Deezer allows 50 requests per 5 seconds; its own client keeps well under that.
|
||||||
|
let deezer_http = http::Http::new(150)?;
|
||||||
|
let r = deezer::collect(db, &deezer_http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 3000 }), &mut log);
|
||||||
|
step("deezer song metadata", r.map(|n| format!("{n} songs looked up")), &mut log);
|
||||||
|
}
|
||||||
|
if wants("musicbrainz") {
|
||||||
|
// MusicBrainz allows one request per second.
|
||||||
|
let mb_http = http::Http::new(1100)?;
|
||||||
|
let r = musicbrainz::collect(db, &mb_http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 2000 }), &mut log);
|
||||||
|
step("musicbrainz release dates", r.map(|n| format!("{n} ISRCs looked up")), &mut log);
|
||||||
|
}
|
||||||
if wants("hitlisten") {
|
if wants("hitlisten") {
|
||||||
let r = hitlisten::collect(db, &http, if backfill { usize::MAX } else { 60 }, &mut log);
|
let r = hitlisten::collect(db, &http, if backfill { usize::MAX } else { 60 }, &mut log);
|
||||||
step("hitlisten airplay chart", r.map(|n| format!("{n} weeks fetched")), &mut log);
|
step("hitlisten airplay chart", r.map(|n| format!("{n} weeks fetched")), &mut log);
|
||||||
@@ -108,5 +125,16 @@ fn stats(db: &db::Db) -> Result<()> {
|
|||||||
"SELECT count(DISTINCT year * 100 + week), min(printf('%d-W%02d', year, week)), max(printf('%d-W%02d', year, week)) FROM hitlisten_airplay",
|
"SELECT count(DISTINCT year * 100 + week), min(printf('%d-W%02d', year, week)), max(printf('%d-W%02d', year, week)) FROM hitlisten_airplay",
|
||||||
[], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?;
|
[], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?;
|
||||||
println!("Hitlisten airplay chart: {n} weeks ({} to {})", first.unwrap_or_default(), last.unwrap_or_default());
|
println!("Hitlisten airplay chart: {n} weeks ({} to {})", first.unwrap_or_default(), last.unwrap_or_default());
|
||||||
|
deezer::create_tables(db)?;
|
||||||
|
let (searched, matched, tracks, albums, labels): (i64, i64, i64, i64, i64) = db.conn.query_row(
|
||||||
|
"SELECT (SELECT count(*) FROM deezer_searches), (SELECT count(*) FROM deezer_searches WHERE best_track_id IS NOT NULL),
|
||||||
|
(SELECT count(*) FROM deezer_tracks), (SELECT count(*) FROM deezer_albums),
|
||||||
|
(SELECT count(*) FROM deezer_albums WHERE coalesce(label, '') != '')", [],
|
||||||
|
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)))?;
|
||||||
|
musicbrainz::create_tables(db)?;
|
||||||
|
let (mb, mb_known): (i64, i64) = db.conn.query_row(
|
||||||
|
"SELECT count(*), count(*) FILTER (WHERE recordings > 0) FROM mb_isrc", [], |r| Ok((r.get(0)?, r.get(1)?)))?;
|
||||||
|
println!("MusicBrainz: {mb} ISRCs looked up, {mb_known} known (original release dates)");
|
||||||
|
println!("Deezer: {searched} songs looked up, {matched} matched; {tracks} tracks, {albums} releases ({labels} with a label)");
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,52 @@
|
|||||||
|
//! Original release dates from MusicBrainz, by ISRC: a recording's `first-release-date` is its
|
||||||
|
//! earliest known release, where Deezer often gives a reissue or compilation instead (2Pac's
|
||||||
|
//! "Changes" as 2009). MusicBrainz misses many new and Danish releases; for those Deezer's
|
||||||
|
//! date is the original anyway. Cleaning takes the earliest date any source gives.
|
||||||
|
//!
|
||||||
|
//! GET https://musicbrainz.org/ws/2/isrc/{isrc}?fmt=json&inc=artist-credits
|
||||||
|
//!
|
||||||
|
//! MusicBrainz allows one request per second and asks for an identifying User-Agent.
|
||||||
|
|
||||||
|
use crate::{db::Db, http::Http};
|
||||||
|
use anyhow::Result;
|
||||||
|
use chrono::Utc;
|
||||||
|
use rusqlite::params;
|
||||||
|
use serde_json::Value;
|
||||||
|
|
||||||
|
pub fn create_tables(db: &Db) -> Result<()> {
|
||||||
|
db.conn.execute_batch(
|
||||||
|
"CREATE TABLE IF NOT EXISTS mb_isrc (
|
||||||
|
isrc TEXT PRIMARY KEY,
|
||||||
|
recordings INTEGER NOT NULL, -- 0: MusicBrainz doesn't know this ISRC
|
||||||
|
first_release_date TEXT, -- earliest over the recordings with this ISRC
|
||||||
|
recording_ids TEXT,
|
||||||
|
fetched_at TEXT NOT NULL, raw TEXT NOT NULL
|
||||||
|
);",
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn collect(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) -> Result<usize> {
|
||||||
|
create_tables(db)?;
|
||||||
|
crate::deezer::create_tables(db)?;
|
||||||
|
let todo: Vec<String> = db.conn.prepare(
|
||||||
|
"SELECT DISTINCT isrc FROM deezer_tracks WHERE coalesce(isrc, '') != ''
|
||||||
|
AND isrc NOT IN (SELECT isrc FROM mb_isrc)")?
|
||||||
|
.query_map([], |r| r.get(0))?.collect::<Result<_, _>>()?;
|
||||||
|
let mut done = 0;
|
||||||
|
for isrc in todo.into_iter().take(limit) {
|
||||||
|
let body = match http.get(&format!("https://musicbrainz.org/ws/2/isrc/{isrc}?fmt=json&inc=artist-credits")) {
|
||||||
|
Ok(b) => b,
|
||||||
|
Err(e) => { log.push(format!("musicbrainz {isrc}: {e}")); continue; }
|
||||||
|
};
|
||||||
|
// 404 = MusicBrainz has no recording with this ISRC: stored as 0 recordings.
|
||||||
|
let v: Value = body.as_deref().map(serde_json::from_str).transpose()?.unwrap_or(Value::Null);
|
||||||
|
let recs = v["recordings"].as_array().cloned().unwrap_or_default();
|
||||||
|
let first = recs.iter().filter_map(|r| r["first-release-date"].as_str()).filter(|d| !d.is_empty()).min().map(str::to_string);
|
||||||
|
let ids: Vec<&str> = recs.iter().filter_map(|r| r["id"].as_str()).collect();
|
||||||
|
db.conn.execute("INSERT OR REPLACE INTO mb_isrc VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
|
||||||
|
params![isrc, recs.len() as i64, first, ids.join(","), Utc::now().to_rfc3339(), v.to_string()])?;
|
||||||
|
done += 1;
|
||||||
|
}
|
||||||
|
Ok(done)
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user