Files
Airplay-Predictor/src/main.rs
T

204 lines
12 KiB
Rust

//! airplay: collects what Danish radio stations play, for modelling when artists get played again.
//!
//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf|nmf-artists] [--backfill]
//! airplay stats [--db data/airplay.db]
//! airplay check [--db data/airplay.db] gaps, duplicates, DR cross-check, chart completeness
//! airplay model panel | backtest | rank the first models (src/model)
//! airplay rotations [--wayback] [--reparse] parse DR's rotation PDFs (src/rotations.rs)
//! airplay serve [--addr 127.0.0.1:8790] [--prefix /airplay] the playground site (src/web.rs)
//!
//! --backfill fetches all history at once (every DR artist page, the whole Hitlisten archive);
//! without it a run takes a bounded slice, which is what the daily timer does.
mod bauer;
mod check;
mod db;
mod deezer;
mod dr;
mod hitlisten;
mod http;
mod model;
mod musicbrainz;
mod playlists;
mod rotations;
mod spotify;
mod web;
use anyhow::Result;
use chrono::Utc;
use rusqlite::params;
fn arg(args: &[String], name: &str) -> Option<String> {
args.iter().position(|a| a == name).and_then(|i| args.get(i + 1)).cloned()
}
fn main() -> Result<()> {
let args: Vec<String> = std::env::args().collect();
let db_path = arg(&args, "--db").unwrap_or_else(|| "data/airplay.db".into());
if let Some(dir) = std::path::Path::new(&db_path).parent() {
std::fs::create_dir_all(dir)?;
}
let db = db::Db::open(&db_path)?;
match args.get(1).map(String::as_str) {
Some("collect") => collect(&db, &args, &db_path),
Some("stats") => stats(&db),
Some("check") => check::run(&db),
Some("model") => model::run(&db, &args),
Some("serve") => web::serve(&db_path, &args),
Some("pdfdump") => rotations::dump(std::path::Path::new(args.get(2).map(String::as_str).unwrap_or(""))),
Some("rotations") => {
let dir = std::path::Path::new(&db_path).parent().unwrap_or(std::path::Path::new(".")).join("rotations");
if args.iter().any(|a| a == "--wayback") {
let mut log = Vec::new();
let n = rotations::wayback(&db, &http::Http::new(1500)?, &dir, &mut log)?;
println!("wayback: {n} PDFs saved");
for l in log { println!(" {l}"); }
}
rotations::store_all(&db, &dir, args.iter().any(|a| a == "--reparse"))
}
_ => {
eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf|nmf-artists] [--backfill]\n airplay stats [--db FILE]\n airplay check [--db FILE]\n airplay model panel|backtest|rank\n airplay serve [--addr HOST:PORT] [--prefix /airplay]");
std::process::exit(2);
}
}
}
fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> {
let days: i64 = arg(args, "--days").and_then(|d| d.parse().ok()).unwrap_or(7);
let only = arg(args, "--only");
let wants = |source: &str| only.as_deref().map_or(true, |o| o == source);
let backfill = args.iter().any(|a| a == "--backfill");
let http = http::Http::new(400)?;
let started = Utc::now().to_rfc3339();
db.conn.execute("INSERT INTO runs (started_at) VALUES (?1)", params![started])?;
let run = db.conn.last_insert_rowid();
let mut log: Vec<String> = Vec::new();
let mut new = 0;
let step = |name: &str, r: Result<String>, log: &mut Vec<String>| match r {
Ok(msg) => { println!("{name}: {msg}"); }
Err(e) => { eprintln!("{name}: {e}"); log.push(format!("{name}: {e}")); }
};
if wants("dr") {
let r = dr::collect_plays(db, &http, days, &mut log);
if let Ok(n) = &r { new += n; }
step("dr plays", r.map(|n| format!("{n} new")), &mut log);
let dir = std::path::Path::new(db_path).parent().unwrap_or(std::path::Path::new(".")).join("rotations");
let r = dr::collect_rotation_pdf(db, &http, &dir);
step("dr rotation list", r.map(|f| f.map(|f| format!("saved {f}")).unwrap_or_else(|| "unchanged".into())), &mut log);
let r = dr::collect_artists(db, &http, 6, if backfill { usize::MAX } else { 1500 }, &mut log);
step("dr artist pages", r.map(|n| format!("{n} fetched")), &mut log);
}
if wants("bauer") {
match bauer::stations(db, &http, args.iter().any(|a| a == "--all-bauer")) {
Ok(stations) => {
for (code, name) in stations {
let r = bauer::collect(db, &http, &code, days, &mut log);
if let Ok(n) = &r { new += n; }
step(&format!("bauer {code} ({name})"), r.map(|n| format!("{n} new")), &mut log);
}
}
Err(e) => step("bauer stations", Err(e), &mut log),
}
}
if wants("deezer") {
// Deezer allows 50 requests per 5 seconds; its own client keeps well under that.
let deezer_http = http::Http::new(150)?;
let r = deezer::collect(db, &deezer_http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 3000 }), &mut log);
step("deezer song metadata", r.map(|n| format!("{n} songs looked up")), &mut log);
let r = deezer::discographies(db, &deezer_http, if backfill { usize::MAX } else { 1500 }, &mut log);
step("deezer discographies", r.map(|n| format!("{n} artists looked up")), &mut log);
}
if wants("discographies") {
// On its own (next to a running song backfill): slower, to stay under Deezer's limit together.
let r = deezer::discographies(db, &http::Http::new(500)?, usize::MAX, &mut log);
step("deezer discographies", r.map(|n| format!("{n} artists looked up")), &mut log);
}
if wants("musicbrainz") {
// MusicBrainz allows one request per second.
let mb_http = http::Http::new(1100)?;
let r = musicbrainz::collect(db, &mb_http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 2000 }), &mut log);
step("musicbrainz release dates", r.map(|n| format!("{n} ISRCs looked up")), &mut log);
}
if wants("spotify") {
// kworb.net is a one-person site: one request a second.
let kworb_http = http::Http::new(1000)?;
let r = spotify::collect(db, &kworb_http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 400 }), &mut log);
step("spotify chart history (kworb)", r.map(|(l, p)| format!("{l} songs listed, {p} histories read")), &mut log);
}
if wants("playlists") {
let r = playlists::snapshot(db, &http, &mut log);
step("spotify playlists (New Music Friday DK, Top 50 DK)", r.map(|n| format!("{n} changed and saved")), &mut log);
}
if wants("nmf") || wants("nmf-artists") {
let sfc_http = http::Http::new(1000)?;
let r = playlists::nmf_artists(db, &sfc_http, if backfill { usize::MAX } else { 800 }, &mut log);
step("new music friday per artist (superfridaychart)", r.map(|n| format!("{n} artists looked up")), &mut log);
}
if wants("nmf") {
let sfc_http = http::Http::new(1000)?;
let r = playlists::nmf_history(db, &sfc_http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 1500 }), &mut log);
step("new music friday history (superfridaychart)", r.map(|n| format!("{n} ISRCs looked up")), &mut log);
}
if wants("hitlisten") {
let r = hitlisten::collect(db, &http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 60 }), &mut log);
step("hitlisten charts (airplay, track, album)", r.map(|n| format!("{n} weeks fetched")), &mut log);
}
db.conn.execute(
"UPDATE runs SET finished_at = ?1, new_plays = ?2, errors = ?3, log = ?4 WHERE id = ?5",
params![Utc::now().to_rfc3339(), new as i64, log.len() as i64, log.join("\n"), run],
)?;
println!("{new} new plays, {} problem(s)", log.len());
for l in log.iter().take(20) { eprintln!(" {l}"); }
Ok(())
}
fn stats(db: &db::Db) -> Result<()> {
let mut st = db.conn.prepare(
"SELECT p.station, coalesce(s.name, p.station), count(*), min(played_at), max(played_at), count(DISTINCT artist)
FROM plays p LEFT JOIN stations s ON s.code = p.station GROUP BY p.station ORDER BY count(*) DESC")?;
println!("{:<12} {:<22} {:>7} {:>8} {:<20} {}", "station", "name", "plays", "artists", "from", "to");
let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, i64>(2)?,
r.get::<_, String>(3)?, r.get::<_, String>(4)?, r.get::<_, i64>(5)?)))?;
for r in rows {
let (code, name, n, from, to, artists) = r?;
println!("{code:<12} {name:<22} {n:>7} {artists:>8} {from:<20} {to}");
}
let weeks: i64 = db.conn.query_row("SELECT count(*) FROM dr_artist_weeks", [], |r| r.get(0))?;
let artists: i64 = db.conn.query_row("SELECT count(*) FROM dr_artists", [], |r| r.get(0))?;
let pdfs: i64 = db.conn.query_row("SELECT count(*) FROM dr_rotation_pdfs", [], |r| r.get(0))?;
println!("DR artist history: {artists} artists, {weeks} artist-week-channel rows; rotation PDFs: {pdfs}");
hitlisten::create_tables(db)?;
for &(chart, _) in hitlisten::CHARTS {
let (n, first, last): (i64, Option<String>, Option<String>) = db.conn.query_row(
"SELECT count(DISTINCT year * 100 + week), min(printf('%d-W%02d', year, week)), max(printf('%d-W%02d', year, week)) FROM hitlisten_charts WHERE chart = ?1",
[chart], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?;
println!("Hitlisten {chart}: {n} weeks ({} to {})", first.unwrap_or_default(), last.unwrap_or_default());
}
deezer::create_tables(db)?;
let (searched, matched, tracks, albums, labels): (i64, i64, i64, i64, i64) = db.conn.query_row(
"SELECT (SELECT count(*) FROM deezer_searches), (SELECT count(*) FROM deezer_searches WHERE best_track_id IS NOT NULL),
(SELECT count(*) FROM deezer_tracks), (SELECT count(*) FROM deezer_albums),
(SELECT count(*) FROM deezer_albums WHERE coalesce(label, '') != '')", [],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)))?;
spotify::create_tables(db)?;
let (sp_tracks, sp_read, sp_rows, sp_from): (i64, i64, i64, Option<String>) = db.conn.query_row(
"SELECT (SELECT count(*) FROM spotify_dk_tracks), (SELECT count(*) FROM spotify_dk_tracks WHERE history_fetched_at IS NOT NULL),
(SELECT count(*) FROM spotify_chart_history WHERE country = 'DK'),
(SELECT min(date) FROM spotify_chart_history WHERE country = 'DK' AND chart = 'daily')", [],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)))?;
println!("Spotify Denmark: {sp_tracks} songs that charted, {sp_read} histories read, {sp_rows} DK chart rows (daily from {})", sp_from.unwrap_or_default());
playlists::create_tables(db)?;
let (snaps, nmf_isrcs, nmf_dk, nmf_from): (i64, i64, i64, Option<String>) = db.conn.query_row(
"SELECT (SELECT count(DISTINCT playlist_id || taken_at) FROM playlist_snapshots), (SELECT count(*) FROM nmf_lookups),
(SELECT count(*) FROM nmf_additions WHERE country = 'Denmark'), (SELECT min(friday) FROM nmf_additions)", [],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)))?;
println!("Playlists: {snaps} snapshots; New Music Friday history: {nmf_isrcs} songs looked up, {nmf_dk} Denmark additions (since {})", nmf_from.unwrap_or_default());
musicbrainz::create_tables(db)?;
let (mb, mb_known): (i64, i64) = db.conn.query_row(
"SELECT count(*), count(*) FILTER (WHERE recordings > 0) FROM mb_isrc", [], |r| Ok((r.get(0)?, r.get(1)?)))?;
println!("MusicBrainz: {mb} ISRCs looked up, {mb_known} known (original release dates)");
println!("Deezer: {searched} songs looked up, {matched} matched; {tracks} tracks, {albums} releases ({labels} with a label)");
Ok(())
}