Rotation-list forecast (listcast): new songs on DR's next lists from releases, NMF, station fit and DR plays; walk-forward scorecard; Friday fetch timer; page reworked around it

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-20 22:57:33 +00:00
co-authored by Claude Opus 5.5
parent 5971fec9a2
commit 39c73ac332
7 changed files with 441 additions and 3 deletions
+2
View File
@@ -12,6 +12,8 @@ ExecStart=/root/dev/Airplay-Predictor/target/release/airplay collect --db /root/
ExecStartPost=/root/dev/Airplay-Predictor/target/release/airplay rotations --db /root/dev/Airplay-Predictor/data/airplay.db
ExecStartPost=/root/dev/Airplay-Predictor/target/release/airplay model calibrate --db /root/dev/Airplay-Predictor/data/airplay.db
ExecStartPost=/root/dev/Airplay-Predictor/target/release/airplay model rank --db /root/dev/Airplay-Predictor/data/airplay.db --top 0
# The rotation-list forecast for next Friday (and scoring of past forecasts).
ExecStartPost=/root/dev/Airplay-Predictor/target/release/airplay listcast --db /root/dev/Airplay-Predictor/data/airplay.db
Nice=10
IOSchedulingClass=idle
MemoryMax=1500M
+12
View File
@@ -0,0 +1,12 @@
[Unit]
Description=Airplay Predictor: fetch DR's rotation list (published on Fridays, often in the afternoon), re-forecast and score
After=network-online.target
Wants=network-online.target
[Service]
Type=oneshot
WorkingDirectory=/root/dev/Airplay-Predictor
ExecStart=/root/dev/Airplay-Predictor/target/release/airplay rotations --fetch --db /root/dev/Airplay-Predictor/data/airplay.db
ExecStartPost=/root/dev/Airplay-Predictor/target/release/airplay listcast --db /root/dev/Airplay-Predictor/data/airplay.db
Nice=10
MemoryMax=1G
+12
View File
@@ -0,0 +1,12 @@
[Unit]
Description=Fetch DR's rotation list on Friday afternoons and Saturday morning
[Timer]
OnCalendar=Fri *-*-* 13:00:00 Europe/Copenhagen
OnCalendar=Fri *-*-* 16:00:00 Europe/Copenhagen
OnCalendar=Fri *-*-* 19:00:00 Europe/Copenhagen
OnCalendar=Sat *-*-* 09:00:00 Europe/Copenhagen
Persistent=true
[Install]
WantedBy=timers.target
+284
View File
@@ -0,0 +1,284 @@
//! The rotation-list forecast: which songs will be NEW on DR's next rotation list (published on
//! Fridays, one list per channel: P3, P4, P5, P6 Beat, P8 Jazz), per song and channel.
//!
//! airplay listcast [--asof YYYY-MM-DD]
//!
//! What the lists showed (20 Nov 2026): of 12 new songs, the 9 by artists DR already played all had a
//! New Music Friday Denmark placement in the weeks before and 6 had a release of that exact song
//! 1-7 weeks earlier; DR lists a song as it starts playing it (0-3 plays the week before, except one).
//! So the candidates are recent releases (iTunes / Deezer, the last 10 weeks) by artists DR plays,
//! one row per song and channel, scored by a logistic regression fitted on every past list change we
//! can reconstruct (lists back to 2023; releases, NMF and DR's per-channel plays are dated):
//! release age, NMF Denmark placement and position, the artist listed on that channel before, the
//! artist's share of DR plays on that channel and recent level there, the song's own DR plays in the
//! last 7 days (where logged), Danish, channel.
//! Walk-forward over the past lists gives the scorecard (made_at 'backtest'); each new list is a new
//! training example and scores the forecast made before it.
use crate::db::Db;
use crate::model::glm::{self, Design};
use anyhow::Result;
use chrono::{Datelike, Duration, NaiveDate, Utc};
use rusqlite::params;
use std::collections::{BTreeMap, HashMap, HashSet};
const GROUPS: [&str; 5] = ["P3", "P4", "P5", "P6", "P8"];
const RELEASE_WINDOW: i64 = 70; // days before the list a release still counts as new
const NF: usize = 15;
pub const FEATURES: [&str; NF] = ["intercept", "weeks since release (/10)", "released within 3 weeks", "NMF Denmark placement of the song",
"NMF Denmark top 20", "artist listed on this channel in the past 2 years", "artist's share of DR plays on this channel (52 weeks)",
"log artist plays per week on this channel (8 weeks)", "log song plays on this channel (7 days)", "song-level plays logged",
"Danish artist", "P4", "P5", "P6", "P8"];
pub fn create_tables(db: &Db) -> Result<()> {
db.conn.execute_batch(
"CREATE TABLE IF NOT EXISTS list_forecasts (
made_at TEXT NOT NULL, asof TEXT NOT NULL, target_list TEXT NOT NULL, grp TEXT NOT NULL, song TEXT NOT NULL,
artist TEXT, title TEXT, artist_id TEXT, prob REAL NOT NULL, release_date TEXT, nmf_pos INTEGER,
plays7 INTEGER, listed_before INTEGER, share REAL,
PRIMARY KEY (made_at, grp, song)
);
CREATE TABLE IF NOT EXISTS list_scorecard (
target_list TEXT NOT NULL, made_at TEXT NOT NULL, grp TEXT NOT NULL,
new_songs INTEGER, in_candidates INTEGER, top5 INTEGER, top10 INTEGER, expected REAL, logloss REAL,
PRIMARY KEY (target_list, made_at, grp)
);",
)?;
// The first version had other columns; start over if so.
let has_old: i64 = db.conn.query_row("SELECT count(*) FROM pragma_table_info('list_forecasts') WHERE name = 'bucket'", [], |r| r.get(0))?;
if has_old > 0 { db.conn.execute_batch("DROP TABLE list_forecasts; DELETE FROM list_scorecard;")?; return create_tables(db); }
Ok(())
}
pub fn group(ch: &str) -> Option<&'static str> {
match ch { "p3" => Some("P3"), "p5" => Some("P5"), "p6beat" => Some("P6"), "p8jazz" => Some("P8"), c if c.starts_with("p4") => Some("P4"), _ => None }
}
fn alnum(s: &str) -> String { s.to_lowercase().chars().filter(|c| c.is_alphanumeric()).collect() }
/// Title key: no brackets, versions or features.
pub fn title_key(title: &str) -> String {
let mut t = title.to_lowercase();
for (o, c) in [('(', ')'), ('[', ']')] {
while let (Some(i), Some(j)) = (t.find(o), t.find(c)) { if j > i { t.replace_range(i..=j, " "); } else { break; } }
}
for cut in [" - ", " feat", " ft.", " ft "] { if let Some(i) = t.find(cut) { t.truncate(i); } }
alnum(&t)
}
/// The first credited artist.
pub fn artist_key(artist: &str) -> String {
let a = artist.to_lowercase();
let mut first = a.as_str();
for sep in [" [+] ", " feat", " ft.", " ft ", " & ", ", ", " x ", " og ", " med ", " vs"] { if let Some(i) = first.find(sep) { first = &first[..i]; } }
alnum(first)
}
pub fn song_key(artist: &str, title: &str) -> String { format!("{}|{}", title_key(title), artist_key(artist)) }
struct Data {
/// list date -> group -> songs; only complete lists
lists: BTreeMap<NaiveDate, HashMap<&'static str, HashSet<String>>>,
/// group -> artist key -> list dates
listed: HashMap<&'static str, HashMap<String, Vec<NaiveDate>>>,
/// DR artist id -> (name, key, danish)
artists: HashMap<String, (String, String, bool)>,
/// artist id -> (title, title key, earliest release date)
releases: HashMap<String, Vec<(String, String, NaiveDate)>>,
/// artist id -> (friday, position, title key)
nmf: HashMap<String, Vec<(NaiveDate, Option<i64>, String)>>,
/// (artist id, group) -> week -> plays (P4: the busiest region)
weekly: HashMap<(String, &'static str), BTreeMap<NaiveDate, f64>>,
/// (group, song key) -> play dates
song_plays: HashMap<(&'static str, String), Vec<NaiveDate>>,
plays_from: Option<NaiveDate>,
}
fn load(db: &Db) -> Result<Data> {
let c = &db.conn;
let mut lists: BTreeMap<NaiveDate, HashMap<&'static str, HashSet<String>>> = BTreeMap::new();
let mut listed: HashMap<&'static str, HashMap<String, Vec<NaiveDate>>> = HashMap::new();
for r in c.prepare("SELECT list_date, channel, artist, title FROM rotation_lists")?.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, String>(3)?)))? {
let (d, ch, a, t) = r?;
let (Ok(d), Some(g)) = (d.parse::<NaiveDate>(), group(&ch)) else { continue };
lists.entry(d).or_default().entry(g).or_default().insert(song_key(&a, &t));
listed.entry(g).or_default().entry(artist_key(&a)).or_default().push(d);
}
// A list is usable per channel when that channel has at least 15 songs (a few PDFs parsed partly).
for m in lists.values_mut() { m.retain(|_, s| s.len() >= 15); }
let mut artists = HashMap::new();
for r in c.prepare("SELECT artist_id, name, coalesce(json_extract(raw, '$.artist.country'), '') FROM dr_artists")?.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?)))? {
let (id, n, country) = r?;
artists.insert(id, (n.clone(), artist_key(&n), country == "Danmark"));
}
let mut rel: HashMap<String, HashMap<String, (String, NaiveDate)>> = HashMap::new();
let mut add = |id: String, t: String, d: String| {
let Ok(d) = d[..10.min(d.len())].parse::<NaiveDate>() else { return };
let t = t.replace(" - Single", "").replace(" - EP", "");
let e = rel.entry(id).or_default().entry(title_key(&t)).or_insert((t, d));
if d < e.1 { e.1 = d; }
};
for r in c.prepare("SELECT m.dr_artist_id, r.name, r.release_date FROM itunes_releases r JOIN itunes_artist_map m ON m.itunes_artist_id = r.itunes_artist_id WHERE r.release_date >= '2022'")?
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?)))? { let (a, t, d) = r?; add(a, t, d); }
for r in c.prepare("SELECT d.dr_artist_id, r.title, r.release_date FROM deezer_releases r JOIN deezer_artists d ON d.deezer_artist_id = r.deezer_artist_id WHERE r.release_date >= '2022' AND coalesce(r.record_type, '') != 'compile'")?
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?)))? { let (a, t, d) = r?; add(a, t, d); }
let releases = rel.into_iter().map(|(a, m)| (a, m.into_iter().map(|(k, (t, d))| (t, k, d)).collect())).collect();
let mut nmf: HashMap<String, Vec<(NaiveDate, Option<i64>, String)>> = HashMap::new();
for r in c.prepare("SELECT l.dr_artist_id, n.friday, n.position, t.title FROM nmf_additions n JOIN nmf_artist_tracks t ON t.isrc = n.isrc AND t.friday = n.friday
JOIN nmf_artist_lookups l ON l.spotify_artist_id = t.spotify_artist_id WHERE lower(n.country) = 'denmark'")?
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, Option<i64>>(2)?, r.get::<_, Option<String>>(3)?)))? {
let (a, f, pos, t) = r?;
if let Ok(f) = f.parse::<NaiveDate>() { nmf.entry(a).or_default().push((f, pos, title_key(&t.unwrap_or_default()))); }
}
let mut weekly: HashMap<(String, &'static str), BTreeMap<NaiveDate, f64>> = HashMap::new();
for q in ["SELECT artist_id, week_start, channel, plays FROM dr_artist_weeks", "SELECT artist_id, week_start, channel, plays FROM dr_artist_weeks_arch"] {
let Ok(mut st) = c.prepare(q) else { continue };
for r in st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, f64>(3)?)))? {
let (a, w, ch, p) = r?;
let (Some(g), Ok(w)) = (group(&ch), w.parse::<NaiveDate>()) else { continue };
let e = weekly.entry((a, g)).or_default().entry(w).or_insert(0.0);
*e = e.max(p);
}
}
let mut song_plays: HashMap<(&'static str, String), Vec<NaiveDate>> = HashMap::new();
let mut plays_from: Option<NaiveDate> = None;
for r in c.prepare("SELECT station, artist, title, substr(played_at, 1, 10) FROM plays WHERE source = 'dr'")?
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, String>(3)?)))? {
let (st, a, t, d) = r?;
let (Some(g), Ok(d)) = (group(&st), d.parse::<NaiveDate>()) else { continue };
plays_from = Some(plays_from.map_or(d, |f| f.min(d)));
song_plays.entry((g, song_key(&a, &t))).or_default().push(d);
}
Ok(Data { lists, listed, artists, releases, nmf, weekly, song_plays, plays_from })
}
struct Row { grp: &'static str, song: String, artist_id: String, artist: String, title: String, release: NaiveDate, nmf_pos: Option<i64>, plays7: i64, listed_before: bool, share: f64, x: [f64; NF] }
/// Candidates for the list on `d` (everything known before it): releases in the 70 days before by
/// artists DR plays, per channel, not on that channel's previous list.
fn candidates(data: &Data, d: NaiveDate) -> Vec<Row> {
let prev: HashMap<&'static str, HashSet<String>> = data.lists.range(..d).next_back().map(|(_, m)| m.clone()).unwrap_or_default();
let logged = data.plays_from.is_some_and(|f| d >= f + Duration::days(7));
let mut out = Vec::new();
for (aid, rels) in &data.releases {
let Some((name, akey, danish)) = data.artists.get(aid) else { continue };
let recent: Vec<&(String, String, NaiveDate)> = rels.iter().filter(|(_, _, r)| *r < d && (d - *r).num_days() <= RELEASE_WINDOW).collect();
if recent.is_empty() { continue; }
// The artist's DR plays per channel: share over 52 weeks, weekly level over 8 weeks (before d).
let tot = |g: &str, days: i64| data.weekly.get(&(aid.clone(), GROUPS.iter().find(|x| **x == g).copied().unwrap_or("P3")))
.map(|m| m.range(d - Duration::days(days)..d - Duration::days(6)).map(|(_, v)| *v).sum::<f64>()).unwrap_or(0.0);
let year: Vec<f64> = GROUPS.iter().map(|g| tot(g, 364)).collect();
let all: f64 = year.iter().sum();
for (t, tk, rdate) in recent {
let song = format!("{tk}|{akey}");
let placed: Vec<&(NaiveDate, Option<i64>, String)> = data.nmf.get(aid).map(|v| v.iter().filter(|(f, _, ntk)| *f < d && (ntk == tk || ((*f - *rdate).num_days().abs() <= 10))).collect()).unwrap_or_default();
let best_pos = placed.iter().filter_map(|x| x.1).min();
for (gi, g) in GROUPS.iter().enumerate() {
if prev.get(g).is_some_and(|s| s.contains(&song)) { continue; }
let listed_before = data.listed.get(g).and_then(|m| m.get(akey)).is_some_and(|ds| ds.iter().any(|x| *x < d && (d - *x).num_days() <= 730));
let share = if all > 0.0 { year[gi] / all } else { 0.0 };
let level = tot(g, 56) / 8.0;
let plays7 = data.song_plays.get(&(*g, song.clone())).map(|v| v.iter().filter(|x| **x < d && (d - **x).num_days() <= 7).count()).unwrap_or(0) as i64;
let wk = (d - *rdate).num_days() as f64 / 7.0;
let mut x = [0.0; NF];
x[0] = 1.0; x[1] = wk / 10.0; x[2] = (wk <= 3.0) as i32 as f64; x[3] = (!placed.is_empty()) as i32 as f64;
x[4] = best_pos.is_some_and(|p| p <= 20) as i32 as f64; x[5] = listed_before as i32 as f64; x[6] = share;
x[7] = level.ln_1p(); x[8] = (plays7 as f64).ln_1p(); x[9] = logged as i32 as f64; x[10] = *danish as i32 as f64;
if gi > 0 { x[10 + gi] = 1.0; }
out.push(Row { grp: g, song: song.clone(), artist_id: aid.clone(), artist: name.clone(), title: t.clone(), release: *rdate, nmf_pos: best_pos, plays7, listed_before, share, x });
}
}
}
out
}
/// Labels: new on the list at d (and on that channel's list both times, so the channel was parsed).
fn labelled(data: &Data, d: NaiveDate) -> Option<Vec<(Row, bool)>> {
let now = data.lists.get(&d)?;
let (_, prev) = data.lists.range(..d).next_back()?;
Some(candidates(data, d).into_iter().filter(|r| now.contains_key(r.grp) && prev.contains_key(r.grp))
.map(|r| { let y = now.get(r.grp).is_some_and(|s| s.contains(&r.song)); (r, y) }).collect())
}
fn fit(rows: &[(Row, bool)]) -> Vec<f64> {
let mut des = Design::new(NF);
for (r, y) in rows { des.push(&r.x, *y as i32 as f64, 1.0); }
glm::logit(&des, 2.0)
}
fn prob(b: &[f64], r: &Row) -> f64 { glm::sigmoid(glm::dot(b, &r.x)).clamp(1e-5, 1.0 - 1e-5) }
pub fn run(db: &Db, args: &[String]) -> Result<()> {
create_tables(db)?;
let asof: NaiveDate = args.iter().position(|a| a == "--asof").and_then(|i| args.get(i + 1)).and_then(|s| s.parse().ok())
.unwrap_or_else(|| Utc::now().date_naive() + Duration::days(1));
let data = load(db)?;
// Every past list we can label, oldest first.
let past: Vec<(NaiveDate, Vec<(Row, bool)>)> = data.lists.keys().filter(|d| **d < asof).filter_map(|d| labelled(&data, *d).map(|v| (*d, v))).collect();
// Walk-forward scorecard: each list from the 6th scored by a model fitted on the lists before it.
let tx = db.conn.unchecked_transaction()?;
tx.execute("DELETE FROM list_scorecard WHERE made_at = 'backtest'", [])?;
for i in 5..past.len() {
let train: Vec<(Row, bool)> = past[..i].iter().flat_map(|(_, v)| v.iter().map(|(r, y)| (Row { x: r.x, grp: r.grp, song: String::new(), artist_id: String::new(), artist: String::new(), title: String::new(), release: r.release, nmf_pos: None, plays7: 0, listed_before: false, share: 0.0 }, *y))).collect();
let b = fit(&train);
let (d, rows) = &past[i];
let new_by_group = new_songs(&data, *d);
for g in GROUPS {
let Some(new) = new_by_group.get(g) else { continue };
let mut mine: Vec<(f64, bool)> = rows.iter().filter(|(r, _)| r.grp == g).map(|(r, y)| (prob(&b, r), *y)).collect();
mine.sort_by(|a, b| b.0.total_cmp(&a.0));
let hit = |k: usize| mine.iter().take(k).filter(|x| x.1).count() as i64;
let ll = mine.iter().map(|(p, y)| if *y { -p.ln() } else { -(1.0 - p).ln() }).sum::<f64>() / mine.len().max(1) as f64;
tx.execute("INSERT OR REPLACE INTO list_scorecard VALUES (?1, 'backtest', ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
params![d.to_string(), g, new.len() as i64, mine.iter().filter(|x| x.1).count() as i64, hit(5), hit(10), mine.iter().map(|x| x.0).sum::<f64>(), ll])?;
}
}
tx.commit()?;
// The forecast for the next list: fitted on every past list.
let all: Vec<(Row, bool)> = past.into_iter().flat_map(|(_, v)| v).collect();
let b = fit(&all);
let latest = data.lists.range(..asof).next_back().map(|(d, _)| *d).unwrap_or(asof);
let mut target = latest + Duration::days(1);
while target.weekday() != chrono::Weekday::Fri { target += Duration::days(1); }
let made = if args.iter().any(|a| a == "--asof") { format!("{asof}T00:00:00Z") } else { Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string() };
let cands = candidates(&data, asof.min(target));
let tx = db.conn.unchecked_transaction()?;
for r in &cands {
tx.execute("INSERT OR REPLACE INTO list_forecasts VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)",
params![made, asof.to_string(), target.to_string(), r.grp, r.song, r.artist, r.title, r.artist_id, prob(&b, r), r.release.to_string(), r.nmf_pos, r.plays7, r.listed_before as i64, r.share])?;
}
tx.execute("CREATE TABLE IF NOT EXISTS list_model (made_at TEXT NOT NULL, feature TEXT NOT NULL, beta REAL, PRIMARY KEY (made_at, feature))", [])?;
for (f, v) in FEATURES.iter().zip(&b) { tx.execute("INSERT OR REPLACE INTO list_model VALUES (?1, ?2, ?3)", params![made, f, v])?; }
tx.commit()?;
println!("list forecast for {target} (as of {asof}): {} candidates from {} labelled past lists ({} rows, {} new)", cands.len(),
db.conn.query_row("SELECT count(DISTINCT target_list) FROM list_scorecard WHERE made_at = 'backtest'", [], |r| r.get::<_, i64>(0))? + 5, all.len(), all.iter().filter(|x| x.1).count());
for (f, v) in FEATURES.iter().zip(&b) { println!(" {f:55} {v:+.3}"); }
score_forecasts(db, &data)?;
Ok(())
}
/// The songs new on the list at d per channel (vs the previous list).
fn new_songs(data: &Data, d: NaiveDate) -> HashMap<&'static str, HashSet<String>> {
let (Some(now), Some((_, prev))) = (data.lists.get(&d), data.lists.range(..d).next_back()) else { return HashMap::new() };
now.iter().filter(|(g, _)| prev.contains_key(*g)).map(|(g, s)| (*g, s.iter().filter(|x| !prev[g].contains(*x)).cloned().collect())).collect()
}
/// Scores the stored forecasts (the latest made before each list) once their list has arrived.
fn score_forecasts(db: &Db, data: &Data) -> Result<()> {
let targets: Vec<(String, String)> = db.conn.prepare(
"SELECT target_list, max(made_at) FROM list_forecasts WHERE made_at < target_list || 'T12:00:00Z' GROUP BY target_list")?
.query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?.collect::<Result<_, _>>()?;
for (target, made) in targets {
let Ok(d) = target.parse::<NaiveDate>() else { continue };
let new_by_group = new_songs(data, d);
if new_by_group.is_empty() { continue; }
let rows: Vec<(String, String, f64)> = db.conn.prepare("SELECT grp, song, prob FROM list_forecasts WHERE made_at = ?1 AND target_list = ?2")?
.query_map(params![made, target], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?.collect::<Result<_, _>>()?;
for (g, new) in &new_by_group {
let mut mine: Vec<&(String, String, f64)> = rows.iter().filter(|r| r.0 == *g).collect();
mine.sort_by(|a, b| b.2.total_cmp(&a.2));
let hit = |k: usize| mine.iter().take(k).filter(|r| new.contains(&r.1)).count() as i64;
let ll = mine.iter().map(|r| { let p = r.2.clamp(1e-5, 1.0 - 1e-5); if new.contains(&r.1) { -p.ln() } else { -(1.0 - p).ln() } }).sum::<f64>() / mine.len().max(1) as f64;
db.conn.execute("INSERT OR REPLACE INTO list_scorecard VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)",
params![target, made, g, new.len() as i64, mine.iter().filter(|r| new.contains(&r.1)).count() as i64, hit(5), hit(10), mine.iter().map(|r| r.2).sum::<f64>(), ll])?;
}
}
Ok(())
}
+10
View File
@@ -29,6 +29,7 @@ mod rotations;
mod spotify;
mod spotify_api;
mod itunes;
pub mod listcast;
mod web;
use anyhow::Result;
@@ -54,8 +55,17 @@ fn main() -> Result<()> {
Some("serve") => web::serve(&db_path, &args),
Some("export") => export::run(&db, &args),
Some("pdfdump") => rotations::dump(std::path::Path::new(args.get(2).map(String::as_str).unwrap_or(""))),
Some("listcast") => listcast::run(&db, &args),
Some("rotations") => {
let dir = std::path::Path::new(&db_path).parent().unwrap_or(std::path::Path::new(".")).join("rotations");
if args.iter().any(|a| a == "--fetch") {
// The current list from dr.dk (DR publishes on Fridays, sometimes late in the day).
match dr::collect_rotation_pdf(&db, &http::Http::new(400)?, &dir) {
Ok(Some(f)) => println!("rotation list saved: {f}"),
Ok(None) => println!("rotation list unchanged"),
Err(e) => eprintln!("rotation list: {e}"),
}
}
if args.iter().any(|a| a == "--wayback") {
let mut log = Vec::new();
let n = rotations::wayback(&db, &http::Http::new(1500)?, &dir, &mut log)?;
+59
View File
@@ -183,10 +183,69 @@ fn handle(st: &State, method: &Method, url: &str) -> Result<Reply> {
"/api/artist" => artist(st, q.get("id").map(String::as_str).unwrap_or("")),
"/api/whatif" => whatif(st, &q),
"/api/nextweek" => nextweek(st),
"/api/listcast" => listcast(st),
_ => Ok((404, "text/plain", Arc::new("not found".into()))),
}
}
/// The rotation-list forecast (`airplay listcast`): the latest forecast for the next list, the
/// latest list's new songs with where the forecast before it had them, and the scorecard.
fn listcast(st: &State) -> Result<Reply> {
let c = ro(&st.db_path)?;
let has: i64 = c.query_row("SELECT count(*) FROM sqlite_master WHERE name = 'list_forecasts'", [], |r| r.get(0))?;
if has == 0 { return ok(json!({"ready": false})) }
// The live forecast: the newest made by the daily run (not an --asof backfill).
let Some((made, target)): Option<(String, String)> = c.query_row(
"SELECT made_at, target_list FROM list_forecasts WHERE made_at NOT LIKE '%T00:00:00Z' ORDER BY made_at DESC LIMIT 1", [], |r| Ok((r.get(0)?, r.get(1)?))).optional()? else { return ok(json!({"ready": false})) };
let mut groups = serde_json::Map::new();
let mut expected = serde_json::Map::new();
for g in ["P3", "P4", "P5", "P6", "P8"] {
let rows: Vec<Value> = c.prepare("SELECT artist, title, prob, release_date, nmf_pos, plays7, listed_before, artist_id FROM list_forecasts WHERE made_at = ?1 AND grp = ?2 ORDER BY prob DESC LIMIT 15")?
.query_map(params![made, g], |r| Ok(json!({"artist": r.get::<_, String>(0)?, "title": r.get::<_, String>(1)?, "p": r.get::<_, f64>(2)?, "release": r.get::<_, Option<String>>(3)?,
"nmf_pos": r.get::<_, Option<i64>>(4)?, "plays7": r.get::<_, i64>(5)?, "listed_before": r.get::<_, i64>(6)? == 1, "id": r.get::<_, Option<String>>(7)?})))?.collect::<Result<_, _>>()?;
groups.insert(g.into(), json!(rows));
let e: f64 = c.query_row("SELECT coalesce(sum(prob), 0) FROM list_forecasts WHERE made_at = ?1 AND grp = ?2", params![made, g], |r| r.get(0))?;
expected.insert(g.into(), json!(e));
}
// The latest list and the one before: what was new, and where our forecast before it had it.
let lists: Vec<String> = c.prepare("SELECT DISTINCT list_date FROM rotation_lists ORDER BY list_date DESC LIMIT 2")?.query_map([], |r| r.get(0))?.collect::<Result<_, _>>()?;
let mut last = json!(null);
if lists.len() == 2 {
let (d, p) = (&lists[0], &lists[1]);
let fc: Option<String> = c.query_row("SELECT max(made_at) FROM list_forecasts WHERE target_list = ?1 AND made_at < ?1 || 'T12:00:00Z'", [d], |r| r.get(0)).optional()?.flatten();
let mut rank: HashMap<(String, String), (usize, f64)> = HashMap::new();
if let Some(fc) = &fc {
let mut by: HashMap<String, Vec<(String, f64)>> = HashMap::new();
for r in c.prepare("SELECT grp, song, prob FROM list_forecasts WHERE made_at = ?1")?.query_map([fc], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, f64>(2)?)))? {
let (g, s, pr) = r?; by.entry(g).or_default().push((s, pr));
}
for (g, mut v) in by { v.sort_by(|a, b| b.1.total_cmp(&a.1)); for (i, (s, pr)) in v.into_iter().enumerate() { rank.insert((g.clone(), s), (i + 1, pr)); } }
}
let prev: std::collections::HashSet<(String, String)> = c.prepare("SELECT channel, artist, title FROM rotation_lists WHERE list_date = ?1")?
.query_map([p], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?)))?
.filter_map(|r| r.ok()).filter_map(|(ch, a, t)| Some((crate::listcast::group(&ch)?.to_string(), crate::listcast::song_key(&a, &t)))).collect();
let new: Vec<Value> = c.prepare("SELECT channel, artist, title, category FROM rotation_lists WHERE list_date = ?1")?
.query_map([d], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, String>(3)?)))?
.filter_map(|r| r.ok()).filter_map(|(ch, a, t, cat)| {
let g = crate::listcast::group(&ch)?.to_string();
let k = crate::listcast::song_key(&a, &t);
if prev.contains(&(g.clone(), k.clone())) { return None; }
let rk = rank.get(&(g.clone(), k));
Some(json!({"grp": g, "artist": a, "title": t, "category": cat, "rank": rk.map(|x| x.0), "p": rk.map(|x| x.1)}))
}).collect();
last = json!({"date": d, "previous": p, "forecast": fc, "new": new});
}
// Scorecard: walk-forward over past lists (precision and recall at 5 and 10 per channel) and the live forecasts.
let bt: (i64, i64, i64, i64, i64) = c.query_row("SELECT count(DISTINCT target_list), coalesce(sum(new_songs), 0), coalesce(sum(in_candidates), 0), coalesce(sum(top5), 0), coalesce(sum(top10), 0) FROM list_scorecard WHERE made_at = 'backtest'", [],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)))?;
let live: Vec<Value> = c.prepare("SELECT target_list, made_at, sum(new_songs), sum(in_candidates), sum(top5), sum(top10) FROM list_scorecard WHERE made_at != 'backtest' GROUP BY target_list, made_at ORDER BY target_list DESC")?
.query_map([], |r| Ok(json!({"list": r.get::<_, String>(0)?, "made_at": r.get::<_, String>(1)?, "new": r.get::<_, i64>(2)?, "candidates": r.get::<_, i64>(3)?, "top5": r.get::<_, i64>(4)?, "top10": r.get::<_, i64>(5)?})))?
.collect::<Result<_, _>>()?;
let model: Vec<Value> = c.prepare("SELECT feature, beta FROM list_model WHERE made_at = ?1")?.query_map([&made], |r| Ok(json!({"feature": r.get::<_, String>(0)?, "beta": r.get::<_, f64>(1)?})))?.collect::<Result<_, _>>().unwrap_or_default();
ok(json!({"ready": true, "made_at": made, "target": target, "groups": groups, "expected": expected, "last": last,
"backtest": {"lists": bt.0, "new": bt.1, "candidates": bt.2, "top5": bt.3, "top10": bt.4}, "live": live, "model": model}))
}
/// The station group a channel belongs to, as shown on the page.
fn group_label(ch: &str) -> &'static str {
match ch { "p3" => "P3", "p5" => "P5", "p6beat" => "P6", "p8jazz" => "P8", c if c.starts_with("p4") => "P4", _ => "" }
+62 -3
View File
@@ -143,9 +143,25 @@ svg text { fill: var(--muted); font-size: 11px; }
.meter i { position: absolute; left: 0; top: 0; bottom: 0; border-radius: 4px; background: var(--c); }
.meter u { position: absolute; top: -2px; bottom: -2px; width: 2px; background: var(--ink); }
.tag2 { font-size: 11px; font-weight: 700; text-transform: uppercase; letter-spacing: .05em; color: var(--muted); border: 1px solid var(--line); border-radius: 6px; padding: 1px 6px; }
.lc-board { display: grid; grid-template-columns: repeat(5, 1fr); gap: 12px; }
.lc-board .col { background: var(--panel); border: 1px solid var(--line); border-radius: 12px; padding: 12px 14px; box-shadow: var(--shadow); border-top: 4px solid var(--c); }
.lc-board h4 { margin: 0 0 2px; font-size: 15px; color: var(--c); display: flex; justify-content: space-between; }
.lc-board h4 small { color: var(--muted); font-weight: 400; font-size: 12px; }
.song { padding: 8px 0; border-bottom: 1px solid var(--line); }
.song:last-child { border-bottom: 0; }
.song .t { display: grid; grid-template-columns: 1fr auto; gap: 6px; align-items: baseline; }
.song .t span { font-weight: 600; font-size: 14px; line-height: 1.25; }
.song .t span small { display: block; font-weight: 400; color: var(--muted); font-size: 12px; }
.song .t b { font-variant-numeric: tabular-nums; }
.song .ev { display: flex; flex-wrap: wrap; gap: 4px; margin-top: 4px; }
.song .ev i { font-style: normal; font-size: 11px; padding: 1px 6px; border-radius: 5px; background: var(--bg); color: var(--muted); }
.song .ev i.hot { background: color-mix(in srgb, var(--c) 16%, var(--panel)); color: var(--ink); }
.lastfri table td, .lastfri table th { padding: 6px 10px; }
details.momentum { margin-top: 30px; border-top: 1px solid var(--line); padding-top: 12px; }
details.momentum summary { cursor: pointer; color: var(--muted); font-weight: 600; }
.track { display: grid; grid-template-columns: 340px 1fr; gap: 18px; align-items: start; }
@media (max-width: 900px) { .nw-hero, .track { grid-template-columns: 1fr; } .board { grid-template-columns: repeat(2, 1fr); } .nw-kpis { grid-template-columns: 1fr 1fr 1fr; } }
@media (max-width: 640px) { .board { grid-template-columns: 1fr; } .nw-kpis { grid-template-columns: 1fr; } .cand { grid-template-columns: 1fr; } }
@media (max-width: 900px) { .nw-hero, .track { grid-template-columns: 1fr; } .board, .lc-board { grid-template-columns: repeat(2, 1fr); } .nw-kpis { grid-template-columns: 1fr 1fr 1fr; } }
@media (max-width: 640px) { .board, .lc-board { grid-template-columns: 1fr; } .nw-kpis { grid-template-columns: 1fr; } .cand { grid-template-columns: 1fr; } }
@media (max-width: 640px) {
header, main { padding-left: 16px; padding-right: 16px; }
input[type=range] { width: 100%; }
@@ -208,7 +224,8 @@ svg text { fill: var(--muted); font-size: 11px; }
<p class="note"><b>New add</b> = if the artist releases a new song in the chosen week, the chance DR adds it within the chosen number of weeks: the artist enters rotation (at or above the station's weekly play threshold) after at least 4 weeks out, or, for artists who are always on, their plays jump well above their own baseline and stay up (that finds 77% of the songs DR's real lists show as new). Learned from past releases with no other release in the 6 weeks before, so a current song's success isn't counted as a new one's; a release less than 6 weeks after the artist's last one gets no estimate ("too soon"). The new song's NMF Denmark placement is weighed by how often the artist's releases were placed. "In rotation now" lists DR stations where the artist is on the list now, then Bauer stations (grey) with 14+ plays in the last 7 days. Click an artist for history and a what-if, or open <b>Best week</b> for their release calendar.</p>
</section>
<section id="tab-next" hidden><div id="nw"><p class="empty">Loading…</p></div></section>
<section id="tab-next" hidden><div id="lc"><p class="empty">Loading…</p></div>
<details class="momentum"><summary>Artist momentum: whose DR plays are about to jump (the earlier view)</summary><div id="nw"><p class="empty">Loading…</p></div></details></section>
<section id="tab-best" hidden>
<div class="controls">
<div class="ctl">Artist <input type="search" id="bw-q" list="bw-list" placeholder="Type an artist…" autocomplete="off"><datalist id="bw-list"></datalist></div>
@@ -505,7 +522,49 @@ const certainty = p => { const m = p / nwBase; return m >= 25 ? [0, 'Top pick']
const oneIn = p => p >= 0.5 ? `${Math.round(p * 10)} in 10` : `1 in ${Math.max(2, Math.round(1 / p))}`;
const fmtLong = s => new Date(s + 'T00:00:00Z').toLocaleDateString('en-GB', { day: 'numeric', month: 'long', timeZone: 'UTC' });
const plusDays = (s, d) => { const x = new Date(s + 'T00:00:00Z'); x.setUTCDate(x.getUTCDate() + d); return x.toISOString().slice(0, 10); };
// ---------- the rotation-list forecast ----------
let lc = null;
async function loadList() {
if (!lc) lc = await (await fetch(`${BASE}/api/listcast`)).json();
const d = lc, G = ['P3', 'P4', 'P5', 'P6', 'P8'];
if (!d || !d.ready) { $('lc').innerHTML = '<p class="empty">The list forecast is made by the daily run; it is not there yet.</p>'; return; }
const b = d.backtest, slots = b.lists * 5;
const prec10 = slots ? b.top10 / (slots * 10) : null, rec10 = b.new ? b.top10 / b.new : null;
const exp = G.reduce((s, g) => s + (d.expected[g] || 0), 0);
const weeks = r => { if (!r) return null; const w = Math.round((new Date(d.made_at) - new Date(r + 'T00:00:00Z')) / 6048e5); return w <= 0 ? 'released this week' : `released ${w} week${w > 1 ? 's' : ''} ago`; };
const songCard = (x, g) => `<div class="song"><div class="t"><span>${esc(x.title)}<small>${esc(x.artist)}</small></span><b>${pct(x.p)}</b></div>
<div class="ev">${x.nmf_pos ? `<i class="hot">NMF #${x.nmf_pos}</i>` : ''}${x.release ? `<i>${weeks(x.release)}</i>` : ''}${x.plays7 ? `<i class="hot">${x.plays7} DR play${x.plays7 > 1 ? 's' : ''} this week</i>` : ''}${x.listed_before ? `<i>listed on ${g} before</i>` : ''}</div></div>`;
const last = d.last;
const newRows = last ? last.new.slice().sort((a, b) => (a.rank || 1e9) - (b.rank || 1e9)) : [];
const caught = newRows.filter(x => x.rank && x.rank <= 10).length;
const lv = (d.live || [])[0];
$('lc').innerHTML = `
<div class="nw-hero">
<div><div class="eyebrow">DR rotation list forecast</div>
<h2>New on DR's rotation list, Friday ${fmtLong(d.target)}</h2>
<p class="note">Every Friday DR publishes which songs are in rotation on P3, P4, P5, P6 Beat and P8 Jazz. These are the songs we think will be <b>new</b> on next Friday's lists, from recent releases by artists DR plays: their New Music Friday Denmark placement, whether DR has listed the artist on that channel before, the artist's fit and current plays there, and the song's own DR plays this week. Forecast made ${fmtLong(d.made_at.slice(0, 10))}.</p></div>
<div class="nw-kpis">
<div class="kpi"><div>New songs expected</div><b>${Math.round(exp)}</b><small>across the five lists, among songs we can see coming</small></div>
<div class="kpi"><div>Hit rate</div><b>${prec10 != null ? (prec10 * 100).toFixed(0) + '%' : '—'}</b><small>of our top 10 per channel got listed (${b.lists} past lists)</small></div>
<div class="kpi"><div>Last Friday</div><b>${last ? `${caught} of ${newRows.length}` : '—'}</b><small>${last ? `new songs on the ${fmtLong(last.date)} lists were in our top 10` : ''}</small></div>
</div>
</div>
<div class="nw-h">Most likely new songs per list</div>
<div class="lc-board">${G.map(g => `<div class="col" style="--c:${GROUP_C[g]}"><h4>${g}<small>~${(d.expected[g] || 0).toFixed(1)} new</small></h4>${(d.groups[g] || []).slice(0, 8).map(x => songCard(x, g)).join('') || '<p class="note">No candidates</p>'}</div>`).join('')}</div>
${last ? `<div class="nw-h">Last Friday: what was new on DR's lists (${fmtLong(last.date)}) and where we had it</div>
<div class="tablewrap lastfri"><table><thead><tr><th class="l">List</th><th class="l">Song</th><th class="l">Artist</th><th>Our rank before the list</th><th>Our chance</th></tr></thead><tbody>
${newRows.map(x => `<tr><td class="l"><span class="chip" style="background:color-mix(in srgb, ${GROUP_C[x.grp]} 18%, var(--panel))">${x.grp}</span></td><td class="l">${esc(x.title)}</td><td class="l">${esc(x.artist)}</td>
<td>${x.rank ? (x.rank <= 10 ? `<b class="up">#${x.rank}</b>` : '#' + x.rank) : '<span class="note">not a candidate</span>'}</td><td>${x.p != null ? pct(x.p) : '—'}</td></tr>`).join('')}
</tbody></table></div>` : ''}
<div class="nw-h">How sure are we?</div>
<div class="prose note">
<p>Tested walk-forward on ${b.lists} past lists (2023 on): each list predicted by a model fitted only on the lists before it. Of the songs in our <b>top 10 per channel</b>, <b>${prec10 != null ? (prec10 * 100).toFixed(1) : '—'}%</b> were new on the next list, against roughly 0.1-0.2% for a random recent release. Of all ${b.new.toLocaleString('en')} newly listed songs we had ${(100 * b.candidates / Math.max(1, b.new)).toFixed(0)}% as candidates and ${rec10 != null ? (100 * rec10).toFixed(0) : '—'}% in a top 10.</p>
<p>What we cannot see yet: album tracks, songs missing from iTunes and Deezer, and artists DR has never played (3 of the 12 new songs on 20 November). DR's own plays of each song, logged since 8 November, close part of that gap as they build up; every Friday's list is added to the training data and scores the forecast made before it.</p>
${lv ? `<p>Forecasts made before their list: ${d.live.map(x => `${fmtLong(x.list)}: ${x.top10} of ${x.new} new songs in a top 10, ${x.candidates} among candidates`).join(' · ')}.</p>` : ''}
</div>`;
}
async function loadNext() {
loadList();
if (nw) return renderNext();
const r = await fetch(`${BASE}/api/nextweek`); nw = await r.json();
renderNext();