New-add model (add = entering rotation after 4+ weeks out; per channel and any DR channel, 8-week window), site ranks by it; DR rotation-list PDF parser (glyph positions, channel by play profile) and Wayback Machine backfill: 29 weekly lists 2023-2026

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-15 02:50:13 +00:00
co-authored by Claude Opus 5.5
parent 387502ebfa
commit c698925814
9 changed files with 1122 additions and 89 deletions
+13
View File
@@ -4,6 +4,7 @@
//! airplay stats [--db data/airplay.db]
//! airplay check [--db data/airplay.db] gaps, duplicates, DR cross-check, chart completeness
//! airplay model panel | backtest | rank the first models (src/model)
//! airplay rotations [--wayback] [--reparse] parse DR's rotation PDFs (src/rotations.rs)
//! airplay serve [--addr 127.0.0.1:8790] [--prefix /airplay] the playground site (src/web.rs)
//!
//! --backfill fetches all history at once (every DR artist page, the whole Hitlisten archive);
@@ -19,6 +20,7 @@ mod http;
mod model;
mod musicbrainz;
mod playlists;
mod rotations;
mod spotify;
mod web;
@@ -43,6 +45,17 @@ fn main() -> Result<()> {
Some("check") => check::run(&db),
Some("model") => model::run(&db, &args),
Some("serve") => web::serve(&db_path, &args),
Some("pdfdump") => rotations::dump(std::path::Path::new(args.get(2).map(String::as_str).unwrap_or(""))),
Some("rotations") => {
let dir = std::path::Path::new(&db_path).parent().unwrap_or(std::path::Path::new(".")).join("rotations");
if args.iter().any(|a| a == "--wayback") {
let mut log = Vec::new();
let n = rotations::wayback(&db, &http::Http::new(1500)?, &dir, &mut log)?;
println!("wayback: {n} PDFs saved");
for l in log { println!(" {l}"); }
}
rotations::store_all(&db, &dir, args.iter().any(|a| a == "--reparse"))
}
_ => {
eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|musicbrainz|spotify|playlists|nmf] [--backfill]\n airplay stats [--db FILE]\n airplay check [--db FILE]\n airplay model panel|backtest|rank\n airplay serve [--addr HOST:PORT] [--prefix /airplay]");
std::process::exit(2);
+308
View File
@@ -0,0 +1,308 @@
//! New adds: the chance that an artist gets a (new) song into rotation, rather than the chance
//! of being played at all.
//!
//! The DR artist pages count plays per artist, not per song, so a song-level add isn't visible.
//! The proxy used here:
//!
//! add on channel c in week w the artist is in rotation on c in w (plays >= tau_c) after at
//! least GAP weeks out of rotation on c
//! add on any DR channel the artist enters rotation on some channel after at least GAP
//! weeks out of rotation on every channel (closer to "a new song")
//!
//! The target is an add within the window of W weeks that starts h weeks after the origin
//! (weeks t+h+1 .. t+h+W): "if a window opened h weeks from now, would the artist get an add in it?"
//! An artist with a song in rotation now has to drop out first, which is how it usually works:
//! in the week of song-level plays we have, only 3-12% of the artists in rotation on a station
//! had two songs in rotation at once. Re-entries of an old song also count as adds; the parsed
//! rotation lists (song level) will separate them.
use super::common::{mix, Rng};
use super::features::{self, LAGS, P};
use super::glm::{self, Design};
use super::panel::{Panel, C};
use anyhow::Result;
use std::fmt::Write as _;
pub const GAP: usize = 4;
pub const W: usize = 8;
/// Per-channel rows: the panel features plus these.
pub const EXTRA: &[&str] = &["in rotation on channel now", "in rotation anywhere now", "channels in rotation now (/14)",
"run on channel = 1 week", "run = 2 weeks", "run = 3-5 weeks", "run >= 6 weeks",
"near threshold (>= half of tau, not in rotation)", "plays / tau (capped 3, /3)",
"weeks since in rotation on channel (/12)"];
pub const PC: usize = P + 10;
/// Artist-level ("any channel") rows.
pub const ANY_NAMES: &[&str] = &[
"intercept", "log DR plays t", "log DR plays 4w", "log DR plays 12w", "log max DR plays 12w",
"in rotation anywhere now", "channels in rotation now (/14)", "current run anywhere (/12)", "weeks since in rotation anywhere (/12)",
"Track Top-40 score", "Airplay Top-20 score", "chart entry recency 6w", "chart entry recency 26w",
"log Track Top-40 hits", "log Airplay Top-20 hits", "Danish", "group", "female", "career years (/50)", "debut unknown",
"Christmas in window",
];
pub const PA: usize = 21;
/// Positions of the chart covariates (for the what-if): (t40 score, air score, recency 6w, recency 26w).
pub const CHART_IDX_CHANNEL: [usize; 4] = [10, 11, 12, 13];
pub const CHART_IDX_ANY: [usize; 4] = [9, 10, 11, 12];
pub struct Rot {
nw: usize,
pub tau: [u32; C],
rot: Vec<bool>, // [a][c][w]
any: Vec<bool>, // [a][w]
}
impl Rot {
pub fn new(p: &Panel, tau: &[u32; C]) -> Rot {
let nw = p.w();
let mut rot = vec![false; p.a() * C * nw];
let mut any = vec![false; p.a() * nw];
for a in 0..p.a() { for c in 0..C { for w in 0..nw {
let r = p.y(a, c, w) >= tau[c] as f64;
rot[(a * C + c) * nw + w] = r;
if r { any[a * nw + w] = true; }
} } }
Rot { nw, tau: *tau, rot, any }
}
pub fn on(&self, a: usize, c: usize, w: usize) -> bool { self.rot[(a * C + c) * self.nw + w] }
pub fn any(&self, a: usize, w: usize) -> bool { self.any[a * self.nw + w] }
pub fn add(&self, a: usize, c: usize, w: usize) -> bool { w >= GAP && self.on(a, c, w) && (1..=GAP).all(|k| !self.on(a, c, w - k)) }
pub fn add_any(&self, a: usize, w: usize) -> bool { w >= GAP && self.any(a, w) && (1..=GAP).all(|k| !self.any(a, w - k)) }
/// An add in weeks t+h+1 ..= t+h+W (all inside the panel).
pub fn add_in_window(&self, a: usize, c: usize, t: usize, h: usize) -> bool { (t + h + 1..=t + h + W).any(|w| self.add(a, c, w)) }
pub fn add_any_in_window(&self, a: usize, t: usize, h: usize) -> bool { (t + h + 1..=t + h + W).any(|w| self.add_any(a, w)) }
}
fn tau_of(_p: &Panel, r: &Rot, _a: usize, c: usize, _t: usize) -> f64 { r.tau[c] as f64 }
fn run_and_since(flag: impl Fn(usize) -> bool, t: usize) -> (f64, f64) {
let run = (0..LAGS).take_while(|&i| flag(t - i)).count() as f64 / LAGS as f64;
let since = (0..LAGS).find(|&i| flag(t - i)).unwrap_or(LAGS) as f64 / LAGS as f64;
(run, since)
}
fn christmas_in_window(p: &Panel, t: usize, h: usize) -> bool { (t + h + 1..=t + h + W).any(|w| features::christmas(p, w)) }
pub fn row_channel(p: &Panel, r: &Rot, a: usize, c: usize, t: usize, h: usize, out: &mut [f64]) {
features::row(p, a, c, t, t + h + 1, &mut out[..P]);
out[21] = christmas_in_window(p, t, h) as i32 as f64;
let n_on = (0..C).filter(|&k| r.on(a, k, t)).count() as f64;
let (run, since) = run_and_since(|w| r.on(a, c, w), t);
let weeks = (run * LAGS as f64).round() as usize;
let on = r.on(a, c, t);
let ratio = p.y(a, c, t) / tau_of(p, r, a, c, t);
let f = |b: bool| b as i32 as f64;
out[P..PC].copy_from_slice(&[f(on), f(r.any(a, t)), n_on / C as f64,
f(weeks == 1), f(weeks == 2), f((3..=5).contains(&weeks)), f(weeks >= 6),
f(!on && ratio >= 0.5), ratio.min(3.0) / 3.0, since]);
}
pub fn row_any(p: &Panel, r: &Rot, a: usize, t: usize, h: usize, out: &mut [f64]) {
let tot = |w: usize| (0..C).map(|c| p.y(a, c, w)).sum::<f64>();
let mean = |n: usize| (0..n).map(|i| tot(t - i)).sum::<f64>() / n as f64;
let max12 = (0..LAGS).map(|i| tot(t - i)).fold(0.0, f64::max);
let n_on = (0..C).filter(|&k| r.on(a, k, t)).count() as f64;
let (run, since) = run_and_since(|w| r.any(a, w), t);
let ch = p.chart(a, t);
let art = &p.artists[a];
let recency = |s: f64| if ch.weeks_since_entry == u16::MAX { 0.0 } else { (-(ch.weeks_since_entry as f64) / s).exp() };
let year = chrono::Datelike::year(&p.weeks[t]);
let v = [
1.0, tot(t).ln_1p(), mean(4).ln_1p(), mean(12).ln_1p(), max12.ln_1p(),
r.any(a, t) as i32 as f64, n_on / C as f64, run, since,
if ch.t40_rank > 0 { (41.0 - ch.t40_rank as f64) / 40.0 } else { 0.0 },
if ch.air_rank > 0 { (21.0 - ch.air_rank as f64) / 20.0 } else { 0.0 },
recency(6.0), recency(26.0), (ch.t40_hits as f64).ln_1p(), (ch.air_hits as f64).ln_1p(),
art.danish as i32 as f64, art.group as i32 as f64, art.female as i32 as f64,
art.debut.map(|d| ((year - d).clamp(0, 50)) as f64 / 50.0).unwrap_or(0.0), art.debut.is_none() as i32 as f64,
christmas_in_window(p, t, h) as i32 as f64,
];
out.copy_from_slice(&v);
}
/// Logit models for one h, trained on origins whose whole window is observed by `last_week`.
pub struct AddModel {
pub h: usize,
pub channel: Vec<f64>,
pub any: Vec<f64>,
}
fn origins(h: usize, last_week: usize) -> std::ops::RangeInclusive<usize> {
let hi = last_week.saturating_sub(h + W);
LAGS - 1..=hi
}
impl AddModel {
pub fn fit(p: &Panel, r: &Rot, h: usize, last_week: usize) -> AddModel {
let (mut dc, mut da) = (Design::new(PC), Design::new(PA));
let (mut x, mut xa) = (vec![0.0; PC], vec![0.0; PA]);
for t in origins(h, last_week) {
for a in 0..p.a() {
for c in 0..C {
// Channels where the artist hasn't been heard for 12 weeks: keep a quarter, weighted 4.
let quiet = (0..LAGS).all(|i| p.y(a, c, t - i) == 0.0);
let wgt = if quiet { if mix((a as u64) << 32 ^ (c as u64) << 20 ^ t as u64 ^ 0xadd) % 4 != 0 { continue; } 4.0 } else { 1.0 };
row_channel(p, r, a, c, t, h, &mut x);
dc.push(&x, r.add_in_window(a, c, t, h) as i32 as f64, wgt);
}
row_any(p, r, a, t, h, &mut xa);
da.push(&xa, r.add_any_in_window(a, t, h) as i32 as f64, 1.0);
}
}
AddModel { h, channel: glm::logit(&dc, 1.0), any: glm::logit(&da, 1.0) }
}
/// (P(add) per channel, P(add on any channel)) from origin t.
pub fn predict(&self, p: &Panel, r: &Rot, a: usize, t: usize, chart: Option<&dyn Fn(&mut [f64], [usize; 4])>) -> ([f64; C], f64) {
let mut x = vec![0.0; PC];
let mut pc = [0.0; C];
for c in 0..C {
row_channel(p, r, a, c, t, self.h, &mut x);
if let Some(f) = chart { f(&mut x, CHART_IDX_CHANNEL); }
pc[c] = glm::sigmoid(glm::dot(&self.channel, &x)).clamp(1e-6, 1.0 - 1e-6);
}
let mut xa = vec![0.0; PA];
row_any(p, r, a, t, self.h, &mut xa);
if let Some(f) = chart { f(&mut xa, CHART_IDX_ANY); }
(pc, glm::sigmoid(glm::dot(&self.any, &xa)).clamp(1e-6, 1.0 - 1e-6))
}
}
/// Sets the chart covariates at `idx` (t40 score, air score, recency 6w, recency 26w).
pub fn set_chart(x: &mut [f64], idx: [usize; 4], entry: Option<Option<f64>>, t40: Option<f64>, air: Option<f64>) {
if let Some(e) = entry {
let (r6, r26) = match e { Some(w) => ((-w / 6.0).exp(), (-w / 26.0).exp()), None => (0.0, 0.0) };
x[idx[2]] = r6; x[idx[3]] = r26;
}
if let Some(r) = t40 { x[idx[0]] = if (1.0..=40.0).contains(&r) { (41.0 - r) / 40.0 } else { 0.0 }; }
if let Some(r) = air { x[idx[1]] = if (1.0..=20.0).contains(&r) { (21.0 - r) / 20.0 } else { 0.0 }; }
}
// ---------------------------------------------------------------- backtest
fn ll(p: f64, y: bool) -> f64 { -(if y { p } else { 1.0 - p }).clamp(1e-9, 1.0).ln() }
fn average_precision(mut v: Vec<(f64, bool)>) -> f64 {
v.sort_by(|a, b| b.0.total_cmp(&a.0));
let (mut hits, mut sum) = (0.0, 0.0);
for (i, (_, y)) in v.iter().enumerate() { if *y { hits += 1.0; sum += hits / (i as f64 + 1.0); } }
if hits > 0.0 { sum / hits } else { f64::NAN }
}
fn skill(model: &[f64], base: &[f64]) -> (f64, f64, f64) {
let point = 1.0 - model.iter().sum::<f64>() / base.iter().sum::<f64>();
let n = model.len();
let block = 3.min(n);
let mut rng = Rng(7);
let mut d: Vec<f64> = (0..2000).map(|_| {
let (mut lm, mut lb, mut k) = (0.0, 0.0, 0);
while k < n { let s = (rng.u() * (n - block + 1) as f64) as usize; for i in s..s + block { if k < n { lm += model[i]; lb += base[i]; k += 1; } } }
1.0 - lm / lb
}).collect();
d.sort_by(f64::total_cmp);
(point, d[100], d[1899])
}
/// Walk-forward: at each origin t0 the models see only windows that had closed by t0.
pub fn backtest(p: &Panel, tau: &[u32; C], horizons: &[usize], start: usize, md: &mut String) -> Result<Vec<(String, usize, String, f64)>> {
let r = Rot::new(p, tau);
let mut rows = Vec::new();
writeln!(md, "# Backtest: new adds\n")?;
writeln!(md, "An add = the artist enters rotation (plays ≥ τ) after at least {GAP} weeks out of rotation: per channel, and on any DR channel after {GAP} weeks out of rotation everywhere. Target: an add within the {W} weeks that start h weeks after the forecast origin. Origins from week {}; each fit uses only windows closed by its origin.\n", start + 1)?;
for &h in horizons {
let last_origin = p.w() - 1 - h - W;
if last_origin < start { continue; }
// [model][week] log-loss sums; models: base(rot now), base(popularity bins), logit
let names = ["rate by in-rotation-now", "rate by last-12-weeks level", "logit"];
let mut wk_c: Vec<[f64; 3]> = Vec::new();
let mut wk_a: Vec<[f64; 3]> = Vec::new();
let mut pairs_c: [Vec<(f64, bool)>; 3] = Default::default();
let mut pairs_a: [Vec<(f64, bool)>; 3] = Default::default();
let mut calib = [[0.0f64; 3]; 10];
let (mut nc, mut na, mut hits_c, mut hits_a) = (0.0, 0.0, 0.0, 0.0);
for t0 in start..=last_origin {
let m = AddModel::fit(p, &r, h, t0);
// Baselines from the same closed windows.
let mut by_rot = [[1.0f64, 2.0]; 2];
let mut by_lvl = [[1.0f64, 2.0]; 8];
let mut any_rot = [[1.0f64, 2.0]; 2];
let mut any_lvl = [[1.0f64, 2.0]; 8];
let lvl = |m12: f64| ((m12.ln_1p() / 0.5) as usize).min(7);
for t in origins(h, t0) {
for a in 0..p.a() {
let mut tot = 0.0;
for c in 0..C {
let y = r.add_in_window(a, c, t, h) as i32 as f64;
let on = r.on(a, c, t) as usize;
by_rot[on][0] += y; by_rot[on][1] += 1.0;
let m12 = (0..LAGS).map(|i| p.y(a, c, t - i)).sum::<f64>() / LAGS as f64;
let b = lvl(m12); by_lvl[b][0] += y; by_lvl[b][1] += 1.0;
tot += m12;
}
let y = r.add_any_in_window(a, t, h) as i32 as f64;
let on = r.any(a, t) as usize;
any_rot[on][0] += y; any_rot[on][1] += 1.0;
let b = lvl(tot); any_lvl[b][0] += y; any_lvl[b][1] += 1.0;
}
}
let (mut lc, mut la) = ([0.0; 3], [0.0; 3]);
for a in 0..p.a() {
let (pc, pa) = m.predict(p, &r, a, t0, None);
let mut tot = 0.0;
for c in 0..C {
let y = r.add_in_window(a, c, t0, h);
let on = r.on(a, c, t0) as usize;
let m12 = (0..LAGS).map(|i| p.y(a, c, t0 - i)).sum::<f64>() / LAGS as f64;
tot += m12;
let preds = [by_rot[on][0] / by_rot[on][1], by_lvl[lvl(m12)][0] / by_lvl[lvl(m12)][1], pc[c]];
for k in 0..3 { lc[k] += ll(preds[k], y); pairs_c[k].push((preds[k], y)); }
let d = ((pc[c] * 10.0) as usize).min(9);
calib[d][0] += pc[c]; calib[d][1] += y as i32 as f64; calib[d][2] += 1.0;
nc += 1.0; hits_c += y as i32 as f64;
}
let y = r.add_any_in_window(a, t0, h);
let on = r.any(a, t0) as usize;
let preds = [any_rot[on][0] / any_rot[on][1], any_lvl[lvl(tot)][0] / any_lvl[lvl(tot)][1], pa];
for k in 0..3 { la[k] += ll(preds[k], y); pairs_a[k].push((preds[k], y)); }
na += 1.0; hits_a += y as i32 as f64;
}
wk_c.push(lc); wk_a.push(la);
eprintln!("adds backtest h={h}: origin {} done", p.weeks[t0]);
}
let nwk = wk_c.len();
writeln!(md, "## Window starting h = {h} weeks ahead ({nwk} forecast origins)\n")?;
writeln!(md, "Base rates: per channel {:.2}% of artist-channels get an add in the window; on any channel {:.1}% of artists.\n", 100.0 * hits_c / nc, 100.0 * hits_a / na)?;
writeln!(md, "| Model | Log-loss, per channel | PR-AUC, per channel | Log-loss, any channel | PR-AUC, any channel |\n|---|---|---|---|---|")?;
for k in 0..3 {
let (sc, sa) = (wk_c.iter().map(|w| w[k]).sum::<f64>() / nc, wk_a.iter().map(|w| w[k]).sum::<f64>() / na);
let (ac, aa) = (average_precision(pairs_c[k].clone()), average_precision(pairs_a[k].clone()));
for (metric, v) in [("add_logloss", sc), ("add_prauc", ac), ("addany_logloss", sa), ("addany_prauc", aa)] {
rows.push((names[k].to_string(), h, metric.to_string(), v));
}
writeln!(md, "| {} | {sc:.4} | {ac:.3} | {sa:.4} | {aa:.3} |", names[k])?;
}
let col = |v: &Vec<[f64; 3]>, k: usize| v.iter().map(|w| w[k]).collect::<Vec<f64>>();
let (s1, s2) = (skill(&col(&wk_c, 2), &col(&wk_c, 0)), skill(&col(&wk_c, 2), &col(&wk_c, 1)));
let (s3, s4) = (skill(&col(&wk_a, 2), &col(&wk_a, 0)), skill(&col(&wk_a, 2), &col(&wk_a, 1)));
rows.push(("logit".into(), h, "add_skill_vs_level".into(), s2.0));
rows.push(("logit".into(), h, "addany_skill_vs_level".into(), s4.0));
writeln!(md, "\nSkill of the logit (90% block-bootstrap interval over origins):\n")?;
writeln!(md, "| | vs in-rotation-now | vs last-12-weeks level |\n|---|---|---|")?;
writeln!(md, "| per channel | {:.3} ({:.3} to {:.3}) | {:.3} ({:.3} to {:.3}) |", s1.0, s1.1, s1.2, s2.0, s2.1, s2.2)?;
writeln!(md, "| any channel | {:.3} ({:.3} to {:.3}) | {:.3} ({:.3} to {:.3}) |\n", s3.0, s3.1, s3.2, s4.0, s4.1, s4.2)?;
writeln!(md, "Calibration, per channel:\n\n| Predicted | Rows | Mean predicted | Observed |\n|---|---|---|---|")?;
for (d, c) in calib.iter().enumerate() {
if c[2] > 0.0 { writeln!(md, "| {:.1}–{:.1} | {} | {:.3} | {:.3} |", d as f64 / 10.0, (d + 1) as f64 / 10.0, c[2], c[0] / c[2], c[1] / c[2])?; }
}
writeln!(md)?;
if nwk < 8 { writeln!(md, "Fewer than 8 forecast origins: no verdict at this horizon (doc rule).\n")?; }
}
let m = AddModel::fit(p, &r, 0, p.w() - 1);
writeln!(md, "## Coefficients, window starting now (h = 0), all closed windows\n\n| Any-channel feature | Coefficient |\n|---|---|")?;
for (n, b) in ANY_NAMES.iter().zip(&m.any) { writeln!(md, "| {n} | {b:+.3} |")?; }
writeln!(md, "\n| Per-channel feature | Coefficient |\n|---|---|")?;
for (n, b) in features::NAMES.iter().chain(EXTRA).zip(&m.channel) { writeln!(md, "| {n} | {b:+.3} |")?; }
writeln!(md)?;
Ok(rows)
}
+17 -1
View File
@@ -3,8 +3,10 @@
//!
//! airplay model panel what the panel holds
//! airplay model backtest [--h 1,4,8] [--refit 1] [--out docs/BACKTEST.md]
//! airplay model adds [--h 0,4] [--out docs/BACKTEST-ADDS.md] backtest of the new-add model
//! airplay model rank [--h 4] [--max-h 12] [--top 50]
pub mod adds;
pub mod backtest;
pub mod common;
pub mod features;
@@ -51,6 +53,20 @@ pub fn run(db: &Db, args: &[String]) -> Result<()> {
print!("{}", report.markdown);
eprintln!("written to {out}");
}
Some("adds") => {
let horizons: Vec<usize> = arg(args, "--h").unwrap_or_else(|| "0,4".into()).split(',').filter_map(|s| s.trim().parse().ok()).collect();
let tau = common::thresholds(&p, 26);
let mut md = String::new();
let rows = adds::backtest(&p, &tau, &horizons, 26, &mut md)?;
rank::create_tables(db)?;
let at = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string();
for (m, h, metric, v) in &rows {
db.conn.execute("INSERT OR REPLACE INTO backtest_scores VALUES (?1, ?2, ?3, ?4, ?5)", rusqlite::params![at, m, *h as i64, metric, v])?;
}
let out = arg(args, "--out").unwrap_or_else(|| "docs/BACKTEST-ADDS.md".into());
std::fs::write(&out, &md)?;
print!("{md}");
}
Some("rank") => {
let h: usize = arg(args, "--h").and_then(|s| s.parse().ok()).unwrap_or(4);
let max_h: usize = arg(args, "--max-h").and_then(|s| s.parse().ok()).unwrap_or(12).max(h);
@@ -67,7 +83,7 @@ pub fn run(db: &Db, args: &[String]) -> Result<()> {
}
}
_ => {
eprintln!("usage: airplay model panel | backtest [--h 1,4,8] [--refit 1] [--out FILE] | rank [--h 4] [--max-h 12] [--top 50]");
eprintln!("usage: airplay model panel | backtest [--h 1,4,8] [--refit 1] [--out FILE] | adds [--h 0,4] | rank [--h 4] [--max-h 12] [--top 50]");
std::process::exit(2);
}
}
+43 -1
View File
@@ -7,6 +7,7 @@
//! independent. Not yet conditioned on a release at t + h: that needs the release-effect curve
//! from release dates (Deezer backfill). Until then this ranks expected airplay given current form.
use super::adds::{AddModel, Rot};
use super::common::{thresholds, Direct, Rng};
use super::features::P;
use super::panel::{Panel, C, CHANNELS};
@@ -31,7 +32,9 @@ pub fn create_tables(db: &Db) -> Result<()> {
db.conn.execute_batch(
"CREATE TABLE IF NOT EXISTS forecasts (
made_at TEXT NOT NULL, model_version TEXT NOT NULL,
target TEXT NOT NULL, -- 'rotation' (prob, mean plays) per channel, or 'any' across channels
target TEXT NOT NULL, -- 'rotation' (prob, mean plays) per channel, 'any' across channels;
-- 'add' / 'add_any': P(new add in the 8 weeks from target_week);
-- 'now': in rotation in the origin week (prob 1/0, mean = plays)
key TEXT NOT NULL, -- '{dr artist id}:{channel}' or '{dr artist id}:any'
horizon INTEGER NOT NULL, -- weeks after origin_week
origin_week TEXT NOT NULL, target_week TEXT NOT NULL,
@@ -105,5 +108,44 @@ pub fn run(db: &Db, p: &Panel, max_h: usize, show_h: usize) -> Result<Vec<Line>>
eprintln!("rank: h={h} stored ({} artists)", lines.len());
if h == show_h { shown = lines; }
}
// New adds: windows of W weeks starting h = 0..=max_h weeks after the origin.
let tau = thresholds(p, p.w());
let rot = Rot::new(p, &tau);
for h in 0..=max_h {
let m = AddModel::fit(p, &rot, h, p.w() - 1);
let start = (origin + Duration::weeks(h as i64 + 1)).to_string();
let tx = db.conn.unchecked_transaction()?;
{
let mut ins = tx.prepare("INSERT OR REPLACE INTO forecasts VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)")?;
for a in 0..p.a() {
let id = &p.artists[a].id;
let (pc, pa) = m.predict(p, &rot, a, p.w() - 1, None);
ins.execute(params![made, VERSION, "add_any", format!("{id}:any"), h as i64, origin.to_string(), start, pa,
Option::<f64>::None, Option::<f64>::None, Option::<f64>::None])?;
for c in 0..C {
ins.execute(params![made, VERSION, "add", format!("{id}:{}", CHANNELS[c]), h as i64, origin.to_string(), start, pc[c],
Option::<f64>::None, Option::<f64>::None, Option::<f64>::None])?;
}
}
}
tx.commit()?;
eprintln!("rank: adds h={h} stored");
}
// Where each artist stands in the origin week: prob = 1 if in rotation, mean = plays.
let tx = db.conn.unchecked_transaction()?;
{
let mut ins = tx.prepare("INSERT OR REPLACE INTO forecasts VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)")?;
let t0 = p.w() - 1;
for a in 0..p.a() {
let id = &p.artists[a].id;
for c in 0..C {
let y = p.y(a, c, t0);
if y == 0.0 { continue; }
ins.execute(params![made, VERSION, "now", format!("{id}:{}", CHANNELS[c]), 0i64, origin.to_string(), origin.to_string(),
rot.on(a, c, t0) as i32 as f64, y, Option::<f64>::None, Option::<f64>::None])?;
}
}
}
tx.commit()?;
Ok(shown)
}
+343
View File
@@ -0,0 +1,343 @@
//! DR's weekly rotation lists (PDF): which songs are in rotation, in which category, per channel.
//!
//! The PDF merges one page per channel. Each page has category headers (A, B, C, and on the
//! P3 page also "Nye i power", "Ugens crush", "P3's uundgåelige") with artist + title lines
//! under them, in one or several columns. Glyph positions are read from the PDF; a line belongs
//! to the nearest header above it in its column. Which channel a page is gets decided from the
//! data: the channel where the page's artists are played most (DR artist pages), so a change in
//! page order doesn't mislabel anything. The P4 list is national and stored as channel "p4".
//!
//! airplay rotations [--reparse] parse every saved PDF into rotation_lists
//! airplay rotations --wayback also fetch older lists archived by the Wayback Machine
use crate::{db::Db, http::Http};
use anyhow::{anyhow, Result};
use chrono::{Datelike, NaiveDate, Utc};
use pdf_extract::{MediaBox, OutputDev, OutputError, Transform};
use regex::Regex;
use rusqlite::params;
use std::collections::HashMap;
pub fn create_tables(db: &Db) -> Result<()> {
db.conn.execute_batch(
"CREATE TABLE IF NOT EXISTS rotation_lists (
list_date TEXT NOT NULL, -- the date in the PDF's file name (published, a Friday)
iso_year INTEGER, iso_week INTEGER, -- 'UGE nn' printed on the list: the week it applies to
channel TEXT NOT NULL, -- p3 | p4 (national, all regions) | p5 | p6beat | p8jazz
category TEXT NOT NULL, -- A | B | C | or the list's own label (NYE I POWER, UGENS CRUSH, UUNDGÅELIGE)
position INTEGER NOT NULL, -- order within the category
artist TEXT NOT NULL, title TEXT NOT NULL,
file TEXT NOT NULL, page INTEGER NOT NULL,
channel_score REAL, -- how clearly the page matched its channel (share of plays)
PRIMARY KEY (list_date, channel, category, position)
);
CREATE TABLE IF NOT EXISTS rotation_wayback (
snapshot TEXT PRIMARY KEY, pdf_url TEXT, fetched_at TEXT NOT NULL, status TEXT
);",
)?;
Ok(())
}
// ------------------------------------------------------------------ glyphs → runs
#[derive(Debug, Clone)]
pub struct Run { pub page: u32, pub x: f64, pub y: f64, pub x_end: f64, pub size: f64, pub text: String }
#[derive(Default)]
struct Glyphs { page: u32, width: f64, height: f64, glyphs: Vec<(u32, f64, f64, f64, f64, String)> }
impl OutputDev for Glyphs {
fn begin_page(&mut self, page_num: u32, media_box: &MediaBox, _: Option<(f64, f64, f64, f64)>) -> Result<(), OutputError> {
self.page = page_num;
self.width = media_box.urx - media_box.llx;
self.height = media_box.ury - media_box.lly;
Ok(())
}
fn end_page(&mut self) -> Result<(), OutputError> { Ok(()) }
fn output_character(&mut self, trm: &Transform, width: f64, _spacing: f64, font_size: f64, ch: &str) -> Result<(), OutputError> {
let scale = (trm.m11 * trm.m11 + trm.m12 * trm.m12).sqrt();
let size = font_size * scale;
let (x, y) = (trm.m31, self.height - trm.m32);
// Decoration: rotated text (the vertical "ROTATIONSLISTEN" frame, the tilted "P3 ROTATIONER"
// background of older lists) and big letters along the page edges.
// (m12 is the rotation; m21 alone is the skew of fake italics, which is kept)
let rotated = trm.m12.abs() > 0.2 * scale.max(1e-9);
let margin = size > 25.0 && (x < 0.06 * self.width || x > 0.94 * self.width || y < 0.06 * self.height || y > 0.94 * self.height);
if !margin && !rotated { self.glyphs.push((self.page, x, y, width * size, size, ch.to_string())); }
Ok(())
}
fn begin_word(&mut self) -> Result<(), OutputError> { Ok(()) }
fn end_word(&mut self) -> Result<(), OutputError> { Ok(()) }
fn end_line(&mut self) -> Result<(), OutputError> { Ok(()) }
}
/// Text runs per page. Glyphs drawn twice at one spot (fake bold) count once; a run ends at a
/// gap wider than ~1.6 times the font size (a column gap).
pub fn runs(bytes: &[u8]) -> Result<Vec<Run>> {
let doc = pdf_extract::Document::load_mem(bytes)?;
let mut g = Glyphs::default();
pdf_extract::output_doc(&doc, &mut g)?;
let mut glyphs = g.glyphs;
glyphs.sort_by(|a, b| a.0.cmp(&b.0).then(a.2.total_cmp(&b.2)).then(a.1.total_cmp(&b.1)));
let mut lines: Vec<Vec<(u32, f64, f64, f64, f64, String)>> = Vec::new();
for gl in glyphs {
match lines.last_mut() {
Some(l) if l[0].0 == gl.0 && (gl.2 - l[0].2).abs() < l[0].4.min(gl.4).max(1.0) * 0.35 => l.push(gl),
_ => lines.push(vec![gl]),
}
}
let mut out = Vec::new();
for mut l in lines {
l.sort_by(|a, b| a.1.total_cmp(&b.1));
l.dedup_by(|b, a| a.5 == b.5 && (a.1 - b.1).abs() < 0.5);
let mut cur: Option<Run> = None;
for (page, x, y, w, size, ch) in l {
if let Some(r) = cur.as_mut() {
let same_size = (size - r.size).abs() < 0.5;
if same_size && x - r.x_end < size * 0.45 { r.text.push_str(&ch); r.x_end = r.x_end.max(x + w); continue; }
if same_size && x - r.x_end < size * 1.6 { r.text.push(' '); r.text.push_str(&ch); r.x_end = x + w; continue; }
out.push(cur.take().unwrap());
}
cur = Some(Run { page, x, y, x_end: x + w, size, text: ch });
}
if let Some(r) = cur { out.push(r); }
}
for r in &mut out { r.text = r.text.split_whitespace().collect::<Vec<_>>().join(" "); }
out.retain(|r| !r.text.is_empty());
Ok(out)
}
pub fn dump(path: &std::path::Path) -> Result<()> {
for r in runs(&std::fs::read(path)?)? {
println!("p{} y{:7.1} x{:6.1}-{:6.1} s{:4.1} | {}", r.page, r.y, r.x, r.x_end, r.size, r.text);
}
Ok(())
}
// ------------------------------------------------------------------ runs → entries
#[derive(Debug, Clone)]
pub struct Entry { pub page: u32, pub category: String, pub position: usize, pub artist: String, pub title: String }
pub struct Parsed { pub week: Option<u32>, pub entries: Vec<Entry> }
pub fn parse(bytes: &[u8]) -> Result<Parsed> {
let runs = runs(bytes)?;
let week_re = Regex::new(r"UGE\s*(\d{1,2})")?;
let week = runs.iter().find_map(|r| week_re.captures(&r.text).and_then(|c| c[1].parse().ok()));
let is_cat = |t: &str| matches!(t, "A" | "B" | "C");
let is_label = |r: &Run| r.size >= 12.0 && r.size < 40.0 && r.text.chars().count() >= 4 && !is_cat(&r.text)
&& !r.text.starts_with("UGE") && !r.text.starts_with("E ") && !r.text.contains("ROTATIONSLISTEN");
let mut entries = Vec::new();
let pages: Vec<u32> = { let mut p: Vec<u32> = runs.iter().map(|r| r.page).collect(); p.dedup(); p };
for page in pages {
let pr: Vec<&Run> = runs.iter().filter(|r| r.page == page).collect();
// Headers: category letters and labelled sections.
let headers: Vec<(&Run, String)> = pr.iter().filter(|r| (r.size >= 12.0 && is_cat(&r.text)) || is_label(r))
.map(|r| (*r, if is_cat(&r.text) { r.text.clone() } else { label(&r.text) })).collect();
// "P3's uundgåelige": the two big runs after the label are artist and title.
if let Some((h, _)) = headers.iter().find(|(r, _)| r.text.to_uppercase().contains("UUNDGÅELIGE")) {
let big: Vec<&&Run> = pr.iter().filter(|r| r.size >= 40.0 && r.y > h.y && !is_cat(&r.text)).take(2).collect();
if big.len() == 2 {
entries.push(Entry { page, category: "UUNDGÅELIGE".into(), position: 1, artist: big[0].text.clone(), title: big[1].text.clone() });
}
}
let body: Vec<&Run> = pr.iter().copied().filter(|r| r.size < 12.0).collect();
// Columns start at the category letters (A, B, C side by side on the P3 summary page).
let mut cols: Vec<f64> = headers.iter().filter(|(r, _)| is_cat(&r.text)).map(|(r, _)| r.x).collect();
cols.sort_by(f64::total_cmp);
cols.dedup_by(|b, a| (*b - *a).abs() < 40.0);
let col_of = |x: f64| cols.iter().rposition(|c| *c <= x + 20.0).unwrap_or(0);
// Lines: same y and same column; the header: the lowest one above, in that column.
let mut lines: Vec<(usize, Vec<&Run>)> = Vec::new();
let mut sorted = body.clone();
sorted.sort_by(|a, b| a.y.total_cmp(&b.y).then(a.x.total_cmp(&b.x)));
for r in sorted {
let c = col_of(r.x);
match lines.iter_mut().find(|(lc, l)| *lc == c && (l[0].y - r.y).abs() < 2.0) {
Some((_, l)) => l.push(r),
None => lines.push((c, vec![r])),
}
}
let mut count: HashMap<usize, usize> = HashMap::new();
for (c, mut l) in lines {
l.sort_by(|a, b| a.x.total_cmp(&b.x));
if l.len() < 2 { continue; } // a lone run: not an artist + title pair
let Some((hi, (_, cat))) = headers.iter().enumerate()
.filter(|(_, (h, _))| col_of(h.x) == c && h.y < l[0].y + 1.0)
.max_by(|a, b| a.1 .0.y.total_cmp(&b.1 .0.y)) else { continue };
let n = count.entry(hi).or_insert(0);
*n += 1;
entries.push(Entry { page, category: cat.clone(), position: *n, artist: l[0].text.clone(),
title: l[1..].iter().map(|r| r.text.as_str()).collect::<Vec<_>>().join(" ") });
}
}
Ok(Parsed { week, entries })
}
fn label(t: &str) -> String {
let u = t.to_uppercase();
if u.contains("CRUSH") { "UGENS CRUSH".into() }
else if u.contains("NYE I POWER") { "NYE I POWER".into() }
else if u.contains("RECURRENT") { "RECURRENTS".into() }
else if u.contains("ROTATIONER") || u.contains("ROTATIONSLISTE") { "LIST".into() }
else { u }
}
// ------------------------------------------------------------------ pages → channels
const LIST_CHANNELS: [&str; 5] = ["p3", "p4", "p5", "p6beat", "p8jazz"];
/// Share of each list channel in the plays of the page's artists (DR artist pages, all weeks).
fn channel_scores(db: &Db, artists: &[&str]) -> Result<[f64; 5]> {
let key = crate::model::panel::key;
let mut by_name: HashMap<String, Vec<String>> = HashMap::new();
for r in db.conn.prepare("SELECT artist_id, name FROM dr_artists")?.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? {
let (id, n) = r?;
by_name.entry(key(&n)).or_default().push(id);
}
let mut st = db.conn.prepare("SELECT channel, sum(plays) FROM dr_artist_weeks WHERE artist_id = ?1 GROUP BY channel")?;
let mut score = [0.0; 5];
for a in artists {
let first = a.split(&['[', ','][..]).next().unwrap_or(a).split(" feat").next().unwrap_or(a).split(" ft.").next().unwrap_or(a)
.split(" & ").next().unwrap_or(a).split(" x ").next().unwrap_or(a).trim();
let Some(ids) = by_name.get(&key(first)) else { continue };
let mut per = [0.0; 5];
for id in ids {
for r in st.query_map([id], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?)))? {
let (ch, n) = r?;
let k = if ch.starts_with("p4") && ch != "p4" { Some(1) } else { LIST_CHANNELS.iter().position(|c| *c == ch) };
// the ten regional P4s count as one channel
if let Some(k) = k { per[k] += n as f64 / if k == 1 { 10.0 } else { 1.0 }; }
}
}
let tot: f64 = per.iter().sum();
if tot > 0.0 { for k in 0..5 { score[k] += per[k] / tot; } }
}
Ok(score)
}
/// Parses one PDF and stores its rows (a list date already stored is left alone unless `replace`).
pub fn store(db: &Db, file: &std::path::Path, replace: bool, log: &mut Vec<String>) -> Result<usize> {
create_tables(db)?;
let name = file.file_name().and_then(|n| n.to_str()).unwrap_or_default().to_string();
let date = Regex::new(r"^(\d{2})-(\d{2})-(\d{4})")?.captures(&name)
.and_then(|c| NaiveDate::from_ymd_opt(c[3].parse().ok()?, c[2].parse().ok()?, c[1].parse().ok()?))
.or_else(|| Regex::new(r"^(\d{4})-(\d{2})-(\d{2})").ok()?.captures(&name)
.and_then(|c| NaiveDate::from_ymd_opt(c[1].parse().ok()?, c[2].parse().ok()?, c[3].parse().ok()?)))
.ok_or_else(|| anyhow!("{name}: no date in the file name"))?;
let have: i64 = db.conn.query_row("SELECT count(*) FROM rotation_lists WHERE list_date = ?1", [date.to_string()], |r| r.get(0))?;
if have > 0 && !replace { return Ok(0); }
let parsed = parse(&std::fs::read(file)?)?;
// ISO week of the list: "UGE nn" near the list date (the list is for the coming week).
let iso = parsed.week.map(|w| {
let y = date.iso_week().year();
let y = if w < 10 && date.iso_week().week() > 40 { y + 1 } else { y };
(y, w)
});
let mut pages: Vec<u32> = parsed.entries.iter().map(|e| e.page).collect();
pages.sort(); pages.dedup();
let mut channel_of: HashMap<u32, (String, f64)> = HashMap::new();
let mut taken: Vec<usize> = Vec::new();
// Most distinctive pages first, each channel used once (P3 may span two pages: the summary page with "uundgåelige").
let mut scored: Vec<(u32, [f64; 5], f64)> = Vec::new();
for &pg in &pages {
let artists: Vec<&str> = parsed.entries.iter().filter(|e| e.page == pg).map(|e| e.artist.as_str()).collect();
let s = channel_scores(db, &artists)?;
let tot: f64 = s.iter().sum();
scored.push((pg, s, tot));
}
scored.sort_by(|a, b| { let m = |x: &[f64; 5], t: f64| x.iter().cloned().fold(0.0, f64::max) / t.max(1e-9); m(&b.1, b.2).total_cmp(&m(&a.1, a.2)) });
for (pg, s, tot) in &scored {
let mut order: Vec<usize> = (0..5).collect();
order.sort_by(|&a, &b| s[b].total_cmp(&s[a]));
let k = order.iter().copied().find(|k| !taken.contains(k) || (*k == 0)).unwrap_or(order[0]);
if k != 0 { taken.push(k); }
channel_of.insert(*pg, (LIST_CHANNELS[k].to_string(), s[k] / tot.max(1e-9)));
if s[k] / tot.max(1e-9) < 0.4 { log.push(format!("{name} page {pg}: weak channel match ({} {:.2})", LIST_CHANNELS[k], s[k] / tot.max(1e-9))); }
}
let tx = db.conn.unchecked_transaction()?;
tx.execute("DELETE FROM rotation_lists WHERE list_date = ?1", [date.to_string()])?;
let mut n = 0;
for e in &parsed.entries {
let (ch, score) = channel_of.get(&e.page).cloned().unwrap_or(("unknown".into(), 0.0));
// The P3 summary page repeats categories A/B/C of a sub-list; keep its own labels apart.
let cat = if ch == "p3" && pages.len() > 5 && e.page == pages[0] && matches!(e.category.as_str(), "A" | "B" | "C") { format!("P3 SIDE {}", e.category) } else { e.category.clone() };
tx.execute("INSERT OR REPLACE INTO rotation_lists VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)",
params![date.to_string(), iso.map(|x| x.0), iso.map(|x| x.1), ch, cat, e.position as i64, e.artist, e.title, name, e.page as i64, score])?;
n += 1;
}
tx.commit()?;
Ok(n)
}
/// Parses every PDF in `dir`.
pub fn store_all(db: &Db, dir: &std::path::Path, replace: bool) -> Result<()> {
let mut log = Vec::new();
let mut files: Vec<_> = std::fs::read_dir(dir)?.filter_map(|e| e.ok()).map(|e| e.path()).filter(|p| p.extension().is_some_and(|x| x == "pdf")).collect();
files.sort();
for f in files {
match store(db, &f, replace, &mut log) {
Ok(n) => println!("{}: {n} rows", f.display()),
Err(e) => println!("{}: {e}", f.display()),
}
}
for l in log { println!(" {l}"); }
let mut st = db.conn.prepare("SELECT list_date, channel, category, count(*), round(avg(channel_score), 2) FROM rotation_lists GROUP BY 1, 2, 3 ORDER BY 1, 2, 3")?;
for r in st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, i64>(3)?, r.get::<_, f64>(4)?)))? {
let (d, c, k, n, s) = r?;
println!(" {d} {c:8} {k:14} {n:3} (channel match {s})");
}
Ok(())
}
// ------------------------------------------------------------------ Wayback Machine
/// Older lists: every archived copy of DR's rotation page links that week's PDF; fetch the PDF
/// from DR if it's still there, else from the archive. Each snapshot is tried once.
pub fn wayback(db: &Db, http: &Http, dir: &std::path::Path, log: &mut Vec<String>) -> Result<usize> {
create_tables(db)?;
let page = "www.dr.dk/om-dr/nyheder/musik-i-rotation-paa-drs-radiokanaler-netop-nu";
let cdx = http.get(&format!("https://web.archive.org/cdx/search/cdx?url={page}&output=json&fl=timestamp,statuscode&filter=statuscode:200&collapse=digest"))?
.ok_or_else(|| anyhow!("no CDX answer"))?;
let rows: Vec<Vec<String>> = serde_json::from_str(&cdx).map_err(|e| anyhow!("CDX: {e}"))?;
// The link has moved over the years: www.dr.dk/static/documents/... (2023-24), asset.dr.dk/... (later).
let pdf_re = Regex::new(r#"https?://(?:asset\.dr\.dk|www\.dr\.dk/static)/[^"'\s<>]+?rotation[^"'\s<>]*?\.pdf"#)?;
let path_date = Regex::new(r"/(\d{4})/(\d{2})/(\d{2})/")?;
std::fs::create_dir_all(dir)?;
let mut saved = 0;
for row in rows.iter().skip(1) {
let ts = &row[0];
let done: i64 = db.conn.query_row("SELECT count(*) FROM rotation_wayback WHERE snapshot = ?1", [ts], |r| r.get(0))?;
if done > 0 { continue; }
let html = http.get(&format!("https://web.archive.org/web/{ts}id_/https://{page}"))?.unwrap_or_default();
let url = pdf_re.find(&html).map(|m| m.as_str().replace("http://", "https://"));
let status = match &url {
None => "no pdf link".to_string(),
Some(u) => {
// Name the file by its publication date (dd-mm-yyyy first, which `store` reads).
let base = u.rsplit('/').next().unwrap_or("rotation.pdf");
let name = match path_date.captures(u) { Some(c) => format!("{}-{}-{}-{base}", &c[3], &c[2], &c[1]), None => base.to_string() };
let file = dir.join(name);
if file.exists() { "already saved".to_string() } else {
let bytes = match http.get_bytes(u) {
Ok(Some(b)) if b.starts_with(b"%PDF") => Some(b),
_ => http.get_bytes(&format!("https://web.archive.org/web/{ts}id_/{u}")).ok().flatten().filter(|b| b.starts_with(b"%PDF")),
};
match bytes {
Some(b) => {
std::fs::write(&file, &b)?;
db.conn.execute("INSERT OR IGNORE INTO dr_rotation_pdfs VALUES (?1, ?2, ?3)", params![u, file.to_string_lossy(), Utc::now().to_rfc3339()])?;
saved += 1;
"saved".to_string()
}
None => { log.push(format!("wayback {ts}: {u} not retrievable")); "pdf not retrievable".to_string() }
}
}
}
};
db.conn.execute("INSERT OR REPLACE INTO rotation_wayback VALUES (?1, ?2, ?3, ?4)", params![ts, url, Utc::now().to_rfc3339(), status])?;
println!("{ts}: {status} {}", url.unwrap_or_default());
}
Ok(saved)
}
+62 -47
View File
@@ -10,6 +10,7 @@
//! Rankings are read from the forecasts table (`airplay model rank`); the what-if models are
//! fitted in the background at start and again whenever a newer forecast run appears.
use crate::model::adds::{set_chart, AddModel, Rot, CHART_IDX_CHANNEL};
use crate::model::common::{thresholds, Direct, Rng};
use crate::model::features::{self, P};
use crate::model::glm;
@@ -30,6 +31,8 @@ struct Models {
panel: Panel,
tau: [u32; C],
direct: Vec<Direct>, // index h - 1
adds: Vec<AddModel>, // index h
rot: Rot,
index: HashMap<String, usize>,
}
@@ -100,9 +103,11 @@ fn fit_models(db_path: &str, made_at: String) -> Result<Models> {
let db = crate::db::Db::open(db_path)?;
let panel = Panel::load(&db)?;
let tau = thresholds(&panel, panel.w());
let direct = (1..=MAX_H).map(|h| Direct::fit(&panel, h, panel.w() - 1, &tau, true)).collect();
let direct = (1..=MAX_H).map(|h| Direct::fit(&panel, h, panel.w() - 1, &tau, false)).collect();
let rot = Rot::new(&panel, &tau);
let adds = (0..=MAX_H).map(|h| AddModel::fit(&panel, &rot, h, panel.w() - 1)).collect();
let index = panel.artists.iter().enumerate().map(|(i, a)| (a.id.clone(), i)).collect();
Ok(Models { made_at, panel, tau, direct, index })
Ok(Models { made_at, panel, tau, direct, adds, rot, index })
}
fn query(url: &str) -> HashMap<String, String> {
@@ -143,7 +148,7 @@ fn handle(st: &State, method: &Method, url: &str) -> Result<Reply> {
"" => Ok((301, "text/plain", Arc::new(String::new()))),
"/" | "/index.html" => Ok((200, "text/html; charset=utf-8", Arc::new(PAGE.to_string()))),
"/api/meta" => meta(st),
"/api/rank" => rank(st, q.get("h").and_then(|h| h.parse().ok()).unwrap_or(4).clamp(1, MAX_H)),
"/api/rank" => rank(st, q.get("h").and_then(|h| h.parse().ok()).unwrap_or(0).min(MAX_H)),
"/api/artist" => artist(st, q.get("id").map(String::as_str).unwrap_or("")),
"/api/whatif" => whatif(st, &q),
_ => Ok((404, "text/plain", Arc::new("not found".into()))),
@@ -169,6 +174,21 @@ fn meta(st: &State) -> Result<Reply> {
"channels": CHANNELS, "tau": tau, "whatif_ready": models.is_some(), "backtest_run": run, "backtest": scores}))
}
/// Puts one forecast row into an artist's JSON: airplay ('rotation', 'any'), new adds ('add',
/// 'add_any') and the origin week ('now').
fn fold(e: &mut Value, target: &str, ch: &str, prob: Option<f64>, mean: Option<f64>, lo: Option<f64>, hi: Option<f64>) {
let r3 = |v: Option<f64>| json!(v.map(|p| (p * 1000.0).round() / 1000.0));
let r1 = |v: Option<f64>| json!(v.map(|m| (m * 10.0).round() / 10.0));
match target {
"any" => { e["p_any"] = r3(prob); e["mean"] = r1(mean); e["lo"] = json!(lo); e["hi"] = json!(hi); }
"rotation" => { e["p"][ch] = r3(prob); e["plays"][ch] = r1(mean); }
"add_any" => { e["add_any"] = r3(prob); }
"add" => { e["add"][ch] = r3(prob); }
"now" => { e["now"][ch] = json!({"on": prob.unwrap_or(0.0) > 0.5, "plays": mean}); }
_ => {}
}
}
fn rank(st: &State, h: usize) -> Result<Reply> {
let c = ro(&st.db_path)?;
let Some(made) = latest_run(&c)? else { return ok(json!({"rows": []})) };
@@ -180,26 +200,24 @@ fn rank(st: &State, h: usize) -> Result<Reply> {
names.insert(id, (n, country.as_deref() == Some("Danmark"), kind.is_some_and(|k| k != "person")));
}
let mut rows: HashMap<String, Value> = HashMap::new();
let mut st_ = c.prepare("SELECT key, target, prob, mean, lo, hi FROM forecasts WHERE made_at = ?1 AND model_version = ?2 AND horizon = ?3")?;
let it = st_.query_map(params![made, VERSION, h as i64], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, Option<f64>>(2)?,
r.get::<_, Option<f64>>(3)?, r.get::<_, Option<f64>>(4)?, r.get::<_, Option<f64>>(5)?)))?;
let mut st_ = c.prepare("SELECT key, target, prob, mean, lo, hi, target_week FROM forecasts WHERE made_at = ?1 AND model_version = ?2
AND ((target IN ('rotation', 'any') AND horizon = ?3) OR (target IN ('add', 'add_any') AND horizon = ?4) OR target = 'now')")?;
let it = st_.query_map(params![made, VERSION, h.max(1) as i64, h as i64], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, Option<f64>>(2)?,
r.get::<_, Option<f64>>(3)?, r.get::<_, Option<f64>>(4)?, r.get::<_, Option<f64>>(5)?, r.get::<_, String>(6)?)))?;
let mut window = String::new();
for r in it {
let (key, target, prob, mean, lo, hi) = r?;
let (key, target, prob, mean, lo, hi, week) = r?;
let Some((id, ch)) = key.split_once(':') else { continue };
let e = rows.entry(id.to_string()).or_insert_with(|| {
let (n, dk, group) = names.get(id).cloned().unwrap_or_default();
json!({"id": id, "name": n, "danish": dk, "group": group, "p": {}, "plays": {}})
json!({"id": id, "name": n, "danish": dk, "group": group, "p": {}, "plays": {}, "add": {}, "now": {}})
});
if target == "any" {
e["p_any"] = json!(prob); e["mean"] = json!(mean); e["lo"] = json!(lo); e["hi"] = json!(hi);
} else {
e["p"][ch] = json!(prob.map(|p| (p * 1000.0).round() / 1000.0));
e["plays"][ch] = json!(mean.map(|m| (m * 10.0).round() / 10.0));
}
if target == "add_any" && window.is_empty() { window = week; }
fold(e, &target, ch, prob, mean, lo, hi);
}
let mut out: Vec<Value> = rows.into_values().collect();
out.sort_by(|a, b| b["mean"].as_f64().unwrap_or(0.0).total_cmp(&a["mean"].as_f64().unwrap_or(0.0)));
let body = Arc::new(json!({"made_at": made, "h": h, "rows": out}).to_string());
out.sort_by(|a, b| b["add_any"].as_f64().unwrap_or(0.0).total_cmp(&a["add_any"].as_f64().unwrap_or(0.0)));
let body = Arc::new(json!({"made_at": made, "h": h, "window_start": window, "rows": out}).to_string());
let mut cache = st.rank_cache.write().unwrap();
cache.retain(|k, _| k.0 == made);
cache.insert((made, h), body.clone());
@@ -232,17 +250,21 @@ fn artist(st: &State, id: &str) -> Result<Reply> {
}
let mut fc: Vec<Value> = Vec::new();
let mut now_out = Value::Null;
if let Some(made) = latest_run(&c)? {
let mut byh: HashMap<i64, Value> = HashMap::new();
for r in c.prepare("SELECT horizon, target_week, key, prob, mean, lo, hi FROM forecasts WHERE made_at = ?1 AND model_version = ?2 AND key LIKE ?3")?
let mut now = json!({});
for r in c.prepare("SELECT horizon, target_week, key, prob, mean, lo, hi, target FROM forecasts WHERE made_at = ?1 AND model_version = ?2 AND key LIKE ?3")?
.query_map(params![made, VERSION, format!("{id}:%")], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?,
r.get::<_, Option<f64>>(3)?, r.get::<_, Option<f64>>(4)?, r.get::<_, Option<f64>>(5)?, r.get::<_, Option<f64>>(6)?)))? {
let (h, week, key, prob, mean, lo, hi) = r?;
let e = byh.entry(h).or_insert_with(|| json!({"h": h, "week": week, "p": {}, "plays": {}}));
r.get::<_, Option<f64>>(3)?, r.get::<_, Option<f64>>(4)?, r.get::<_, Option<f64>>(5)?, r.get::<_, Option<f64>>(6)?, r.get::<_, String>(7)?)))? {
let (h, week, key, prob, mean, lo, hi, target) = r?;
let ch = key.split_once(':').map(|x| x.1).unwrap_or("");
if ch == "any" { e["p_any"] = json!(prob); e["mean"] = json!(mean); e["lo"] = json!(lo); e["hi"] = json!(hi); }
else { e["p"][ch] = json!(prob); e["plays"][ch] = json!(mean); }
if target == "now" { fold(&mut now, "now", ch, prob, mean, lo, hi); continue; }
let e = byh.entry(h).or_insert_with(|| json!({"h": h, "p": {}, "plays": {}, "add": {}}));
if target.starts_with("add") { e["window_start"] = json!(week); } else { e["week"] = json!(week); }
fold(e, &target, ch, prob, mean, lo, hi);
}
now_out = now["now"].clone();
let mut v: Vec<(i64, Value)> = byh.into_iter().collect();
v.sort_by_key(|x| x.0);
fc = v.into_iter().map(|x| x.1).collect();
@@ -267,7 +289,7 @@ fn artist(st: &State, id: &str) -> Result<Reply> {
ok(json!({
"id": id, "name": name, "country": a["country"], "gender": a["gender"], "description": a["description"],
"debut": a["lifespan"]["beginYear"], "type": a["type"], "image": image, "top_tracks": top,
"weeks": weeks, "plays": plays, "forecasts": fc, "charts": charts,
"weeks": weeks, "plays": plays, "forecasts": fc, "now": now_out, "charts": charts,
}))
}
@@ -275,59 +297,52 @@ fn artist(st: &State, id: &str) -> Result<Reply> {
/// entry weeks since a new chart entry (a release that charted), "none" = never
/// t40 position on the latest Track Top-40 (0 = not on it)
/// air position on the latest Airplay Top-20 (0 = not on it)
/// h = 0..12: the add window starts h weeks after the latest week; plays are for week max(h, 1).
fn whatif(st: &State, q: &HashMap<String, String>) -> Result<Reply> {
let guard = st.models.read().unwrap();
let Some(m) = guard.as_ref() else { return Ok((503, "application/json", Arc::new(json!({"loading": true}).to_string()))) };
let Some(&a) = m.index.get(q.get("id").map(String::as_str).unwrap_or("")) else {
return Ok((404, "application/json", Arc::new(json!({"error": "artist not in the model"}).to_string())));
};
let h: usize = q.get("h").and_then(|v| v.parse().ok()).unwrap_or(4).clamp(1, MAX_H);
let d = &m.direct[h - 1];
let h: usize = q.get("h").and_then(|v| v.parse().ok()).unwrap_or(0).min(MAX_H);
let d = &m.direct[h.max(1) - 1];
let am = &m.adds[h];
let p = &m.panel;
let t0 = p.w() - 1;
let chart = p.chart(a, t0);
let num = |k: &str| q.get(k).filter(|v| !v.is_empty()).and_then(|v| v.parse::<f64>().ok());
let entry: Option<Option<f64>> = q.get("entry").filter(|v| !v.is_empty()).map(|v| if v == "none" { None } else { v.parse().ok() });
let t40 = num("t40");
let air = num("air");
let (t40, air) = (num("t40"), num("air"));
let over = move |x: &mut [f64], idx: [usize; 4]| set_chart(x, idx, entry, t40, air);
let run = |overrides: bool| -> Value {
let f: Option<&dyn Fn(&mut [f64], [usize; 4])> = if overrides { Some(&over) } else { None };
let (add_c, add_any) = am.predict(p, &m.rot, a, t0, f);
let mut x = vec![0.0; P];
let (mut mu, mut p_nb, mut p_lg) = ([0.0; C], [0.0; C], [0.0; C]);
let mut mu = [0.0; C];
for c in 0..C {
features::row(p, a, c, t0, t0 + h, &mut x);
if overrides {
if let Some(e) = entry {
let (r6, r26) = match e { Some(w) => ((-w / 6.0).exp(), (-w / 26.0).exp()), None => (0.0, 0.0) };
x[12] = r6; x[13] = r26;
}
if let Some(r) = t40 { x[10] = if r >= 1.0 && r <= 40.0 { (41.0 - r) / 40.0 } else { 0.0 }; }
if let Some(r) = air { x[11] = if r >= 1.0 && r <= 20.0 { (21.0 - r) / 20.0 } else { 0.0 }; }
}
features::row(p, a, c, t0, t0 + h.max(1), &mut x);
if overrides { over(&mut x, CHART_IDX_CHANNEL); }
mu[c] = glm::dot(&d.nb, &x).clamp(-30.0, 12.0).exp();
p_nb[c] = glm::nb_tail(mu[c], d.alpha, m.tau[c]);
p_lg[c] = glm::sigmoid(glm::dot(&d.logit, &x));
}
let sims = 4000;
let shape = 1.0 / d.alpha;
let mut rng = Rng(0xa1 ^ a as u64);
let (mut any, mut tot) = (0usize, Vec::with_capacity(sims));
let mut tot = Vec::with_capacity(sims);
for _ in 0..sims {
let g = rng.gamma(shape) / shape;
let (mut s, mut hit) = (0.0, false);
for c in 0..C { let y = rng.poisson(mu[c] * g); s += y; hit |= y >= m.tau[c] as f64; }
any += hit as usize; tot.push(s);
tot.push((0..C).map(|c| rng.poisson(mu[c] * g)).sum::<f64>());
}
tot.sort_by(f64::total_cmp);
let ch = |v: &[f64; C]| CHANNELS.iter().zip(v).map(|(c, p)| (c.to_string(), json!(p))).collect::<serde_json::Map<_, _>>();
json!({"p_any": any as f64 / sims as f64, "mean": mu.iter().sum::<f64>(), "lo": tot[sims / 10], "hi": tot[sims * 9 / 10],
"p": ch(&p_nb), "p_logit": ch(&p_lg), "plays": ch(&mu)})
json!({"add_any": add_any, "add": ch(&add_c), "mean": mu.iter().sum::<f64>(), "lo": tot[sims / 10], "hi": tot[sims * 9 / 10], "plays": ch(&mu)})
};
let now: serde_json::Map<String, Value> = (0..C).filter(|&c| m.rot.on(a, c, t0)).map(|c| (CHANNELS[c].to_string(), json!(p.y(a, c, t0)))).collect();
ok(json!({
"h": h, "week": (p.weeks[t0] + chrono::Duration::weeks(h as i64)).to_string(),
"h": h, "window_start": (p.weeks[t0] + chrono::Duration::weeks(h as i64 + 1)).to_string(),
"current": {"t40": chart.t40_rank, "air": chart.air_rank,
"entry": if chart.weeks_since_entry == u16::MAX { Value::Null } else { json!(chart.weeks_since_entry) },
"t40_hits": chart.t40_hits, "air_hits": chart.air_hits},
"t40_hits": chart.t40_hits, "air_hits": chart.air_hits, "in_rotation": now},
"base": run(false), "scenario": run(true),
}))
}
+61 -39
View File
@@ -107,7 +107,7 @@ svg text { fill: var(--muted); font-size: 11px; }
<main>
<section id="tab-rank">
<div class="controls">
<div class="ctl">Week <b id="hlabel">—</b><input type="range" id="h" min="1" max="12" value="4"></div>
<div class="ctl">8-week window starting <b id="hlabel">—</b><input type="range" id="h" min="0" max="12" value="0"></div>
<div class="ctl">Station
<select id="ch">
<option value="any">Any DR station</option>
@@ -120,7 +120,7 @@ svg text { fill: var(--muted); font-size: 11px; }
</div>
<div class="ctl">Sort by
<select id="sort">
<option value="p">Chance of rotation</option>
<option value="p">Chance of a new add</option>
<option value="plays">Expected plays</option>
<option value="up">Biggest climbers</option>
</select>
@@ -131,24 +131,25 @@ svg text { fill: var(--muted); font-size: 11px; }
<div class="tablewrap">
<table>
<thead><tr>
<th class="l">#</th><th class="l">Artist</th><th>Chance</th>
<th class="l">#</th><th class="l">Artist</th><th>New add</th>
<th class="hide-sm">P3</th><th class="hide-sm">P4</th><th class="hide-sm">P5</th><th class="hide-sm">P6</th><th class="hide-sm">P8</th>
<th>Plays / wk (80%)</th>
<th class="l hide-sm">In rotation now</th><th>Plays / wk (80%)</th>
</tr></thead>
<tbody id="rows"><tr><td colspan="9" class="empty">Loading…</td></tr></tbody>
<tbody id="rows"><tr><td colspan="10" class="empty">Loading…</td></tr></tbody>
</table>
</div>
<button class="more" id="more" hidden>Show more</button>
<p class="note">Chance = probability the artist is in rotation (plays at or above the station's weekly threshold) in the chosen week. "Biggest climbers" ranks by expected plays in that week versus the latest week. Click an artist for history and a what-if.</p>
<p class="note"><b>New add</b> = the chance that the artist gets a song into rotation within the 8-week window: entering rotation (at or above the station's weekly play threshold) after at least 4 weeks out of rotation. For "any DR station", out of rotation everywhere first. An artist with a song in rotation now usually has to drop out before a new add, which is why current hits rank lower here. "In rotation now" shows where the artist is in rotation in the latest week; plays per week are for the window's first week. Click an artist for history and a what-if.</p>
</section>
<section id="tab-model" hidden>
<div class="prose">
<h2>What the model does</h2>
<p>For every artist DR has played, it forecasts each DR music station's weekly plays 1–12 weeks ahead, and the chance that the artist is "in rotation" (at or above the station's weekly play threshold). It learns from 51 weeks of DR's per-artist weekly play counts plus the artist's IFPI chart history (Track Top-40, Airplay Top-20).</p>
<p>For every artist DR has played, it estimates the chance of a <b>new add</b>: the artist entering rotation on a station (at or above its weekly play threshold) after at least 4 weeks out of rotation, within an 8-week window starting 0–12 weeks from now. It also forecasts weekly plays. It learns from 51 weeks of DR's per-artist weekly play counts plus the artist's IFPI chart history (Track Top-40, Airplay Top-20).</p>
<p>DR publishes plays per artist, not per song, so "new add" can't yet tell a new song from an old one coming back. In the one week of song-level plays we have, only 3–12% of artists in rotation on a station had two songs in rotation at once, so "drops out first, then gets added" is the usual pattern. The weekly rotation lists (song level) will sharpen this.</p>
<p id="taus" class="note"></p>
</div>
<h2>Backtest</h2>
<h2>Backtest: new adds</h2>
<p class="note prose">Walk-forward: for each week from week 27, the models are fitted only on what was known at the time and forecast 1, 4 and 8 weeks ahead. "Adds" are rows where the artist was not in rotation at the time of the forecast, which is the hard and interesting part. Skill above 0 means better than the baseline.</p>
<div class="cards" id="cards"></div>
<div class="prose">
@@ -178,10 +179,23 @@ const esc = s => String(s ?? '').replace(/[&<>"']/g, c => ({ '&': '&amp;', '<':
const pct = p => p == null ? '—' : (p >= 0.995 ? '>99' : p < 0.005 ? '<1' : Math.round(p * 100)) + '%';
const fmtDate = s => new Date(s + 'T00:00:00Z').toLocaleDateString('en-GB', { day: 'numeric', month: 'short', timeZone: 'UTC' });
// Chance of a new add in the window (station group g, or any DR station).
function chanP(r, g) {
if (g === 'any') return r.p_any;
if (g === 'p4') return Math.max(...P4S.map(c => r.p[c] ?? 0));
return r.p[g];
if (g === 'any') return r.add_any;
if (g === 'p4') return Math.max(...P4S.map(c => r.add?.[c] ?? 0));
return r.add?.[g];
}
function nowPlays(r, g) {
const now = r.now || {};
const v = c => now[c]?.plays ?? 0;
if (g === 'any') return Object.keys(now).reduce((s, c) => s + v(c), 0);
if (g === 'p4') return P4S.reduce((s, c) => s + v(c), 0) / P4S.length;
return v(g);
}
function nowChips(now) {
now = now || {};
const on = GROUPS.filter(([g]) => g === 'p4' ? P4S.some(c => now[c]?.on) : now[g]?.on).map(([, l]) => l);
return on.length ? on.join(' · ') : '<span class="note">—</span>';
}
function chanPlays(r, g) {
if (g === 'any') return r.mean;
@@ -190,7 +204,7 @@ function chanPlays(r, g) {
}
function heat(p) {
if (p == null) return '';
const a = Math.min(1, Math.max(0, p));
const a = Math.min(1, Math.max(0, p / 0.5));
return `background: color-mix(in srgb, var(--accent) ${Math.round(a * 85)}%, transparent); color: ${a > .55 ? '#fff' : 'inherit'}`;
}
@@ -203,22 +217,24 @@ async function loadMeta() {
renderModel();
}
function windowStart(h) {
const d = new Date(meta.origin_week + 'T00:00:00Z');
d.setUTCDate(d.getUTCDate() + 7 * (h + 1));
return d.toISOString().slice(0, 10);
}
async function loadRank() {
const h = +$('h').value;
const hz = meta.horizons.find(x => x.h === h);
$('hlabel').textContent = hz ? `${fmtDate(hz.week)} (+${h} wk)` : `+${h} wk`;
$('hlabel').textContent = `${fmtDate(windowStart(h))}${h ? ` (+${h} wk)` : ' (next week)'}`;
if (!data[h]) data[h] = (await API('rank?h=' + h)).rows;
if (!data[1]) data[1] = (await API('rank?h=1')).rows;
render();
}
function render() {
const h = +$('h').value, g = $('ch').value, sort = $('sort').value, q = $('q').value.trim().toLowerCase(), dk = $('dk').checked;
const now = Object.fromEntries((data[1] || []).map(r => [r.id, r]));
let rows = data[h].filter(r => (!q || r.name.toLowerCase().includes(q)) && (!dk || r.danish));
const key = sort === 'p' ? r => [chanP(r, g) ?? 0, chanPlays(r, g) ?? 0]
: sort === 'plays' ? r => [chanPlays(r, g) ?? 0, 0]
: r => [(chanPlays(r, g) ?? 0) - (chanPlays(now[r.id] || r, g) ?? 0), 0];
: r => [(chanPlays(r, g) ?? 0) - nowPlays(r, g), 0];
rows = rows.map(r => [key(r), r]).sort((a, b) => b[0][0] - a[0][0] || b[0][1] - a[0][1]).map(x => x[1]);
current = rows;
const body = rows.slice(0, shown).map((r, i) => {
@@ -229,10 +245,11 @@ function render() {
<td class="l name">${esc(r.name)}${r.danish ? '<small>DK</small>' : ''}</td>
<td><span class="bar"><i style="width:${Math.round((p ?? 0) * 70)}px"></i>${pct(p)}</span></td>
${GROUPS.map(([c]) => `<td class="hide-sm"><span class="cell" style="${heat(chanP(r, c))}">${pct(chanP(r, c))}</span></td>`).join('')}
<td class="l hide-sm">${nowChips(r.now)}</td>
<td>${pl == null ? '—' : pl.toFixed(pl < 10 ? 1 : 0)}${range}</td>
</tr>`;
}).join('');
$('rows').innerHTML = body || '<tr><td colspan="9" class="empty">No artists match.</td></tr>';
$('rows').innerHTML = body || '<tr><td colspan="10" class="empty">No artists match.</td></tr>';
$('more').hidden = rows.length <= shown;
}
@@ -243,12 +260,13 @@ function closeDrawer() { $('drawer').classList.remove('on'); $('shade').classLis
function historyChart(a) {
// Weekly plays: P3, P4 (mean of regions), P5, P6, P8 as lines, then the forecast of total plays as a band.
const W = 700, H = 220, L = 34, R = 10, T = 10, B = 26;
const n = a.weeks.length, f = a.forecasts.length;
const fcs = a.forecasts.filter(x => x.mean != null);
const n = a.weeks.length, f = fcs.length;
const series = GROUPS.map(([g]) => g === 'p4'
? a.weeks.map((_, i) => P4S.reduce((s, c) => s + (a.plays[c]?.[i] ?? 0), 0) / P4S.length)
: (a.plays[g] || []).slice());
const total = a.weeks.map((_, i) => Object.values(a.plays).reduce((s, v) => s + (v[i] ?? 0), 0));
const maxY = Math.max(5, ...series.flat(), ...a.forecasts.map(x => (x.hi ?? 0) / 4), ...total.map(t => t / 4));
const maxY = Math.max(5, ...series.flat(), ...fcs.map(x => (x.hi ?? 0) / 4), ...total.map(t => t / 4));
const X = i => L + (W - L - R) * i / (n + f - 1);
const Y = v => T + (H - T - B) * (1 - v / maxY);
let s = `<svg viewBox="0 0 ${W} ${H}" width="100%" role="img" aria-label="Weekly plays per station">`;
@@ -260,10 +278,10 @@ function historyChart(a) {
s += `<text x="${X(n - 0.3)}" y="${T + 12}">forecast</text>`;
// forecast band of total plays scaled to "per station" (÷ 4 so it shares the axis), labelled
if (f) {
const up = a.forecasts.map((x, i) => `${X(n + i)},${Y((x.hi ?? 0) / 4)}`).join(' ');
const dn = a.forecasts.map((x, i) => `${X(n + i)},${Y((x.lo ?? 0) / 4)}`).reverse().join(' ');
const up = fcs.map((x, i) => `${X(n + i)},${Y((x.hi ?? 0) / 4)}`).join(' ');
const dn = fcs.map((x, i) => `${X(n + i)},${Y((x.lo ?? 0) / 4)}`).reverse().join(' ');
s += `<polygon points="${up} ${dn}" fill="var(--muted)" opacity=".18"/>`;
s += `<polyline points="${X(n - 1)},${Y(total[n - 1] / 4)} ${a.forecasts.map((x, i) => `${X(n + i)},${Y((x.mean ?? 0) / 4)}`).join(' ')}" fill="none" stroke="var(--muted)" stroke-width="2" stroke-dasharray="4 3"/>`;
s += `<polyline points="${X(n - 1)},${Y(total[n - 1] / 4)} ${fcs.map((x, i) => `${X(n + i)},${Y((x.mean ?? 0) / 4)}`).join(' ')}" fill="none" stroke="var(--muted)" stroke-width="2" stroke-dasharray="4 3"/>`;
s += `<polyline points="${total.map((t, i) => `${X(i)},${Y(t / 4)}`).join(' ')}" fill="none" stroke="var(--muted)" stroke-width="1.5" opacity=".6"/>`;
}
GROUPS.forEach(([g], k) => {
@@ -272,7 +290,7 @@ function historyChart(a) {
s += `<polyline points="${v.map((y, i) => `${X(i)},${Y(y)}`).join(' ')}" fill="none" stroke="${COLORS[g]}" stroke-width="2"/>`;
});
[0, Math.floor(n / 2), n - 1].forEach(i => s += `<text x="${X(i)}" y="${H - 8}" text-anchor="middle">${fmtDate(a.weeks[i])}</text>`);
if (f) s += `<text x="${X(n + f - 1)}" y="${H - 8}" text-anchor="end">${fmtDate(a.forecasts[f - 1].week)}</text>`;
if (f) s += `<text x="${X(n + f - 1)}" y="${H - 8}" text-anchor="end">${fmtDate(fcs[f - 1].week)}</text>`;
s += '</svg>';
s += `<div class="legend">${GROUPS.map(([g, l]) => `<span><i style="background:${COLORS[g]}"></i>${l}${g === 'p4' ? ' (avg region)' : ''}</span>`).join('')}<span><i style="background:var(--muted)"></i>All DR ÷ 4, forecast with 80% band</span></div>`;
return s;
@@ -284,6 +302,7 @@ async function showArtist(id) {
if (a.error) { openDrawer(`<div class="sec">${esc(a.error)}</div>`); return; }
const h = +$('h').value;
const fc = a.forecasts.find(x => x.h === h) || a.forecasts[0] || {};
const fp = a.forecasts.find(x => x.h === Math.max(h, 1)) || {};
const facts = [a.country, a.description, a.debut ? `active since ${a.debut}` : null].filter(Boolean).map(esc).join(' · ');
const runs = {};
a.charts.forEach(c => { const k = c.chart + '|' + c.title; (runs[k] ??= { ...c, weeks: 0, best: 99, first: c }); runs[k].weeks++; runs[k].best = Math.min(runs[k].best, c.position); runs[k].first = c; });
@@ -295,12 +314,12 @@ async function showArtist(id) {
<button class="x" aria-label="Close" onclick="closeDrawer()">×</button>
</div>
<div class="sec">
<h3>Week of ${fc.week ? fmtDate(fc.week) : '—'} (+${fc.h ?? h} weeks)</h3>
<h3>Chance of a new add in the 8 weeks from ${fc.window_start ? fmtDate(fc.window_start) : '—'}</h3>
<div class="kpis">
<div class="kpi"><div>Any DR station</div><b>${pct(fc.p_any)}</b></div>
<div class="kpi"><div>Any DR station</div><b>${pct(fc.add_any)}</b></div>
${GROUPS.map(([g, l]) => `<div class="kpi"><div>${l}</div><b>${pct(chanP(fc, g))}</b></div>`).join('')}
<div class="kpi"><div>Plays that week</div><b>${Math.round(fc.mean ?? 0)}</b> <span class="note">${Math.round(fc.lo ?? 0)}–${Math.round(fc.hi ?? 0)}</span></div>
</div>
<p class="note">In rotation now: ${nowChips(a.now)} · plays in the week of ${fp.week ? fmtDate(fp.week) : '—'}: <b>${Math.round(fp.mean ?? 0)}</b> (${Math.round(fp.lo ?? 0)}–${Math.round(fp.hi ?? 0)})</p>
</div>
<div class="sec"><h3>Weekly plays, last 51 weeks and next 12</h3>${historyChart(a)}</div>
<div class="sec"><h3>What if…</h3><div id="wi" class="note">Loading the model…</div></div>
@@ -313,7 +332,7 @@ async function showArtist(id) {
function whatIfUI(a, h) {
const box = $('wi');
box.innerHTML = `<div class="wi">
<div class="row"><label for="wi-h">Weeks ahead</label><input type="range" id="wi-h" min="1" max="${meta.horizons.length}" value="${h}"><b id="wi-hv"></b></div>
<div class="row"><label for="wi-h">Window starts in (weeks)</label><input type="range" id="wi-h" min="0" max="${meta.horizons.length}" value="${h}"><b id="wi-hv"></b></div>
<div class="row"><label for="wi-e">New chart entry, weeks ago</label><input type="range" id="wi-e" min="-1" max="26" value="-1"><b id="wi-ev"></b></div>
<div class="row"><label for="wi-t">Track Top-40 position now</label><input type="range" id="wi-t" min="0" max="40" value="0"><b id="wi-tv"></b></div>
<div class="row"><label for="wi-a">Airplay Top-20 position now</label><input type="range" id="wi-a" min="0" max="20" value="0"><b id="wi-av"></b></div>
@@ -342,10 +361,11 @@ function whatIfUI(a, h) {
label();
}
const d = (x, y, f) => { const v = y - x; if (Math.abs(v) < 0.005 * (f === 'p' ? 1 : 100)) return ''; return `<span class="d ${v > 0 ? 'up' : 'down'}">${v > 0 ? '+' : ''}${f === 'p' ? Math.round(v * 100) + ' pts' : Math.round(v)}</span>`; };
const rows = [['Any DR station', w.base.p_any, w.scenario.p_any]].concat(GROUPS.map(([g, l]) => [l, chanP(w.base, g), chanP(w.scenario, g)]));
$('wi-out').innerHTML = `<p class="note">Week of ${fmtDate(w.week)}. Now: ${w.current.t40 ? 'Track Top-40 #' + w.current.t40 : 'not on the Track Top-40'}, ${w.current.air ? 'Airplay Top-20 #' + w.current.air : 'not on the Airplay Top-20'}, ${w.current.entry == null ? 'no chart entry on record' : 'latest new chart entry ' + w.current.entry + ' weeks ago'}.</p>
const rows = [['Any DR station', w.base.add_any, w.scenario.add_any]].concat(GROUPS.map(([g, l]) => [l, chanP(w.base, g), chanP(w.scenario, g)]));
const onNow = Object.keys(w.current.in_rotation || {});
$('wi-out').innerHTML = `<p class="note">Chance of a new add in the 8 weeks from ${fmtDate(w.window_start)}. In rotation now on ${onNow.length ? onNow.length + ' station(s)' : 'no station'}. Now: ${w.current.t40 ? 'Track Top-40 #' + w.current.t40 : 'not on the Track Top-40'}, ${w.current.air ? 'Airplay Top-20 #' + w.current.air : 'not on the Airplay Top-20'}, ${w.current.entry == null ? 'no chart entry on record' : 'latest new chart entry ' + w.current.entry + ' weeks ago'}.</p>
<div class="kpis">${rows.map(([l, b, s]) => `<div class="kpi"><div>${l}</div><b>${pct(s)}</b>${d(b, s, 'p')}<div>was ${pct(b)}</div></div>`).join('')}
<div class="kpi"><div>Plays that week</div><b>${Math.round(w.scenario.mean)}</b>${d(w.base.mean, w.scenario.mean, 'n')}<div>was ${Math.round(w.base.mean)}</div></div></div>`;
<div class="kpi"><div>Plays, first week</div><b>${Math.round(w.scenario.mean)}</b>${d(w.base.mean, w.scenario.mean, 'n')}<div>was ${Math.round(w.base.mean)}</div></div></div>`;
};
['wi-h', 'wi-e', 'wi-t', 'wi-a'].forEach(id => $(id).addEventListener('input', () => { label(); clearTimeout(timer); timer = setTimeout(run, 120); }));
run();
@@ -355,15 +375,17 @@ function whatIfUI(a, h) {
function renderModel() {
const sc = meta.backtest || [];
const get = (m, h, k) => sc.find(x => x.model === m && x.h === h && x.metric === k)?.value;
const hs = [...new Set(sc.map(x => x.h))].sort((a, b) => a - b);
const names = { persistence: 'Same as now', popularity: 'Last 12 weeks', kalman: 'Kalman form', nb: 'Play-count model', logit: 'Rotation model' };
const hs = [...new Set(sc.filter(x => x.metric.startsWith('add')).map(x => x.h))].sort((a, b) => a - b);
const ms = [['rate by in-rotation-now', 'Rate by "in rotation now"'], ['rate by last-12-weeks level', 'Rate by last-12-weeks level'], ['logit', 'Model']];
const f = (v, d) => v == null || isNaN(v) ? '—' : v.toFixed(d);
$('cards').innerHTML = hs.map(h => {
const ms = ['persistence', 'popularity', 'kalman', 'nb', 'logit'];
return `<div class="card"><h3>${h} week${h > 1 ? 's' : ''} ahead</h3>
<table class="wtable"><thead><tr><th class="l">Model</th><th>Skill on adds<br>vs last 12 wks</th><th>Ranking<br>NDCG@50</th></tr></thead><tbody>
${ms.map(m => { const s = get(m, h, 'skill_add_vs_popularity'); return `<tr><td class="l">${names[m]}</td><td class="${s > 0 ? 'up' : s < 0 ? 'down' : ''}">${s == null ? 'baseline' : (s > 0 ? '+' : '') + s.toFixed(3)}</td><td>${(get(m, h, 'ndcg50') ?? 0).toFixed(3)}</td></tr>`; }).join('')}
</tbody></table></div>`;
}).join('') || '<p class="note">No backtest stored yet (`airplay model backtest`).</p>';
const sk = get('logit', h, 'addany_skill_vs_level'), skc = get('logit', h, 'add_skill_vs_level');
return `<div class="card"><h3>Window starting ${h ? h + ' weeks ahead' : 'now'}</h3>
<table class="wtable"><thead><tr><th class="l"></th><th>Any station<br>log-loss</th><th>Any station<br>PR-AUC</th><th>Per station<br>PR-AUC</th></tr></thead><tbody>
${ms.map(([m, l]) => `<tr><td class="l">${l}</td><td>${f(get(m, h, 'addany_logloss'), 3)}</td><td>${f(get(m, h, 'addany_prauc'), 3)}</td><td>${f(get(m, h, 'add_prauc'), 3)}</td></tr>`).join('')}
</tbody></table>
<p>Skill vs the last-12-weeks baseline: <b class="${sk > 0 ? 'up' : 'down'}">${sk > 0 ? '+' : ''}${f(sk, 3)}</b> any station, <b class="${skc > 0 ? 'up' : 'down'}">${skc > 0 ? '+' : ''}${f(skc, 3)}</b> per station. Lower log-loss and higher PR-AUC are better.</p></div>`;
}).join('') || '<p class="note">No add backtest stored yet (`airplay model adds`).</p>';
}
// ---------- wiring ----------