Models, first layer: artist × DR channel × week panel with IFPI chart history; NB and logit direct h-step models, Kalman artist form, persistence/popularity baselines; walk-forward backtest (docs/BACKTEST.md) and the ranking (forecasts table)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,270 @@
|
||||
//! Walk-forward backtest on the DR bed (doc: "Two backtest beds"). For each origin week t0 from
|
||||
//! week 26 on, every model is fitted on what was known at the end of t0 and forecasts week
|
||||
//! t0 + h for every artist × channel. Scored against the doc's baselines:
|
||||
//!
|
||||
//! persistence P(in rotation at t0+h | in rotation at t0), per channel; plays carried forward
|
||||
//! popularity P(in rotation) by bins of the artist's last-12-week mean; that mean as plays
|
||||
//! kalman local level on log plays (artist form, first version)
|
||||
//! nb direct h-step negative binomial on the panel features
|
||||
//! logit direct h-step logistic model of "in rotation"
|
||||
//!
|
||||
//! ("Release timing only" needs release dates, which arrive with the Deezer backfill.)
|
||||
|
||||
use super::common::{thresholds, Direct, Rng};
|
||||
use super::features::{LAGS, P};
|
||||
use super::kalman::{self, LocalLevel};
|
||||
use super::panel::{Panel, C, CHANNELS};
|
||||
use anyhow::Result;
|
||||
use std::fmt::Write as _;
|
||||
|
||||
pub const MODELS: [&str; 5] = ["persistence", "popularity", "kalman", "nb", "logit"];
|
||||
const M: usize = MODELS.len();
|
||||
const START: usize = 26;
|
||||
|
||||
/// One model's scores in one forecast week.
|
||||
#[derive(Clone, Default)]
|
||||
struct Week {
|
||||
ll: f64, brier: f64, n: f64, // all rows: "in rotation" at t0 + h
|
||||
ll_add: f64, brier_add: f64, n_add: f64, // rows not in rotation at t0 (adds)
|
||||
abs_err: f64, // weekly plays
|
||||
spearman: f64, ndcg: f64, // artists ranked by total plays across channels
|
||||
}
|
||||
|
||||
fn ll(p: f64, y: bool) -> f64 { -(if y { p } else { 1.0 - p }).clamp(1e-9, 1.0).ln() }
|
||||
|
||||
fn ranks(v: &[f64]) -> Vec<f64> {
|
||||
let mut idx: Vec<usize> = (0..v.len()).collect();
|
||||
idx.sort_by(|&a, &b| v[a].total_cmp(&v[b]));
|
||||
let mut r = vec![0.0; v.len()];
|
||||
let mut i = 0;
|
||||
while i < idx.len() {
|
||||
let mut j = i;
|
||||
while j + 1 < idx.len() && v[idx[j + 1]] == v[idx[i]] { j += 1; }
|
||||
for k in i..=j { r[idx[k]] = (i + j) as f64 / 2.0; }
|
||||
i = j + 1;
|
||||
}
|
||||
r
|
||||
}
|
||||
|
||||
pub fn spearman(a: &[f64], b: &[f64]) -> f64 {
|
||||
let (ra, rb) = (ranks(a), ranks(b));
|
||||
let n = a.len() as f64;
|
||||
let (ma, mb) = (ra.iter().sum::<f64>() / n, rb.iter().sum::<f64>() / n);
|
||||
let (mut sab, mut saa, mut sbb) = (0.0, 0.0, 0.0);
|
||||
for i in 0..a.len() { let (x, y) = (ra[i] - ma, rb[i] - mb); sab += x * y; saa += x * x; sbb += y * y; }
|
||||
sab / (saa * sbb).sqrt().max(1e-12)
|
||||
}
|
||||
|
||||
pub fn ndcg(pred: &[f64], truth: &[f64], k: usize) -> f64 {
|
||||
let dcg = |order: &[usize]| order.iter().take(k).enumerate().map(|(i, &j)| truth[j] / (i as f64 + 2.0).log2()).sum::<f64>();
|
||||
let mut by_pred: Vec<usize> = (0..pred.len()).collect();
|
||||
by_pred.sort_by(|&a, &b| pred[b].total_cmp(&pred[a]));
|
||||
let mut ideal: Vec<usize> = (0..truth.len()).collect();
|
||||
ideal.sort_by(|&a, &b| truth[b].total_cmp(&truth[a]));
|
||||
dcg(&by_pred) / dcg(&ideal).max(1e-12)
|
||||
}
|
||||
|
||||
fn average_precision(mut v: Vec<(f64, bool)>) -> f64 {
|
||||
v.sort_by(|a, b| b.0.total_cmp(&a.0));
|
||||
let (mut hits, mut sum) = (0.0, 0.0);
|
||||
for (i, (_, y)) in v.iter().enumerate() { if *y { hits += 1.0; sum += hits / (i as f64 + 1.0); } }
|
||||
if hits > 0.0 { sum / hits } else { f64::NAN }
|
||||
}
|
||||
|
||||
/// Skill 1 - loss(model) / loss(base) over all weeks, with a moving-block bootstrap (blocks of 4
|
||||
/// weeks) 90% interval.
|
||||
fn skill(model: &[f64], base: &[f64]) -> (f64, f64, f64) {
|
||||
let point = 1.0 - model.iter().sum::<f64>() / base.iter().sum::<f64>();
|
||||
let n = model.len();
|
||||
let block = 4.min(n);
|
||||
let mut rng = Rng(42);
|
||||
let mut draws: Vec<f64> = (0..2000).map(|_| {
|
||||
let (mut lm, mut lb, mut k) = (0.0, 0.0, 0);
|
||||
while k < n {
|
||||
let s = (rng.u() * (n - block + 1) as f64) as usize;
|
||||
for i in s..s + block { if k < n { lm += model[i]; lb += base[i]; k += 1; } }
|
||||
}
|
||||
1.0 - lm / lb
|
||||
}).collect();
|
||||
draws.sort_by(f64::total_cmp);
|
||||
(point, draws[100], draws[1899])
|
||||
}
|
||||
|
||||
struct Baselines { persist: [[f64; 2]; C], popular: [[f64; 10]; C] }
|
||||
|
||||
fn pop_bin(m12: f64) -> usize {
|
||||
const EDGES: [f64; 9] = [0.05, 0.3, 0.6, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5];
|
||||
EDGES.iter().take_while(|&&e| m12.ln_1p() >= e).count()
|
||||
}
|
||||
|
||||
fn mean12(p: &Panel, a: usize, c: usize, t: usize) -> f64 { (0..LAGS).map(|i| p.y(a, c, t - i)).sum::<f64>() / LAGS as f64 }
|
||||
|
||||
fn fit_baselines(p: &Panel, h: usize, t0: usize, tau: &[u32; C]) -> Baselines {
|
||||
let mut pc = [[[1.0f64, 2.0]; 2]; C]; // [hits, n] with a Laplace-style prior
|
||||
let mut bc = [[[1.0f64, 2.0]; 10]; C];
|
||||
for t in LAGS - 1..=t0 - h {
|
||||
for a in 0..p.a() {
|
||||
for c in 0..C {
|
||||
let now = p.y(a, c, t) >= tau[c] as f64;
|
||||
let later = (p.y(a, c, t + h) >= tau[c] as f64) as i32 as f64;
|
||||
pc[c][now as usize][0] += later; pc[c][now as usize][1] += 1.0;
|
||||
let b = pop_bin(mean12(p, a, c, t));
|
||||
bc[c][b][0] += later; bc[c][b][1] += 1.0;
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut out = Baselines { persist: [[0.0; 2]; C], popular: [[0.0; 10]; C] };
|
||||
for c in 0..C {
|
||||
for s in 0..2 { out.persist[c][s] = pc[c][s][0] / pc[c][s][1]; }
|
||||
for b in 0..10 { out.popular[c][b] = bc[c][b][0] / bc[c][b][1]; }
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Kalman level per artist × channel after week t0, with each channel's (q, r).
|
||||
fn kalman_states(p: &Panel, t0: usize, models: &[LocalLevel; C]) -> Vec<(f64, f64)> {
|
||||
let mut z = vec![0.0; t0 + 1];
|
||||
let mut out = Vec::with_capacity(p.a() * C);
|
||||
for a in 0..p.a() {
|
||||
for c in 0..C {
|
||||
for (w, v) in z.iter_mut().enumerate() { *v = p.y(a, c, w).ln_1p(); }
|
||||
let (l, v, _) = kalman::filter(&z, models[c]);
|
||||
out.push((l, v));
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn fit_kalman(p: &Panel, t0: usize) -> [LocalLevel; C] {
|
||||
let mut out = [LocalLevel { q: 0.05, r: 0.3 }; C];
|
||||
for (c, m) in out.iter_mut().enumerate() {
|
||||
let series: Vec<Vec<f64>> = (0..p.a())
|
||||
.map(|a| (0..=t0).map(|w| p.y(a, c, w).ln_1p()).collect::<Vec<f64>>())
|
||||
.filter(|z| z.iter().any(|&v| v > 0.0))
|
||||
.collect();
|
||||
*m = kalman::fit(series.iter().map(|v| v.as_slice()));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
pub struct Report { pub markdown: String, pub rows: Vec<(String, usize, String, f64)> }
|
||||
|
||||
/// `refit`: refit the regressions every this many origins (1 = weekly, as the doc's protocol).
|
||||
pub fn run(p: &Panel, horizons: &[usize], refit: usize) -> Result<Report> {
|
||||
let tau = thresholds(p, START);
|
||||
let mut md = String::new();
|
||||
let mut rows = Vec::new();
|
||||
writeln!(md, "# Backtest: DR artist × channel × week\n")?;
|
||||
writeln!(md, "Panel: {} artists × {} channels × {} complete weeks ({} to {}). Forecast origins from week {}; regressions refitted every {} origin(s).\n",
|
||||
p.a(), C, p.w(), p.weeks[0], p.weeks[p.w() - 1], START + 1, refit)?;
|
||||
writeln!(md, "\"In rotation\" = at least τ plays in the week: {}.\n",
|
||||
CHANNELS.iter().zip(tau).map(|(c, t)| format!("{c} {t}")).collect::<Vec<_>>().join(", "))?;
|
||||
let mut x = vec![0.0; P];
|
||||
for &h in horizons {
|
||||
if START + h >= p.w() { continue; }
|
||||
let origins: Vec<usize> = (START..p.w() - h).collect();
|
||||
let mut weeks: Vec<[Week; M]> = Vec::new();
|
||||
let mut add_pairs: [Vec<(f64, bool)>; M] = Default::default();
|
||||
let mut calib = [[0.0f64; 3]; 10]; // nb adds: [sum p, hits, n] per decile of p
|
||||
let mut direct: Option<Direct> = None;
|
||||
let mut kal = [LocalLevel { q: 0.05, r: 0.3 }; C];
|
||||
for (i, &t0) in origins.iter().enumerate() {
|
||||
if i % refit == 0 || direct.is_none() {
|
||||
direct = Some(Direct::fit(p, h, t0, &tau, true));
|
||||
kal = fit_kalman(p, t0);
|
||||
}
|
||||
let d = direct.as_ref().unwrap();
|
||||
let base = fit_baselines(p, h, t0, &tau);
|
||||
let ks = kalman_states(p, t0, &kal);
|
||||
let target = t0 + h;
|
||||
let mut wk: [Week; M] = Default::default();
|
||||
let mut totals = vec![[0.0f64; M]; p.a()];
|
||||
let mut truth = vec![0.0f64; p.a()];
|
||||
for a in 0..p.a() {
|
||||
for c in 0..C {
|
||||
let y = p.y(a, c, target);
|
||||
let rot = y >= tau[c] as f64;
|
||||
let now = p.y(a, c, t0) >= tau[c] as f64;
|
||||
let m12 = mean12(p, a, c, t0);
|
||||
let (level, var) = ks[a * C + c];
|
||||
let (km, kv) = kalman::forecast(level, var, kal[c], h);
|
||||
let (mu, p_nb, p_lg) = d.predict(p, a, c, t0, tau[c], &mut x);
|
||||
let preds: [(f64, f64); M] = [
|
||||
(base.persist[c][now as usize], p.y(a, c, t0)),
|
||||
(base.popular[c][pop_bin(m12)], m12),
|
||||
(kalman::norm_sf((((tau[c] as f64) - 0.5).ln_1p() - km) / kv.sqrt()).clamp(1e-9, 1.0 - 1e-9), ((km + kv / 2.0).exp() - 1.0).max(0.0)),
|
||||
(p_nb, mu),
|
||||
(p_lg, mu),
|
||||
];
|
||||
truth[a] += y;
|
||||
for (m, (pr, mean)) in preds.iter().enumerate() {
|
||||
let s = &mut wk[m];
|
||||
s.ll += ll(*pr, rot); s.brier += (pr - rot as i32 as f64).powi(2); s.n += 1.0;
|
||||
if !now {
|
||||
s.ll_add += ll(*pr, rot); s.brier_add += (pr - rot as i32 as f64).powi(2); s.n_add += 1.0;
|
||||
add_pairs[m].push((*pr, rot));
|
||||
}
|
||||
s.abs_err += (mean - y).abs();
|
||||
totals[a][m] += mean;
|
||||
}
|
||||
if !now {
|
||||
let dec = ((p_nb * 10.0) as usize).min(9);
|
||||
calib[dec][0] += p_nb; calib[dec][1] += rot as i32 as f64; calib[dec][2] += 1.0;
|
||||
}
|
||||
}
|
||||
}
|
||||
for m in 0..M {
|
||||
let pred: Vec<f64> = totals.iter().map(|t| t[m]).collect();
|
||||
wk[m].spearman = spearman(&pred, &truth);
|
||||
wk[m].ndcg = ndcg(&pred, &truth, 50);
|
||||
}
|
||||
weeks.push(wk);
|
||||
eprintln!("backtest h={h}: origin {} done ({}/{})", p.weeks[t0], i + 1, origins.len());
|
||||
}
|
||||
|
||||
let nw = weeks.len();
|
||||
writeln!(md, "## h = {h} week(s) ahead ({nw} forecast weeks)\n")?;
|
||||
writeln!(md, "| Model | Log-loss, all | Log-loss, adds | Brier, adds | PR-AUC, adds | MAE plays | Spearman | NDCG@50 |")?;
|
||||
writeln!(md, "|---|---|---|---|---|---|---|---|")?;
|
||||
for m in 0..M {
|
||||
let sum = |f: &dyn Fn(&Week) -> f64| weeks.iter().map(|w| f(&w[m])).sum::<f64>();
|
||||
let (n, na) = (sum(&|w| w.n), sum(&|w| w.n_add));
|
||||
let vals = [
|
||||
sum(&|w| w.ll) / n, sum(&|w| w.ll_add) / na, sum(&|w| w.brier_add) / na,
|
||||
average_precision(add_pairs[m].clone()), sum(&|w| w.abs_err) / n,
|
||||
sum(&|w| w.spearman) / nw as f64, sum(&|w| w.ndcg) / nw as f64,
|
||||
];
|
||||
let names = ["logloss_all", "logloss_add", "brier_add", "prauc_add", "mae", "spearman", "ndcg50"];
|
||||
for (k, v) in vals.iter().enumerate() { rows.push((MODELS[m].to_string(), h, names[k].to_string(), *v)); }
|
||||
writeln!(md, "| {} | {:.4} | {:.4} | {:.4} | {:.3} | {:.2} | {:.3} | {:.3} |", MODELS[m],
|
||||
vals[0], vals[1], vals[2], vals[3], vals[4], vals[5], vals[6])?;
|
||||
}
|
||||
writeln!(md, "\nSkill on adds (1 − log-loss ÷ baseline log-loss; 90% block-bootstrap interval over weeks):\n")?;
|
||||
writeln!(md, "| Model | vs persistence | vs popularity |")?;
|
||||
writeln!(md, "|---|---|---|")?;
|
||||
for m in 2..M {
|
||||
let l = |k: usize| weeks.iter().map(|w| w[k].ll_add).collect::<Vec<f64>>();
|
||||
let (a, b) = (skill(&l(m), &l(0)), skill(&l(m), &l(1)));
|
||||
rows.push((MODELS[m].to_string(), h, "skill_add_vs_persistence".into(), a.0));
|
||||
rows.push((MODELS[m].to_string(), h, "skill_add_vs_popularity".into(), b.0));
|
||||
writeln!(md, "| {} | {:.3} ({:.3} to {:.3}) | {:.3} ({:.3} to {:.3}) |", MODELS[m], a.0, a.1, a.2, b.0, b.1, b.2)?;
|
||||
}
|
||||
writeln!(md, "\nCalibration of the NB's add probabilities (rows not in rotation at the origin):\n")?;
|
||||
writeln!(md, "| Predicted | Rows | Mean predicted | Observed |")?;
|
||||
writeln!(md, "|---|---|---|---|")?;
|
||||
for (dec, c) in calib.iter().enumerate() {
|
||||
if c[2] == 0.0 { continue; }
|
||||
writeln!(md, "| {:.1}–{:.1} | {} | {:.3} | {:.3} |", dec as f64 / 10.0, (dec + 1) as f64 / 10.0, c[2], c[0] / c[2], c[1] / c[2])?;
|
||||
}
|
||||
writeln!(md)?;
|
||||
if nw < 8 { writeln!(md, "Fewer than 8 forecast weeks: no verdict at this horizon (doc rule).\n")?; }
|
||||
}
|
||||
{
|
||||
let d = Direct::fit(p, 4, p.w() - 1, &tau, false);
|
||||
writeln!(md, "## NB coefficients, h = 4, all weeks (alpha = {:.3})\n", d.alpha)?;
|
||||
writeln!(md, "| Feature | Coefficient |\n|---|---|")?;
|
||||
for (n, b) in super::features::NAMES.iter().zip(&d.nb) { writeln!(md, "| {n} | {b:+.3} |")?; }
|
||||
writeln!(md)?;
|
||||
}
|
||||
Ok(Report { markdown: md, rows })
|
||||
}
|
||||
Reference in New Issue
Block a user