//! Walk-forward backtest on the DR bed (doc: "Two backtest beds"). For each origin week t0 from //! week 26 on, every model is fitted on what was known at the end of t0 and forecasts week //! t0 + h for every artist × channel. Scored against the doc's baselines: //! //! persistence P(in rotation at t0+h | in rotation at t0), per channel; plays carried forward //! popularity P(in rotation) by bins of the artist's last-12-week mean; that mean as plays //! kalman local level on log plays (artist form, first version) //! nb direct h-step negative binomial on the panel features //! logit direct h-step logistic model of "in rotation" //! //! ("Release timing only" needs release dates, which arrive with the Deezer backfill.) use super::common::{thresholds, Direct, Rng}; use super::features::{LAGS, P}; use super::kalman::{self, LocalLevel}; use super::panel::{Panel, C, CHANNELS}; use anyhow::Result; use std::fmt::Write as _; pub const MODELS: [&str; 5] = ["persistence", "popularity", "kalman", "nb", "logit"]; const M: usize = MODELS.len(); const START: usize = 26; /// One model's scores in one forecast week. #[derive(Clone, Default)] struct Week { ll: f64, brier: f64, n: f64, // all rows: "in rotation" at t0 + h ll_add: f64, brier_add: f64, n_add: f64, // rows not in rotation at t0 (adds) abs_err: f64, // weekly plays spearman: f64, ndcg: f64, // artists ranked by total plays across channels } fn ll(p: f64, y: bool) -> f64 { -(if y { p } else { 1.0 - p }).clamp(1e-9, 1.0).ln() } fn ranks(v: &[f64]) -> Vec { let mut idx: Vec = (0..v.len()).collect(); idx.sort_by(|&a, &b| v[a].total_cmp(&v[b])); let mut r = vec![0.0; v.len()]; let mut i = 0; while i < idx.len() { let mut j = i; while j + 1 < idx.len() && v[idx[j + 1]] == v[idx[i]] { j += 1; } for k in i..=j { r[idx[k]] = (i + j) as f64 / 2.0; } i = j + 1; } r } pub fn spearman(a: &[f64], b: &[f64]) -> f64 { let (ra, rb) = (ranks(a), ranks(b)); let n = a.len() as f64; let (ma, mb) = (ra.iter().sum::() / n, rb.iter().sum::() / n); let (mut sab, mut saa, mut sbb) = (0.0, 0.0, 0.0); for i in 0..a.len() { let (x, y) = (ra[i] - ma, rb[i] - mb); sab += x * y; saa += x * x; sbb += y * y; } sab / (saa * sbb).sqrt().max(1e-12) } pub fn ndcg(pred: &[f64], truth: &[f64], k: usize) -> f64 { let dcg = |order: &[usize]| order.iter().take(k).enumerate().map(|(i, &j)| truth[j] / (i as f64 + 2.0).log2()).sum::(); let mut by_pred: Vec = (0..pred.len()).collect(); by_pred.sort_by(|&a, &b| pred[b].total_cmp(&pred[a])); let mut ideal: Vec = (0..truth.len()).collect(); ideal.sort_by(|&a, &b| truth[b].total_cmp(&truth[a])); dcg(&by_pred) / dcg(&ideal).max(1e-12) } fn average_precision(mut v: Vec<(f64, bool)>) -> f64 { v.sort_by(|a, b| b.0.total_cmp(&a.0)); let (mut hits, mut sum) = (0.0, 0.0); for (i, (_, y)) in v.iter().enumerate() { if *y { hits += 1.0; sum += hits / (i as f64 + 1.0); } } if hits > 0.0 { sum / hits } else { f64::NAN } } /// Skill 1 - loss(model) / loss(base) over all weeks, with a moving-block bootstrap (blocks of 4 /// weeks) 90% interval. fn skill(model: &[f64], base: &[f64]) -> (f64, f64, f64) { let point = 1.0 - model.iter().sum::() / base.iter().sum::(); let n = model.len(); // Blocks of 8 weeks: neighbouring origins share most of an 8-week target window. let block = 8.min(n); let mut rng = Rng(42); let mut draws: Vec = (0..2000).map(|_| { let (mut lm, mut lb, mut k) = (0.0, 0.0, 0); while k < n { let s = (rng.u() * (n - block + 1) as f64) as usize; for i in s..s + block { if k < n { lm += model[i]; lb += base[i]; k += 1; } } } 1.0 - lm / lb }).collect(); draws.sort_by(f64::total_cmp); (point, draws[100], draws[1899]) } struct Baselines { persist: [[f64; 2]; C], popular: [[f64; 10]; C] } fn pop_bin(m12: f64) -> usize { const EDGES: [f64; 9] = [0.05, 0.3, 0.6, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5]; EDGES.iter().take_while(|&&e| m12.ln_1p() >= e).count() } fn mean12(p: &Panel, a: usize, c: usize, t: usize) -> f64 { (0..LAGS).map(|i| p.y(a, c, t - i)).sum::() / LAGS as f64 } fn fit_baselines(p: &Panel, h: usize, t0: usize, tau: &[u32; C]) -> Baselines { let mut pc = [[[1.0f64, 2.0]; 2]; C]; // [hits, n] with a Laplace-style prior let mut bc = [[[1.0f64, 2.0]; 10]; C]; for t in LAGS - 1..=t0 - h { for a in 0..p.a() { for c in 0..C { let now = p.y(a, c, t) >= tau[c] as f64; let later = (p.y(a, c, t + h) >= tau[c] as f64) as i32 as f64; pc[c][now as usize][0] += later; pc[c][now as usize][1] += 1.0; let b = pop_bin(mean12(p, a, c, t)); bc[c][b][0] += later; bc[c][b][1] += 1.0; } } } let mut out = Baselines { persist: [[0.0; 2]; C], popular: [[0.0; 10]; C] }; for c in 0..C { for s in 0..2 { out.persist[c][s] = pc[c][s][0] / pc[c][s][1]; } for b in 0..10 { out.popular[c][b] = bc[c][b][0] / bc[c][b][1]; } } out } /// Kalman level per artist × channel after week t0, with each channel's (q, r). fn kalman_states(p: &Panel, t0: usize, models: &[LocalLevel; C]) -> Vec<(f64, f64)> { let mut z = vec![0.0; t0 + 1]; let mut out = Vec::with_capacity(p.a() * C); for a in 0..p.a() { for c in 0..C { for (w, v) in z.iter_mut().enumerate() { *v = p.y(a, c, w).ln_1p(); } let (l, v, _) = kalman::filter(&z, models[c]); out.push((l, v)); } } out } fn fit_kalman(p: &Panel, t0: usize) -> [LocalLevel; C] { let mut out = [LocalLevel { q: 0.05, r: 0.3 }; C]; for (c, m) in out.iter_mut().enumerate() { let series: Vec> = (0..p.a()) .map(|a| (0..=t0).map(|w| p.y(a, c, w).ln_1p()).collect::>()) .filter(|z| z.iter().any(|&v| v > 0.0)) .collect(); *m = kalman::fit(series.iter().map(|v| v.as_slice())); } out } pub struct Report { pub markdown: String, pub rows: Vec<(String, usize, String, f64)> } /// `refit`: refit the regressions every this many origins (1 = weekly, as the doc's protocol). pub fn run(p: &Panel, horizons: &[usize], refit: usize) -> Result { let tau = thresholds(p, START); let mut md = String::new(); let mut rows = Vec::new(); writeln!(md, "# Backtest: DR artist × channel × week\n")?; writeln!(md, "Panel: {} artists × {} channels × {} complete weeks ({} to {}). Forecast origins from week {}; regressions refitted every {} origin(s).\n", p.a(), C, p.w(), p.weeks[0], p.weeks[p.w() - 1], START + 1, refit)?; writeln!(md, "\"In rotation\" = at least τ plays in the week: {}.\n", CHANNELS.iter().zip(tau).map(|(c, t)| format!("{c} {t}")).collect::>().join(", "))?; let mut x = vec![0.0; P]; for &h in horizons { if START + h >= p.w() { continue; } let origins: Vec = (START..p.w() - h).collect(); let mut weeks: Vec<[Week; M]> = Vec::new(); let mut add_pairs: [Vec<(f64, bool)>; M] = Default::default(); let mut calib = [[0.0f64; 3]; 10]; // nb adds: [sum p, hits, n] per decile of p let mut direct: Option = None; let mut kal = [LocalLevel { q: 0.05, r: 0.3 }; C]; for (i, &t0) in origins.iter().enumerate() { if i % refit == 0 || direct.is_none() { direct = Some(Direct::fit(p, h, t0, &tau, true)); kal = fit_kalman(p, t0); } let d = direct.as_ref().unwrap(); let base = fit_baselines(p, h, t0, &tau); let ks = kalman_states(p, t0, &kal); let target = t0 + h; let mut wk: [Week; M] = Default::default(); let mut totals = vec![[0.0f64; M]; p.a()]; let mut truth = vec![0.0f64; p.a()]; for a in 0..p.a() { for c in 0..C { let y = p.y(a, c, target); let rot = y >= tau[c] as f64; let now = p.y(a, c, t0) >= tau[c] as f64; let m12 = mean12(p, a, c, t0); let (level, var) = ks[a * C + c]; let (km, kv) = kalman::forecast(level, var, kal[c], h); let (mu, p_nb, p_lg) = d.predict(p, a, c, t0, tau[c], &mut x); let preds: [(f64, f64); M] = [ (base.persist[c][now as usize], p.y(a, c, t0)), (base.popular[c][pop_bin(m12)], m12), (kalman::norm_sf((((tau[c] as f64) - 0.5).ln_1p() - km) / kv.sqrt()).clamp(1e-9, 1.0 - 1e-9), ((km + kv / 2.0).exp() - 1.0).max(0.0)), (p_nb, mu), (p_lg, mu), ]; truth[a] += y; for (m, (pr, mean)) in preds.iter().enumerate() { let s = &mut wk[m]; s.ll += ll(*pr, rot); s.brier += (pr - rot as i32 as f64).powi(2); s.n += 1.0; if !now { s.ll_add += ll(*pr, rot); s.brier_add += (pr - rot as i32 as f64).powi(2); s.n_add += 1.0; add_pairs[m].push((*pr, rot)); } s.abs_err += (mean - y).abs(); totals[a][m] += mean; } if !now { let dec = ((p_nb * 10.0) as usize).min(9); calib[dec][0] += p_nb; calib[dec][1] += rot as i32 as f64; calib[dec][2] += 1.0; } } } for m in 0..M { let pred: Vec = totals.iter().map(|t| t[m]).collect(); wk[m].spearman = spearman(&pred, &truth); wk[m].ndcg = ndcg(&pred, &truth, 50); } weeks.push(wk); eprintln!("backtest h={h}: origin {} done ({}/{})", p.weeks[t0], i + 1, origins.len()); } let nw = weeks.len(); writeln!(md, "## h = {h} week(s) ahead ({nw} forecast weeks)\n")?; writeln!(md, "| Model | Log-loss, all | Log-loss, adds | Brier, adds | PR-AUC, adds | MAE plays | Spearman | NDCG@50 |")?; writeln!(md, "|---|---|---|---|---|---|---|---|")?; for m in 0..M { let sum = |f: &dyn Fn(&Week) -> f64| weeks.iter().map(|w| f(&w[m])).sum::(); let (n, na) = (sum(&|w| w.n), sum(&|w| w.n_add)); let vals = [ sum(&|w| w.ll) / n, sum(&|w| w.ll_add) / na, sum(&|w| w.brier_add) / na, average_precision(add_pairs[m].clone()), sum(&|w| w.abs_err) / n, sum(&|w| w.spearman) / nw as f64, sum(&|w| w.ndcg) / nw as f64, ]; let names = ["logloss_all", "logloss_add", "brier_add", "prauc_add", "mae", "spearman", "ndcg50"]; for (k, v) in vals.iter().enumerate() { rows.push((MODELS[m].to_string(), h, names[k].to_string(), *v)); } writeln!(md, "| {} | {:.4} | {:.4} | {:.4} | {:.3} | {:.2} | {:.3} | {:.3} |", MODELS[m], vals[0], vals[1], vals[2], vals[3], vals[4], vals[5], vals[6])?; } writeln!(md, "\nSkill on adds (1 − log-loss ÷ baseline log-loss; 90% block-bootstrap interval over weeks):\n")?; writeln!(md, "| Model | vs persistence | vs popularity |")?; writeln!(md, "|---|---|---|")?; for m in 2..M { let l = |k: usize| weeks.iter().map(|w| w[k].ll_add).collect::>(); let (a, b) = (skill(&l(m), &l(0)), skill(&l(m), &l(1))); rows.push((MODELS[m].to_string(), h, "skill_add_vs_persistence".into(), a.0)); rows.push((MODELS[m].to_string(), h, "skill_add_vs_popularity".into(), b.0)); writeln!(md, "| {} | {:.3} ({:.3} to {:.3}) | {:.3} ({:.3} to {:.3}) |", MODELS[m], a.0, a.1, a.2, b.0, b.1, b.2)?; } writeln!(md, "\nCalibration of the NB's add probabilities (rows not in rotation at the origin):\n")?; writeln!(md, "| Predicted | Rows | Mean predicted | Observed |")?; writeln!(md, "|---|---|---|---|")?; for (dec, c) in calib.iter().enumerate() { if c[2] == 0.0 { continue; } writeln!(md, "| {:.1}–{:.1} | {} | {:.3} | {:.3} |", dec as f64 / 10.0, (dec + 1) as f64 / 10.0, c[2], c[0] / c[2], c[1] / c[2])?; } writeln!(md)?; if nw < 8 { writeln!(md, "Fewer than 8 forecast weeks: no verdict at this horizon (doc rule).\n")?; } } { let d = Direct::fit(p, 4, p.w() - 1, &tau, false); writeln!(md, "## NB coefficients, h = 4, all weeks (alpha = {:.3})\n", d.alpha)?; writeln!(md, "| Feature | Coefficient |\n|---|---|")?; for (n, b) in super::features::NAMES.iter().zip(&d.nb) { writeln!(md, "| {n} | {b:+.3} |")?; } writeln!(md)?; } Ok(Report { markdown: md, rows }) }