Models, first layer: artist × DR channel × week panel with IFPI chart history; NB and logit direct h-step models, Kalman artist form, persistence/popularity baselines; walk-forward backtest (docs/BACKTEST.md) and the ranking (forecasts table)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-15 02:08:48 +00:00
co-authored by Claude Opus 5.5
parent 6272fbd75d
commit eededc7571
12 changed files with 1318 additions and 1 deletions
+270
View File
@@ -0,0 +1,270 @@
//! Walk-forward backtest on the DR bed (doc: "Two backtest beds"). For each origin week t0 from
//! week 26 on, every model is fitted on what was known at the end of t0 and forecasts week
//! t0 + h for every artist × channel. Scored against the doc's baselines:
//!
//! persistence P(in rotation at t0+h | in rotation at t0), per channel; plays carried forward
//! popularity P(in rotation) by bins of the artist's last-12-week mean; that mean as plays
//! kalman local level on log plays (artist form, first version)
//! nb direct h-step negative binomial on the panel features
//! logit direct h-step logistic model of "in rotation"
//!
//! ("Release timing only" needs release dates, which arrive with the Deezer backfill.)
use super::common::{thresholds, Direct, Rng};
use super::features::{LAGS, P};
use super::kalman::{self, LocalLevel};
use super::panel::{Panel, C, CHANNELS};
use anyhow::Result;
use std::fmt::Write as _;
pub const MODELS: [&str; 5] = ["persistence", "popularity", "kalman", "nb", "logit"];
const M: usize = MODELS.len();
const START: usize = 26;
/// One model's scores in one forecast week.
#[derive(Clone, Default)]
struct Week {
ll: f64, brier: f64, n: f64, // all rows: "in rotation" at t0 + h
ll_add: f64, brier_add: f64, n_add: f64, // rows not in rotation at t0 (adds)
abs_err: f64, // weekly plays
spearman: f64, ndcg: f64, // artists ranked by total plays across channels
}
fn ll(p: f64, y: bool) -> f64 { -(if y { p } else { 1.0 - p }).clamp(1e-9, 1.0).ln() }
fn ranks(v: &[f64]) -> Vec<f64> {
let mut idx: Vec<usize> = (0..v.len()).collect();
idx.sort_by(|&a, &b| v[a].total_cmp(&v[b]));
let mut r = vec![0.0; v.len()];
let mut i = 0;
while i < idx.len() {
let mut j = i;
while j + 1 < idx.len() && v[idx[j + 1]] == v[idx[i]] { j += 1; }
for k in i..=j { r[idx[k]] = (i + j) as f64 / 2.0; }
i = j + 1;
}
r
}
pub fn spearman(a: &[f64], b: &[f64]) -> f64 {
let (ra, rb) = (ranks(a), ranks(b));
let n = a.len() as f64;
let (ma, mb) = (ra.iter().sum::<f64>() / n, rb.iter().sum::<f64>() / n);
let (mut sab, mut saa, mut sbb) = (0.0, 0.0, 0.0);
for i in 0..a.len() { let (x, y) = (ra[i] - ma, rb[i] - mb); sab += x * y; saa += x * x; sbb += y * y; }
sab / (saa * sbb).sqrt().max(1e-12)
}
pub fn ndcg(pred: &[f64], truth: &[f64], k: usize) -> f64 {
let dcg = |order: &[usize]| order.iter().take(k).enumerate().map(|(i, &j)| truth[j] / (i as f64 + 2.0).log2()).sum::<f64>();
let mut by_pred: Vec<usize> = (0..pred.len()).collect();
by_pred.sort_by(|&a, &b| pred[b].total_cmp(&pred[a]));
let mut ideal: Vec<usize> = (0..truth.len()).collect();
ideal.sort_by(|&a, &b| truth[b].total_cmp(&truth[a]));
dcg(&by_pred) / dcg(&ideal).max(1e-12)
}
fn average_precision(mut v: Vec<(f64, bool)>) -> f64 {
v.sort_by(|a, b| b.0.total_cmp(&a.0));
let (mut hits, mut sum) = (0.0, 0.0);
for (i, (_, y)) in v.iter().enumerate() { if *y { hits += 1.0; sum += hits / (i as f64 + 1.0); } }
if hits > 0.0 { sum / hits } else { f64::NAN }
}
/// Skill 1 - loss(model) / loss(base) over all weeks, with a moving-block bootstrap (blocks of 4
/// weeks) 90% interval.
fn skill(model: &[f64], base: &[f64]) -> (f64, f64, f64) {
let point = 1.0 - model.iter().sum::<f64>() / base.iter().sum::<f64>();
let n = model.len();
let block = 4.min(n);
let mut rng = Rng(42);
let mut draws: Vec<f64> = (0..2000).map(|_| {
let (mut lm, mut lb, mut k) = (0.0, 0.0, 0);
while k < n {
let s = (rng.u() * (n - block + 1) as f64) as usize;
for i in s..s + block { if k < n { lm += model[i]; lb += base[i]; k += 1; } }
}
1.0 - lm / lb
}).collect();
draws.sort_by(f64::total_cmp);
(point, draws[100], draws[1899])
}
struct Baselines { persist: [[f64; 2]; C], popular: [[f64; 10]; C] }
fn pop_bin(m12: f64) -> usize {
const EDGES: [f64; 9] = [0.05, 0.3, 0.6, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5];
EDGES.iter().take_while(|&&e| m12.ln_1p() >= e).count()
}
fn mean12(p: &Panel, a: usize, c: usize, t: usize) -> f64 { (0..LAGS).map(|i| p.y(a, c, t - i)).sum::<f64>() / LAGS as f64 }
fn fit_baselines(p: &Panel, h: usize, t0: usize, tau: &[u32; C]) -> Baselines {
let mut pc = [[[1.0f64, 2.0]; 2]; C]; // [hits, n] with a Laplace-style prior
let mut bc = [[[1.0f64, 2.0]; 10]; C];
for t in LAGS - 1..=t0 - h {
for a in 0..p.a() {
for c in 0..C {
let now = p.y(a, c, t) >= tau[c] as f64;
let later = (p.y(a, c, t + h) >= tau[c] as f64) as i32 as f64;
pc[c][now as usize][0] += later; pc[c][now as usize][1] += 1.0;
let b = pop_bin(mean12(p, a, c, t));
bc[c][b][0] += later; bc[c][b][1] += 1.0;
}
}
}
let mut out = Baselines { persist: [[0.0; 2]; C], popular: [[0.0; 10]; C] };
for c in 0..C {
for s in 0..2 { out.persist[c][s] = pc[c][s][0] / pc[c][s][1]; }
for b in 0..10 { out.popular[c][b] = bc[c][b][0] / bc[c][b][1]; }
}
out
}
/// Kalman level per artist × channel after week t0, with each channel's (q, r).
fn kalman_states(p: &Panel, t0: usize, models: &[LocalLevel; C]) -> Vec<(f64, f64)> {
let mut z = vec![0.0; t0 + 1];
let mut out = Vec::with_capacity(p.a() * C);
for a in 0..p.a() {
for c in 0..C {
for (w, v) in z.iter_mut().enumerate() { *v = p.y(a, c, w).ln_1p(); }
let (l, v, _) = kalman::filter(&z, models[c]);
out.push((l, v));
}
}
out
}
fn fit_kalman(p: &Panel, t0: usize) -> [LocalLevel; C] {
let mut out = [LocalLevel { q: 0.05, r: 0.3 }; C];
for (c, m) in out.iter_mut().enumerate() {
let series: Vec<Vec<f64>> = (0..p.a())
.map(|a| (0..=t0).map(|w| p.y(a, c, w).ln_1p()).collect::<Vec<f64>>())
.filter(|z| z.iter().any(|&v| v > 0.0))
.collect();
*m = kalman::fit(series.iter().map(|v| v.as_slice()));
}
out
}
pub struct Report { pub markdown: String, pub rows: Vec<(String, usize, String, f64)> }
/// `refit`: refit the regressions every this many origins (1 = weekly, as the doc's protocol).
pub fn run(p: &Panel, horizons: &[usize], refit: usize) -> Result<Report> {
let tau = thresholds(p, START);
let mut md = String::new();
let mut rows = Vec::new();
writeln!(md, "# Backtest: DR artist × channel × week\n")?;
writeln!(md, "Panel: {} artists × {} channels × {} complete weeks ({} to {}). Forecast origins from week {}; regressions refitted every {} origin(s).\n",
p.a(), C, p.w(), p.weeks[0], p.weeks[p.w() - 1], START + 1, refit)?;
writeln!(md, "\"In rotation\" = at least τ plays in the week: {}.\n",
CHANNELS.iter().zip(tau).map(|(c, t)| format!("{c} {t}")).collect::<Vec<_>>().join(", "))?;
let mut x = vec![0.0; P];
for &h in horizons {
if START + h >= p.w() { continue; }
let origins: Vec<usize> = (START..p.w() - h).collect();
let mut weeks: Vec<[Week; M]> = Vec::new();
let mut add_pairs: [Vec<(f64, bool)>; M] = Default::default();
let mut calib = [[0.0f64; 3]; 10]; // nb adds: [sum p, hits, n] per decile of p
let mut direct: Option<Direct> = None;
let mut kal = [LocalLevel { q: 0.05, r: 0.3 }; C];
for (i, &t0) in origins.iter().enumerate() {
if i % refit == 0 || direct.is_none() {
direct = Some(Direct::fit(p, h, t0, &tau, true));
kal = fit_kalman(p, t0);
}
let d = direct.as_ref().unwrap();
let base = fit_baselines(p, h, t0, &tau);
let ks = kalman_states(p, t0, &kal);
let target = t0 + h;
let mut wk: [Week; M] = Default::default();
let mut totals = vec![[0.0f64; M]; p.a()];
let mut truth = vec![0.0f64; p.a()];
for a in 0..p.a() {
for c in 0..C {
let y = p.y(a, c, target);
let rot = y >= tau[c] as f64;
let now = p.y(a, c, t0) >= tau[c] as f64;
let m12 = mean12(p, a, c, t0);
let (level, var) = ks[a * C + c];
let (km, kv) = kalman::forecast(level, var, kal[c], h);
let (mu, p_nb, p_lg) = d.predict(p, a, c, t0, tau[c], &mut x);
let preds: [(f64, f64); M] = [
(base.persist[c][now as usize], p.y(a, c, t0)),
(base.popular[c][pop_bin(m12)], m12),
(kalman::norm_sf((((tau[c] as f64) - 0.5).ln_1p() - km) / kv.sqrt()).clamp(1e-9, 1.0 - 1e-9), ((km + kv / 2.0).exp() - 1.0).max(0.0)),
(p_nb, mu),
(p_lg, mu),
];
truth[a] += y;
for (m, (pr, mean)) in preds.iter().enumerate() {
let s = &mut wk[m];
s.ll += ll(*pr, rot); s.brier += (pr - rot as i32 as f64).powi(2); s.n += 1.0;
if !now {
s.ll_add += ll(*pr, rot); s.brier_add += (pr - rot as i32 as f64).powi(2); s.n_add += 1.0;
add_pairs[m].push((*pr, rot));
}
s.abs_err += (mean - y).abs();
totals[a][m] += mean;
}
if !now {
let dec = ((p_nb * 10.0) as usize).min(9);
calib[dec][0] += p_nb; calib[dec][1] += rot as i32 as f64; calib[dec][2] += 1.0;
}
}
}
for m in 0..M {
let pred: Vec<f64> = totals.iter().map(|t| t[m]).collect();
wk[m].spearman = spearman(&pred, &truth);
wk[m].ndcg = ndcg(&pred, &truth, 50);
}
weeks.push(wk);
eprintln!("backtest h={h}: origin {} done ({}/{})", p.weeks[t0], i + 1, origins.len());
}
let nw = weeks.len();
writeln!(md, "## h = {h} week(s) ahead ({nw} forecast weeks)\n")?;
writeln!(md, "| Model | Log-loss, all | Log-loss, adds | Brier, adds | PR-AUC, adds | MAE plays | Spearman | NDCG@50 |")?;
writeln!(md, "|---|---|---|---|---|---|---|---|")?;
for m in 0..M {
let sum = |f: &dyn Fn(&Week) -> f64| weeks.iter().map(|w| f(&w[m])).sum::<f64>();
let (n, na) = (sum(&|w| w.n), sum(&|w| w.n_add));
let vals = [
sum(&|w| w.ll) / n, sum(&|w| w.ll_add) / na, sum(&|w| w.brier_add) / na,
average_precision(add_pairs[m].clone()), sum(&|w| w.abs_err) / n,
sum(&|w| w.spearman) / nw as f64, sum(&|w| w.ndcg) / nw as f64,
];
let names = ["logloss_all", "logloss_add", "brier_add", "prauc_add", "mae", "spearman", "ndcg50"];
for (k, v) in vals.iter().enumerate() { rows.push((MODELS[m].to_string(), h, names[k].to_string(), *v)); }
writeln!(md, "| {} | {:.4} | {:.4} | {:.4} | {:.3} | {:.2} | {:.3} | {:.3} |", MODELS[m],
vals[0], vals[1], vals[2], vals[3], vals[4], vals[5], vals[6])?;
}
writeln!(md, "\nSkill on adds (1 − log-loss ÷ baseline log-loss; 90% block-bootstrap interval over weeks):\n")?;
writeln!(md, "| Model | vs persistence | vs popularity |")?;
writeln!(md, "|---|---|---|")?;
for m in 2..M {
let l = |k: usize| weeks.iter().map(|w| w[k].ll_add).collect::<Vec<f64>>();
let (a, b) = (skill(&l(m), &l(0)), skill(&l(m), &l(1)));
rows.push((MODELS[m].to_string(), h, "skill_add_vs_persistence".into(), a.0));
rows.push((MODELS[m].to_string(), h, "skill_add_vs_popularity".into(), b.0));
writeln!(md, "| {} | {:.3} ({:.3} to {:.3}) | {:.3} ({:.3} to {:.3}) |", MODELS[m], a.0, a.1, a.2, b.0, b.1, b.2)?;
}
writeln!(md, "\nCalibration of the NB's add probabilities (rows not in rotation at the origin):\n")?;
writeln!(md, "| Predicted | Rows | Mean predicted | Observed |")?;
writeln!(md, "|---|---|---|---|")?;
for (dec, c) in calib.iter().enumerate() {
if c[2] == 0.0 { continue; }
writeln!(md, "| {:.1}–{:.1} | {} | {:.3} | {:.3} |", dec as f64 / 10.0, (dec + 1) as f64 / 10.0, c[2], c[0] / c[2], c[1] / c[2])?;
}
writeln!(md)?;
if nw < 8 { writeln!(md, "Fewer than 8 forecast weeks: no verdict at this horizon (doc rule).\n")?; }
}
{
let d = Direct::fit(p, 4, p.w() - 1, &tau, false);
writeln!(md, "## NB coefficients, h = 4, all weeks (alpha = {:.3})\n", d.alpha)?;
writeln!(md, "| Feature | Coefficient |\n|---|---|")?;
for (n, b) in super::features::NAMES.iter().zip(&d.nb) { writeln!(md, "| {n} | {b:+.3} |")?; }
writeln!(md)?;
}
Ok(Report { markdown: md, rows })
}