Files
Airplay-Predictor/src/model/backtest.rs
T

272 lines
13 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! Walk-forward backtest on the DR bed (doc: "Two backtest beds"). For each origin week t0 from
//! week 26 on, every model is fitted on what was known at the end of t0 and forecasts week
//! t0 + h for every artist × channel. Scored against the doc's baselines:
//!
//! persistence P(in rotation at t0+h | in rotation at t0), per channel; plays carried forward
//! popularity P(in rotation) by bins of the artist's last-12-week mean; that mean as plays
//! kalman local level on log plays (artist form, first version)
//! nb direct h-step negative binomial on the panel features
//! logit direct h-step logistic model of "in rotation"
//!
//! ("Release timing only" needs release dates, which arrive with the Deezer backfill.)
use super::common::{thresholds, Direct, Rng};
use super::features::{LAGS, P};
use super::kalman::{self, LocalLevel};
use super::panel::{Panel, C, CHANNELS};
use anyhow::Result;
use std::fmt::Write as _;
pub const MODELS: [&str; 5] = ["persistence", "popularity", "kalman", "nb", "logit"];
const M: usize = MODELS.len();
const START: usize = 26;
/// One model's scores in one forecast week.
#[derive(Clone, Default)]
struct Week {
ll: f64, brier: f64, n: f64, // all rows: "in rotation" at t0 + h
ll_add: f64, brier_add: f64, n_add: f64, // rows not in rotation at t0 (adds)
abs_err: f64, // weekly plays
spearman: f64, ndcg: f64, // artists ranked by total plays across channels
}
fn ll(p: f64, y: bool) -> f64 { -(if y { p } else { 1.0 - p }).clamp(1e-9, 1.0).ln() }
fn ranks(v: &[f64]) -> Vec<f64> {
let mut idx: Vec<usize> = (0..v.len()).collect();
idx.sort_by(|&a, &b| v[a].total_cmp(&v[b]));
let mut r = vec![0.0; v.len()];
let mut i = 0;
while i < idx.len() {
let mut j = i;
while j + 1 < idx.len() && v[idx[j + 1]] == v[idx[i]] { j += 1; }
for k in i..=j { r[idx[k]] = (i + j) as f64 / 2.0; }
i = j + 1;
}
r
}
pub fn spearman(a: &[f64], b: &[f64]) -> f64 {
let (ra, rb) = (ranks(a), ranks(b));
let n = a.len() as f64;
let (ma, mb) = (ra.iter().sum::<f64>() / n, rb.iter().sum::<f64>() / n);
let (mut sab, mut saa, mut sbb) = (0.0, 0.0, 0.0);
for i in 0..a.len() { let (x, y) = (ra[i] - ma, rb[i] - mb); sab += x * y; saa += x * x; sbb += y * y; }
sab / (saa * sbb).sqrt().max(1e-12)
}
pub fn ndcg(pred: &[f64], truth: &[f64], k: usize) -> f64 {
let dcg = |order: &[usize]| order.iter().take(k).enumerate().map(|(i, &j)| truth[j] / (i as f64 + 2.0).log2()).sum::<f64>();
let mut by_pred: Vec<usize> = (0..pred.len()).collect();
by_pred.sort_by(|&a, &b| pred[b].total_cmp(&pred[a]));
let mut ideal: Vec<usize> = (0..truth.len()).collect();
ideal.sort_by(|&a, &b| truth[b].total_cmp(&truth[a]));
dcg(&by_pred) / dcg(&ideal).max(1e-12)
}
fn average_precision(mut v: Vec<(f64, bool)>) -> f64 {
v.sort_by(|a, b| b.0.total_cmp(&a.0));
let (mut hits, mut sum) = (0.0, 0.0);
for (i, (_, y)) in v.iter().enumerate() { if *y { hits += 1.0; sum += hits / (i as f64 + 1.0); } }
if hits > 0.0 { sum / hits } else { f64::NAN }
}
/// Skill 1 - loss(model) / loss(base) over all weeks, with a moving-block bootstrap (blocks of 4
/// weeks) 90% interval.
fn skill(model: &[f64], base: &[f64]) -> (f64, f64, f64) {
let point = 1.0 - model.iter().sum::<f64>() / base.iter().sum::<f64>();
let n = model.len();
// Blocks of 8 weeks: neighbouring origins share most of an 8-week target window.
let block = 8.min(n);
let mut rng = Rng(42);
let mut draws: Vec<f64> = (0..2000).map(|_| {
let (mut lm, mut lb, mut k) = (0.0, 0.0, 0);
while k < n {
let s = (rng.u() * (n - block + 1) as f64) as usize;
for i in s..s + block { if k < n { lm += model[i]; lb += base[i]; k += 1; } }
}
1.0 - lm / lb
}).collect();
draws.sort_by(f64::total_cmp);
(point, draws[100], draws[1899])
}
struct Baselines { persist: [[f64; 2]; C], popular: [[f64; 10]; C] }
fn pop_bin(m12: f64) -> usize {
const EDGES: [f64; 9] = [0.05, 0.3, 0.6, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5];
EDGES.iter().take_while(|&&e| m12.ln_1p() >= e).count()
}
fn mean12(p: &Panel, a: usize, c: usize, t: usize) -> f64 { (0..LAGS).map(|i| p.y(a, c, t - i)).sum::<f64>() / LAGS as f64 }
fn fit_baselines(p: &Panel, h: usize, t0: usize, tau: &[u32; C]) -> Baselines {
let mut pc = [[[1.0f64, 2.0]; 2]; C]; // [hits, n] with a Laplace-style prior
let mut bc = [[[1.0f64, 2.0]; 10]; C];
for t in LAGS - 1..=t0 - h {
for a in 0..p.a() {
for c in 0..C {
let now = p.y(a, c, t) >= tau[c] as f64;
let later = (p.y(a, c, t + h) >= tau[c] as f64) as i32 as f64;
pc[c][now as usize][0] += later; pc[c][now as usize][1] += 1.0;
let b = pop_bin(mean12(p, a, c, t));
bc[c][b][0] += later; bc[c][b][1] += 1.0;
}
}
}
let mut out = Baselines { persist: [[0.0; 2]; C], popular: [[0.0; 10]; C] };
for c in 0..C {
for s in 0..2 { out.persist[c][s] = pc[c][s][0] / pc[c][s][1]; }
for b in 0..10 { out.popular[c][b] = bc[c][b][0] / bc[c][b][1]; }
}
out
}
/// Kalman level per artist × channel after week t0, with each channel's (q, r).
fn kalman_states(p: &Panel, t0: usize, models: &[LocalLevel; C]) -> Vec<(f64, f64)> {
let mut z = vec![0.0; t0 + 1];
let mut out = Vec::with_capacity(p.a() * C);
for a in 0..p.a() {
for c in 0..C {
for (w, v) in z.iter_mut().enumerate() { *v = p.y(a, c, w).ln_1p(); }
let (l, v, _) = kalman::filter(&z, models[c]);
out.push((l, v));
}
}
out
}
fn fit_kalman(p: &Panel, t0: usize) -> [LocalLevel; C] {
let mut out = [LocalLevel { q: 0.05, r: 0.3 }; C];
for (c, m) in out.iter_mut().enumerate() {
let series: Vec<Vec<f64>> = (0..p.a())
.map(|a| (0..=t0).map(|w| p.y(a, c, w).ln_1p()).collect::<Vec<f64>>())
.filter(|z| z.iter().any(|&v| v > 0.0))
.collect();
*m = kalman::fit(series.iter().map(|v| v.as_slice()));
}
out
}
pub struct Report { pub markdown: String, pub rows: Vec<(String, usize, String, f64)> }
/// `refit`: refit the regressions every this many origins (1 = weekly, as the doc's protocol).
pub fn run(p: &Panel, horizons: &[usize], refit: usize) -> Result<Report> {
let tau = thresholds(p, START);
let mut md = String::new();
let mut rows = Vec::new();
writeln!(md, "# Backtest: DR artist × channel × week\n")?;
writeln!(md, "Panel: {} artists × {} channels × {} complete weeks ({} to {}). Forecast origins from week {}; regressions refitted every {} origin(s).\n",
p.a(), C, p.w(), p.weeks[0], p.weeks[p.w() - 1], START + 1, refit)?;
writeln!(md, "\"In rotation\" = at least τ plays in the week: {}.\n",
CHANNELS.iter().zip(tau).map(|(c, t)| format!("{c} {t}")).collect::<Vec<_>>().join(", "))?;
let mut x = vec![0.0; P];
for &h in horizons {
if START + h >= p.w() { continue; }
let origins: Vec<usize> = (START..p.w() - h).collect();
let mut weeks: Vec<[Week; M]> = Vec::new();
let mut add_pairs: [Vec<(f64, bool)>; M] = Default::default();
let mut calib = [[0.0f64; 3]; 10]; // nb adds: [sum p, hits, n] per decile of p
let mut direct: Option<Direct> = None;
let mut kal = [LocalLevel { q: 0.05, r: 0.3 }; C];
for (i, &t0) in origins.iter().enumerate() {
if i % refit == 0 || direct.is_none() {
direct = Some(Direct::fit(p, h, t0, &tau, true));
kal = fit_kalman(p, t0);
}
let d = direct.as_ref().unwrap();
let base = fit_baselines(p, h, t0, &tau);
let ks = kalman_states(p, t0, &kal);
let target = t0 + h;
let mut wk: [Week; M] = Default::default();
let mut totals = vec![[0.0f64; M]; p.a()];
let mut truth = vec![0.0f64; p.a()];
for a in 0..p.a() {
for c in 0..C {
let y = p.y(a, c, target);
let rot = y >= tau[c] as f64;
let now = p.y(a, c, t0) >= tau[c] as f64;
let m12 = mean12(p, a, c, t0);
let (level, var) = ks[a * C + c];
let (km, kv) = kalman::forecast(level, var, kal[c], h);
let (mu, p_nb, p_lg) = d.predict(p, a, c, t0, tau[c], &mut x);
let preds: [(f64, f64); M] = [
(base.persist[c][now as usize], p.y(a, c, t0)),
(base.popular[c][pop_bin(m12)], m12),
(kalman::norm_sf((((tau[c] as f64) - 0.5).ln_1p() - km) / kv.sqrt()).clamp(1e-9, 1.0 - 1e-9), ((km + kv / 2.0).exp() - 1.0).max(0.0)),
(p_nb, mu),
(p_lg, mu),
];
truth[a] += y;
for (m, (pr, mean)) in preds.iter().enumerate() {
let s = &mut wk[m];
s.ll += ll(*pr, rot); s.brier += (pr - rot as i32 as f64).powi(2); s.n += 1.0;
if !now {
s.ll_add += ll(*pr, rot); s.brier_add += (pr - rot as i32 as f64).powi(2); s.n_add += 1.0;
add_pairs[m].push((*pr, rot));
}
s.abs_err += (mean - y).abs();
totals[a][m] += mean;
}
if !now {
let dec = ((p_nb * 10.0) as usize).min(9);
calib[dec][0] += p_nb; calib[dec][1] += rot as i32 as f64; calib[dec][2] += 1.0;
}
}
}
for m in 0..M {
let pred: Vec<f64> = totals.iter().map(|t| t[m]).collect();
wk[m].spearman = spearman(&pred, &truth);
wk[m].ndcg = ndcg(&pred, &truth, 50);
}
weeks.push(wk);
eprintln!("backtest h={h}: origin {} done ({}/{})", p.weeks[t0], i + 1, origins.len());
}
let nw = weeks.len();
writeln!(md, "## h = {h} week(s) ahead ({nw} forecast weeks)\n")?;
writeln!(md, "| Model | Log-loss, all | Log-loss, adds | Brier, adds | PR-AUC, adds | MAE plays | Spearman | NDCG@50 |")?;
writeln!(md, "|---|---|---|---|---|---|---|---|")?;
for m in 0..M {
let sum = |f: &dyn Fn(&Week) -> f64| weeks.iter().map(|w| f(&w[m])).sum::<f64>();
let (n, na) = (sum(&|w| w.n), sum(&|w| w.n_add));
let vals = [
sum(&|w| w.ll) / n, sum(&|w| w.ll_add) / na, sum(&|w| w.brier_add) / na,
average_precision(add_pairs[m].clone()), sum(&|w| w.abs_err) / n,
sum(&|w| w.spearman) / nw as f64, sum(&|w| w.ndcg) / nw as f64,
];
let names = ["logloss_all", "logloss_add", "brier_add", "prauc_add", "mae", "spearman", "ndcg50"];
for (k, v) in vals.iter().enumerate() { rows.push((MODELS[m].to_string(), h, names[k].to_string(), *v)); }
writeln!(md, "| {} | {:.4} | {:.4} | {:.4} | {:.3} | {:.2} | {:.3} | {:.3} |", MODELS[m],
vals[0], vals[1], vals[2], vals[3], vals[4], vals[5], vals[6])?;
}
writeln!(md, "\nSkill on adds (1 − log-loss ÷ baseline log-loss; 90% block-bootstrap interval over weeks):\n")?;
writeln!(md, "| Model | vs persistence | vs popularity |")?;
writeln!(md, "|---|---|---|")?;
for m in 2..M {
let l = |k: usize| weeks.iter().map(|w| w[k].ll_add).collect::<Vec<f64>>();
let (a, b) = (skill(&l(m), &l(0)), skill(&l(m), &l(1)));
rows.push((MODELS[m].to_string(), h, "skill_add_vs_persistence".into(), a.0));
rows.push((MODELS[m].to_string(), h, "skill_add_vs_popularity".into(), b.0));
writeln!(md, "| {} | {:.3} ({:.3} to {:.3}) | {:.3} ({:.3} to {:.3}) |", MODELS[m], a.0, a.1, a.2, b.0, b.1, b.2)?;
}
writeln!(md, "\nCalibration of the NB's add probabilities (rows not in rotation at the origin):\n")?;
writeln!(md, "| Predicted | Rows | Mean predicted | Observed |")?;
writeln!(md, "|---|---|---|---|")?;
for (dec, c) in calib.iter().enumerate() {
if c[2] == 0.0 { continue; }
writeln!(md, "| {:.1}–{:.1} | {} | {:.3} | {:.3} |", dec as f64 / 10.0, (dec + 1) as f64 / 10.0, c[2], c[0] / c[2], c[1] / c[2])?;
}
writeln!(md)?;
if nw < 8 { writeln!(md, "Fewer than 8 forecast weeks: no verdict at this horizon (doc rule).\n")?; }
}
{
let d = Direct::fit(p, 4, p.w() - 1, &tau, false);
writeln!(md, "## NB coefficients, h = 4, all weeks (alpha = {:.3})\n", d.alpha)?;
writeln!(md, "| Feature | Coefficient |\n|---|---|")?;
for (n, b) in super::features::NAMES.iter().zip(&d.nb) { writeln!(md, "| {n} | {b:+.3} |")?; }
writeln!(md)?;
}
Ok(Report { markdown: md, rows })
}