Models, first layer: artist × DR channel × week panel with IFPI chart history; NB and logit direct h-step models, Kalman artist form, persistence/popularity baselines; walk-forward backtest (docs/BACKTEST.md) and the ranking (forecasts table)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,270 @@
|
||||
//! Walk-forward backtest on the DR bed (doc: "Two backtest beds"). For each origin week t0 from
|
||||
//! week 26 on, every model is fitted on what was known at the end of t0 and forecasts week
|
||||
//! t0 + h for every artist × channel. Scored against the doc's baselines:
|
||||
//!
|
||||
//! persistence P(in rotation at t0+h | in rotation at t0), per channel; plays carried forward
|
||||
//! popularity P(in rotation) by bins of the artist's last-12-week mean; that mean as plays
|
||||
//! kalman local level on log plays (artist form, first version)
|
||||
//! nb direct h-step negative binomial on the panel features
|
||||
//! logit direct h-step logistic model of "in rotation"
|
||||
//!
|
||||
//! ("Release timing only" needs release dates, which arrive with the Deezer backfill.)
|
||||
|
||||
use super::common::{thresholds, Direct, Rng};
|
||||
use super::features::{LAGS, P};
|
||||
use super::kalman::{self, LocalLevel};
|
||||
use super::panel::{Panel, C, CHANNELS};
|
||||
use anyhow::Result;
|
||||
use std::fmt::Write as _;
|
||||
|
||||
pub const MODELS: [&str; 5] = ["persistence", "popularity", "kalman", "nb", "logit"];
|
||||
const M: usize = MODELS.len();
|
||||
const START: usize = 26;
|
||||
|
||||
/// One model's scores in one forecast week.
|
||||
#[derive(Clone, Default)]
|
||||
struct Week {
|
||||
ll: f64, brier: f64, n: f64, // all rows: "in rotation" at t0 + h
|
||||
ll_add: f64, brier_add: f64, n_add: f64, // rows not in rotation at t0 (adds)
|
||||
abs_err: f64, // weekly plays
|
||||
spearman: f64, ndcg: f64, // artists ranked by total plays across channels
|
||||
}
|
||||
|
||||
fn ll(p: f64, y: bool) -> f64 { -(if y { p } else { 1.0 - p }).clamp(1e-9, 1.0).ln() }
|
||||
|
||||
fn ranks(v: &[f64]) -> Vec<f64> {
|
||||
let mut idx: Vec<usize> = (0..v.len()).collect();
|
||||
idx.sort_by(|&a, &b| v[a].total_cmp(&v[b]));
|
||||
let mut r = vec![0.0; v.len()];
|
||||
let mut i = 0;
|
||||
while i < idx.len() {
|
||||
let mut j = i;
|
||||
while j + 1 < idx.len() && v[idx[j + 1]] == v[idx[i]] { j += 1; }
|
||||
for k in i..=j { r[idx[k]] = (i + j) as f64 / 2.0; }
|
||||
i = j + 1;
|
||||
}
|
||||
r
|
||||
}
|
||||
|
||||
pub fn spearman(a: &[f64], b: &[f64]) -> f64 {
|
||||
let (ra, rb) = (ranks(a), ranks(b));
|
||||
let n = a.len() as f64;
|
||||
let (ma, mb) = (ra.iter().sum::<f64>() / n, rb.iter().sum::<f64>() / n);
|
||||
let (mut sab, mut saa, mut sbb) = (0.0, 0.0, 0.0);
|
||||
for i in 0..a.len() { let (x, y) = (ra[i] - ma, rb[i] - mb); sab += x * y; saa += x * x; sbb += y * y; }
|
||||
sab / (saa * sbb).sqrt().max(1e-12)
|
||||
}
|
||||
|
||||
pub fn ndcg(pred: &[f64], truth: &[f64], k: usize) -> f64 {
|
||||
let dcg = |order: &[usize]| order.iter().take(k).enumerate().map(|(i, &j)| truth[j] / (i as f64 + 2.0).log2()).sum::<f64>();
|
||||
let mut by_pred: Vec<usize> = (0..pred.len()).collect();
|
||||
by_pred.sort_by(|&a, &b| pred[b].total_cmp(&pred[a]));
|
||||
let mut ideal: Vec<usize> = (0..truth.len()).collect();
|
||||
ideal.sort_by(|&a, &b| truth[b].total_cmp(&truth[a]));
|
||||
dcg(&by_pred) / dcg(&ideal).max(1e-12)
|
||||
}
|
||||
|
||||
fn average_precision(mut v: Vec<(f64, bool)>) -> f64 {
|
||||
v.sort_by(|a, b| b.0.total_cmp(&a.0));
|
||||
let (mut hits, mut sum) = (0.0, 0.0);
|
||||
for (i, (_, y)) in v.iter().enumerate() { if *y { hits += 1.0; sum += hits / (i as f64 + 1.0); } }
|
||||
if hits > 0.0 { sum / hits } else { f64::NAN }
|
||||
}
|
||||
|
||||
/// Skill 1 - loss(model) / loss(base) over all weeks, with a moving-block bootstrap (blocks of 4
|
||||
/// weeks) 90% interval.
|
||||
fn skill(model: &[f64], base: &[f64]) -> (f64, f64, f64) {
|
||||
let point = 1.0 - model.iter().sum::<f64>() / base.iter().sum::<f64>();
|
||||
let n = model.len();
|
||||
let block = 4.min(n);
|
||||
let mut rng = Rng(42);
|
||||
let mut draws: Vec<f64> = (0..2000).map(|_| {
|
||||
let (mut lm, mut lb, mut k) = (0.0, 0.0, 0);
|
||||
while k < n {
|
||||
let s = (rng.u() * (n - block + 1) as f64) as usize;
|
||||
for i in s..s + block { if k < n { lm += model[i]; lb += base[i]; k += 1; } }
|
||||
}
|
||||
1.0 - lm / lb
|
||||
}).collect();
|
||||
draws.sort_by(f64::total_cmp);
|
||||
(point, draws[100], draws[1899])
|
||||
}
|
||||
|
||||
struct Baselines { persist: [[f64; 2]; C], popular: [[f64; 10]; C] }
|
||||
|
||||
fn pop_bin(m12: f64) -> usize {
|
||||
const EDGES: [f64; 9] = [0.05, 0.3, 0.6, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5];
|
||||
EDGES.iter().take_while(|&&e| m12.ln_1p() >= e).count()
|
||||
}
|
||||
|
||||
fn mean12(p: &Panel, a: usize, c: usize, t: usize) -> f64 { (0..LAGS).map(|i| p.y(a, c, t - i)).sum::<f64>() / LAGS as f64 }
|
||||
|
||||
fn fit_baselines(p: &Panel, h: usize, t0: usize, tau: &[u32; C]) -> Baselines {
|
||||
let mut pc = [[[1.0f64, 2.0]; 2]; C]; // [hits, n] with a Laplace-style prior
|
||||
let mut bc = [[[1.0f64, 2.0]; 10]; C];
|
||||
for t in LAGS - 1..=t0 - h {
|
||||
for a in 0..p.a() {
|
||||
for c in 0..C {
|
||||
let now = p.y(a, c, t) >= tau[c] as f64;
|
||||
let later = (p.y(a, c, t + h) >= tau[c] as f64) as i32 as f64;
|
||||
pc[c][now as usize][0] += later; pc[c][now as usize][1] += 1.0;
|
||||
let b = pop_bin(mean12(p, a, c, t));
|
||||
bc[c][b][0] += later; bc[c][b][1] += 1.0;
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut out = Baselines { persist: [[0.0; 2]; C], popular: [[0.0; 10]; C] };
|
||||
for c in 0..C {
|
||||
for s in 0..2 { out.persist[c][s] = pc[c][s][0] / pc[c][s][1]; }
|
||||
for b in 0..10 { out.popular[c][b] = bc[c][b][0] / bc[c][b][1]; }
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Kalman level per artist × channel after week t0, with each channel's (q, r).
|
||||
fn kalman_states(p: &Panel, t0: usize, models: &[LocalLevel; C]) -> Vec<(f64, f64)> {
|
||||
let mut z = vec![0.0; t0 + 1];
|
||||
let mut out = Vec::with_capacity(p.a() * C);
|
||||
for a in 0..p.a() {
|
||||
for c in 0..C {
|
||||
for (w, v) in z.iter_mut().enumerate() { *v = p.y(a, c, w).ln_1p(); }
|
||||
let (l, v, _) = kalman::filter(&z, models[c]);
|
||||
out.push((l, v));
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn fit_kalman(p: &Panel, t0: usize) -> [LocalLevel; C] {
|
||||
let mut out = [LocalLevel { q: 0.05, r: 0.3 }; C];
|
||||
for (c, m) in out.iter_mut().enumerate() {
|
||||
let series: Vec<Vec<f64>> = (0..p.a())
|
||||
.map(|a| (0..=t0).map(|w| p.y(a, c, w).ln_1p()).collect::<Vec<f64>>())
|
||||
.filter(|z| z.iter().any(|&v| v > 0.0))
|
||||
.collect();
|
||||
*m = kalman::fit(series.iter().map(|v| v.as_slice()));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
pub struct Report { pub markdown: String, pub rows: Vec<(String, usize, String, f64)> }
|
||||
|
||||
/// `refit`: refit the regressions every this many origins (1 = weekly, as the doc's protocol).
|
||||
pub fn run(p: &Panel, horizons: &[usize], refit: usize) -> Result<Report> {
|
||||
let tau = thresholds(p, START);
|
||||
let mut md = String::new();
|
||||
let mut rows = Vec::new();
|
||||
writeln!(md, "# Backtest: DR artist × channel × week\n")?;
|
||||
writeln!(md, "Panel: {} artists × {} channels × {} complete weeks ({} to {}). Forecast origins from week {}; regressions refitted every {} origin(s).\n",
|
||||
p.a(), C, p.w(), p.weeks[0], p.weeks[p.w() - 1], START + 1, refit)?;
|
||||
writeln!(md, "\"In rotation\" = at least τ plays in the week: {}.\n",
|
||||
CHANNELS.iter().zip(tau).map(|(c, t)| format!("{c} {t}")).collect::<Vec<_>>().join(", "))?;
|
||||
let mut x = vec![0.0; P];
|
||||
for &h in horizons {
|
||||
if START + h >= p.w() { continue; }
|
||||
let origins: Vec<usize> = (START..p.w() - h).collect();
|
||||
let mut weeks: Vec<[Week; M]> = Vec::new();
|
||||
let mut add_pairs: [Vec<(f64, bool)>; M] = Default::default();
|
||||
let mut calib = [[0.0f64; 3]; 10]; // nb adds: [sum p, hits, n] per decile of p
|
||||
let mut direct: Option<Direct> = None;
|
||||
let mut kal = [LocalLevel { q: 0.05, r: 0.3 }; C];
|
||||
for (i, &t0) in origins.iter().enumerate() {
|
||||
if i % refit == 0 || direct.is_none() {
|
||||
direct = Some(Direct::fit(p, h, t0, &tau, true));
|
||||
kal = fit_kalman(p, t0);
|
||||
}
|
||||
let d = direct.as_ref().unwrap();
|
||||
let base = fit_baselines(p, h, t0, &tau);
|
||||
let ks = kalman_states(p, t0, &kal);
|
||||
let target = t0 + h;
|
||||
let mut wk: [Week; M] = Default::default();
|
||||
let mut totals = vec![[0.0f64; M]; p.a()];
|
||||
let mut truth = vec![0.0f64; p.a()];
|
||||
for a in 0..p.a() {
|
||||
for c in 0..C {
|
||||
let y = p.y(a, c, target);
|
||||
let rot = y >= tau[c] as f64;
|
||||
let now = p.y(a, c, t0) >= tau[c] as f64;
|
||||
let m12 = mean12(p, a, c, t0);
|
||||
let (level, var) = ks[a * C + c];
|
||||
let (km, kv) = kalman::forecast(level, var, kal[c], h);
|
||||
let (mu, p_nb, p_lg) = d.predict(p, a, c, t0, tau[c], &mut x);
|
||||
let preds: [(f64, f64); M] = [
|
||||
(base.persist[c][now as usize], p.y(a, c, t0)),
|
||||
(base.popular[c][pop_bin(m12)], m12),
|
||||
(kalman::norm_sf((((tau[c] as f64) - 0.5).ln_1p() - km) / kv.sqrt()).clamp(1e-9, 1.0 - 1e-9), ((km + kv / 2.0).exp() - 1.0).max(0.0)),
|
||||
(p_nb, mu),
|
||||
(p_lg, mu),
|
||||
];
|
||||
truth[a] += y;
|
||||
for (m, (pr, mean)) in preds.iter().enumerate() {
|
||||
let s = &mut wk[m];
|
||||
s.ll += ll(*pr, rot); s.brier += (pr - rot as i32 as f64).powi(2); s.n += 1.0;
|
||||
if !now {
|
||||
s.ll_add += ll(*pr, rot); s.brier_add += (pr - rot as i32 as f64).powi(2); s.n_add += 1.0;
|
||||
add_pairs[m].push((*pr, rot));
|
||||
}
|
||||
s.abs_err += (mean - y).abs();
|
||||
totals[a][m] += mean;
|
||||
}
|
||||
if !now {
|
||||
let dec = ((p_nb * 10.0) as usize).min(9);
|
||||
calib[dec][0] += p_nb; calib[dec][1] += rot as i32 as f64; calib[dec][2] += 1.0;
|
||||
}
|
||||
}
|
||||
}
|
||||
for m in 0..M {
|
||||
let pred: Vec<f64> = totals.iter().map(|t| t[m]).collect();
|
||||
wk[m].spearman = spearman(&pred, &truth);
|
||||
wk[m].ndcg = ndcg(&pred, &truth, 50);
|
||||
}
|
||||
weeks.push(wk);
|
||||
eprintln!("backtest h={h}: origin {} done ({}/{})", p.weeks[t0], i + 1, origins.len());
|
||||
}
|
||||
|
||||
let nw = weeks.len();
|
||||
writeln!(md, "## h = {h} week(s) ahead ({nw} forecast weeks)\n")?;
|
||||
writeln!(md, "| Model | Log-loss, all | Log-loss, adds | Brier, adds | PR-AUC, adds | MAE plays | Spearman | NDCG@50 |")?;
|
||||
writeln!(md, "|---|---|---|---|---|---|---|---|")?;
|
||||
for m in 0..M {
|
||||
let sum = |f: &dyn Fn(&Week) -> f64| weeks.iter().map(|w| f(&w[m])).sum::<f64>();
|
||||
let (n, na) = (sum(&|w| w.n), sum(&|w| w.n_add));
|
||||
let vals = [
|
||||
sum(&|w| w.ll) / n, sum(&|w| w.ll_add) / na, sum(&|w| w.brier_add) / na,
|
||||
average_precision(add_pairs[m].clone()), sum(&|w| w.abs_err) / n,
|
||||
sum(&|w| w.spearman) / nw as f64, sum(&|w| w.ndcg) / nw as f64,
|
||||
];
|
||||
let names = ["logloss_all", "logloss_add", "brier_add", "prauc_add", "mae", "spearman", "ndcg50"];
|
||||
for (k, v) in vals.iter().enumerate() { rows.push((MODELS[m].to_string(), h, names[k].to_string(), *v)); }
|
||||
writeln!(md, "| {} | {:.4} | {:.4} | {:.4} | {:.3} | {:.2} | {:.3} | {:.3} |", MODELS[m],
|
||||
vals[0], vals[1], vals[2], vals[3], vals[4], vals[5], vals[6])?;
|
||||
}
|
||||
writeln!(md, "\nSkill on adds (1 − log-loss ÷ baseline log-loss; 90% block-bootstrap interval over weeks):\n")?;
|
||||
writeln!(md, "| Model | vs persistence | vs popularity |")?;
|
||||
writeln!(md, "|---|---|---|")?;
|
||||
for m in 2..M {
|
||||
let l = |k: usize| weeks.iter().map(|w| w[k].ll_add).collect::<Vec<f64>>();
|
||||
let (a, b) = (skill(&l(m), &l(0)), skill(&l(m), &l(1)));
|
||||
rows.push((MODELS[m].to_string(), h, "skill_add_vs_persistence".into(), a.0));
|
||||
rows.push((MODELS[m].to_string(), h, "skill_add_vs_popularity".into(), b.0));
|
||||
writeln!(md, "| {} | {:.3} ({:.3} to {:.3}) | {:.3} ({:.3} to {:.3}) |", MODELS[m], a.0, a.1, a.2, b.0, b.1, b.2)?;
|
||||
}
|
||||
writeln!(md, "\nCalibration of the NB's add probabilities (rows not in rotation at the origin):\n")?;
|
||||
writeln!(md, "| Predicted | Rows | Mean predicted | Observed |")?;
|
||||
writeln!(md, "|---|---|---|---|")?;
|
||||
for (dec, c) in calib.iter().enumerate() {
|
||||
if c[2] == 0.0 { continue; }
|
||||
writeln!(md, "| {:.1}–{:.1} | {} | {:.3} | {:.3} |", dec as f64 / 10.0, (dec + 1) as f64 / 10.0, c[2], c[0] / c[2], c[1] / c[2])?;
|
||||
}
|
||||
writeln!(md)?;
|
||||
if nw < 8 { writeln!(md, "Fewer than 8 forecast weeks: no verdict at this horizon (doc rule).\n")?; }
|
||||
}
|
||||
{
|
||||
let d = Direct::fit(p, 4, p.w() - 1, &tau, false);
|
||||
writeln!(md, "## NB coefficients, h = 4, all weeks (alpha = {:.3})\n", d.alpha)?;
|
||||
writeln!(md, "| Feature | Coefficient |\n|---|---|")?;
|
||||
for (n, b) in super::features::NAMES.iter().zip(&d.nb) { writeln!(md, "| {n} | {b:+.3} |")?; }
|
||||
writeln!(md)?;
|
||||
}
|
||||
Ok(Report { markdown: md, rows })
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
//! Shared pieces: rotation thresholds, training designs, the fitted direct h-step models,
|
||||
//! random numbers.
|
||||
|
||||
use super::features::{self, LAGS, P};
|
||||
use super::glm::{self, Design};
|
||||
use super::panel::{Panel, C};
|
||||
|
||||
/// "In rotation" on channel c = at least tau[c] plays in the week. Set from each channel's
|
||||
/// distribution of non-zero artist-weeks (75th percentile, at least 3) over weeks < `upto`;
|
||||
/// to be calibrated on the Friday rotation lists once they're parsed.
|
||||
pub fn thresholds(p: &Panel, upto: usize) -> [u32; C] {
|
||||
let mut tau = [3u32; C];
|
||||
for (c, t) in tau.iter_mut().enumerate() {
|
||||
let mut v: Vec<f64> = (0..p.a()).flat_map(|a| (0..upto).map(move |w| (a, w))).map(|(a, w)| p.y(a, c, w)).filter(|&y| y > 0.0).collect();
|
||||
if v.is_empty() { continue; }
|
||||
v.sort_by(f64::total_cmp);
|
||||
*t = (v[v.len() * 3 / 4] as u32).max(3);
|
||||
}
|
||||
tau
|
||||
}
|
||||
|
||||
pub fn mix(mut x: u64) -> u64 {
|
||||
x = x.wrapping_add(0x9E3779B97F4A7C15);
|
||||
x = (x ^ (x >> 30)).wrapping_mul(0xBF58476D1CE4E5B9);
|
||||
x = (x ^ (x >> 27)).wrapping_mul(0x94D049BB133111EB);
|
||||
x ^ (x >> 31)
|
||||
}
|
||||
|
||||
/// Training rows for horizon h: every origin t with LAGS - 1 <= t and t + h <= last_target.
|
||||
/// Rows with no plays on the channel in the 12 weeks before the origin are mostly zeros;
|
||||
/// a fixed quarter of them is kept, weighted 4.
|
||||
pub fn design(p: &Panel, h: usize, last_target: usize, tau: &[u32; C]) -> (Design, Design) {
|
||||
let mut counts = Design::new(P);
|
||||
let mut rot = Design::new(P);
|
||||
let mut x = vec![0.0; P];
|
||||
if last_target < LAGS - 1 + h { return (counts, rot); }
|
||||
for t in LAGS - 1..=last_target - h {
|
||||
for a in 0..p.a() {
|
||||
for c in 0..C {
|
||||
let quiet = (0..LAGS).all(|i| p.y(a, c, t - i) == 0.0);
|
||||
let w = if quiet {
|
||||
if mix((a as u64) << 32 ^ (c as u64) << 20 ^ t as u64) % 4 != 0 { continue; }
|
||||
4.0
|
||||
} else { 1.0 };
|
||||
features::row(p, a, c, t, t + h, &mut x);
|
||||
let y = p.y(a, c, t + h);
|
||||
counts.push(&x, y, w);
|
||||
rot.push(&x, (y >= tau[c] as f64) as i32 as f64, w);
|
||||
}
|
||||
}
|
||||
}
|
||||
(counts, rot)
|
||||
}
|
||||
|
||||
/// The direct h-step models fitted on one design.
|
||||
pub struct Direct {
|
||||
pub h: usize,
|
||||
pub nb: Vec<f64>,
|
||||
pub alpha: f64,
|
||||
pub logit: Vec<f64>,
|
||||
}
|
||||
|
||||
impl Direct {
|
||||
pub fn fit(p: &Panel, h: usize, last_target: usize, tau: &[u32; C], with_logit: bool) -> Direct {
|
||||
let (counts, rot) = design(p, h, last_target, tau);
|
||||
let (nb, alpha) = glm::nb(&counts, 1.0);
|
||||
drop(counts);
|
||||
let logit = if with_logit { glm::logit(&rot, 1.0) } else { Vec::new() };
|
||||
Direct { h, nb, alpha, logit }
|
||||
}
|
||||
|
||||
/// (expected plays, P(rotation) from the NB, P(rotation) from the logit or NaN).
|
||||
pub fn predict(&self, p: &Panel, a: usize, c: usize, t: usize, tau: u32, x: &mut [f64]) -> (f64, f64, f64) {
|
||||
features::row(p, a, c, t, t + self.h, x);
|
||||
let mu = glm::dot(&self.nb, x).clamp(-30.0, 12.0).exp();
|
||||
let pl = if self.logit.is_empty() { f64::NAN } else { glm::sigmoid(glm::dot(&self.logit, x)).clamp(1e-9, 1.0 - 1e-9) };
|
||||
(mu, glm::nb_tail(mu, self.alpha, tau), pl)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Rng(pub u64);
|
||||
|
||||
impl Rng {
|
||||
pub fn u(&mut self) -> f64 { self.0 = mix(self.0); (self.0 >> 11) as f64 / (1u64 << 53) as f64 }
|
||||
pub fn normal(&mut self) -> f64 {
|
||||
let (u1, u2) = (self.u().max(1e-300), self.u());
|
||||
(-2.0 * u1.ln()).sqrt() * (2.0 * std::f64::consts::PI * u2).cos()
|
||||
}
|
||||
/// Gamma(shape k, scale 1) by Marsaglia–Tsang.
|
||||
pub fn gamma(&mut self, k: f64) -> f64 {
|
||||
if k < 1.0 { return self.gamma(k + 1.0) * self.u().max(1e-300).powf(1.0 / k); }
|
||||
let d = k - 1.0 / 3.0;
|
||||
let c = 1.0 / (9.0 * d).sqrt();
|
||||
loop {
|
||||
let x = self.normal();
|
||||
let v = (1.0 + c * x).powi(3);
|
||||
if v <= 0.0 { continue; }
|
||||
let u = self.u();
|
||||
if u.ln() < 0.5 * x * x + d - d * v + d * v.ln() { return d * v; }
|
||||
}
|
||||
}
|
||||
pub fn poisson(&mut self, lambda: f64) -> f64 {
|
||||
if lambda <= 0.0 { return 0.0; }
|
||||
if lambda > 40.0 { return (lambda + lambda.sqrt() * self.normal()).round().max(0.0); }
|
||||
let l = (-lambda).exp();
|
||||
let (mut k, mut prod) = (0.0, self.u());
|
||||
while prod > l { k += 1.0; prod *= self.u(); }
|
||||
k
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
//! The covariates of one artist × channel row, using only what was known at the end of the
|
||||
//! origin week t (the doc's "features known at w − 1 only"), for a target h weeks later.
|
||||
|
||||
use super::panel::{is_p4, Panel, C, CHANNELS};
|
||||
use chrono::Datelike;
|
||||
|
||||
/// Weeks of history a row needs before its origin.
|
||||
pub const LAGS: usize = 12;
|
||||
|
||||
pub const NAMES: &[&str] = &[
|
||||
"intercept",
|
||||
"log plays t", "log plays t-1", "log mean plays 4w", "log mean plays 12w", "log max plays 12w",
|
||||
"log other channels t", "log other channels 4w", "log P3 t", "weeks since played (/12)",
|
||||
"Track Top-40 score", "Airplay Top-20 score", "chart entry recency 6w", "chart entry recency 26w",
|
||||
"log Track Top-40 hits", "log Airplay Top-20 hits",
|
||||
"Danish", "group", "female", "career years (/50)", "debut unknown",
|
||||
"Christmas target week",
|
||||
"Danish x P3", "Danish x P4", "Danish x P6",
|
||||
// one dummy per channel after the first
|
||||
"ch p4kbh", "ch p4sjaelland", "ch p4fyn", "ch p4syd", "ch p4trekanten", "ch p4vest", "ch p4aarhus",
|
||||
"ch p4nord", "ch p4bornholm", "ch p4esbjerg", "ch p5", "ch p6beat", "ch p8jazz",
|
||||
];
|
||||
pub const P: usize = 25 + C - 1;
|
||||
|
||||
fn ln1p(x: f64) -> f64 { x.ln_1p() }
|
||||
|
||||
/// Is `target` (a week index, possibly past the panel) in the Christmas regime (ISO weeks 48–52)?
|
||||
pub fn christmas(p: &Panel, target: usize) -> bool {
|
||||
let last = *p.weeks.last().unwrap();
|
||||
let d = last + chrono::Duration::weeks(target as i64 - (p.w() as i64 - 1));
|
||||
d.iso_week().week() >= 48
|
||||
}
|
||||
|
||||
/// Writes the P covariates of artist a, channel c, origin t (t >= LAGS - 1) into `out`.
|
||||
pub fn row(p: &Panel, a: usize, c: usize, t: usize, target: usize, out: &mut [f64]) {
|
||||
debug_assert!(t + 1 >= LAGS && out.len() == P);
|
||||
let y = |w: usize| p.y(a, c, w);
|
||||
let others = |w: usize| (0..C).filter(|&k| k != c).map(|k| p.y(a, k, w)).sum::<f64>();
|
||||
let mean = |f: &dyn Fn(usize) -> f64, n: usize| (0..n).map(|i| f(t - i)).sum::<f64>() / n as f64;
|
||||
let max12 = (0..LAGS).map(|i| y(t - i)).fold(0.0, f64::max);
|
||||
let since = (0..LAGS).find(|&i| y(t - i) > 0.0).unwrap_or(LAGS) as f64 / LAGS as f64;
|
||||
let ch = p.chart(a, t);
|
||||
let art = &p.artists[a];
|
||||
let year = p.weeks[t].year();
|
||||
let career = art.debut.map(|d| ((year - d).clamp(0, 50)) as f64 / 50.0).unwrap_or(0.0);
|
||||
let recency = |scale: f64| if ch.weeks_since_entry == u16::MAX { 0.0 } else { (-(ch.weeks_since_entry as f64) / scale).exp() };
|
||||
let danish = art.danish as i32 as f64;
|
||||
let v = [
|
||||
1.0,
|
||||
ln1p(y(t)), ln1p(y(t - 1)), ln1p(mean(&y, 4)), ln1p(mean(&y, 12)), ln1p(max12),
|
||||
ln1p(others(t)), ln1p(mean(&others, 4)), if c == 0 { 0.0 } else { ln1p(p.y(a, 0, t)) }, since,
|
||||
if ch.t40_rank > 0 { (41.0 - ch.t40_rank as f64) / 40.0 } else { 0.0 },
|
||||
if ch.air_rank > 0 { (21.0 - ch.air_rank as f64) / 20.0 } else { 0.0 },
|
||||
recency(6.0), recency(26.0),
|
||||
ln1p(ch.t40_hits as f64), ln1p(ch.air_hits as f64),
|
||||
danish, art.group as i32 as f64, art.female as i32 as f64, career, art.debut.is_none() as i32 as f64,
|
||||
christmas(p, target) as i32 as f64,
|
||||
danish * (c == 0) as i32 as f64, danish * is_p4(c) as i32 as f64, danish * (CHANNELS[c] == "p6beat") as i32 as f64,
|
||||
];
|
||||
out[..v.len()].copy_from_slice(&v);
|
||||
for k in 1..C { out[v.len() + k - 1] = (c == k) as i32 as f64; }
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
//! Weighted logistic and negative binomial (NB2, log link) regression by Newton's method with a
|
||||
//! small ridge penalty. Rows are dense f32; the Hessian is accumulated over threads.
|
||||
|
||||
pub struct Design {
|
||||
pub p: usize,
|
||||
pub x: Vec<f32>,
|
||||
pub y: Vec<f32>,
|
||||
pub w: Vec<f32>,
|
||||
}
|
||||
|
||||
impl Design {
|
||||
pub fn new(p: usize) -> Design { Design { p, x: Vec::new(), y: Vec::new(), w: Vec::new() } }
|
||||
pub fn push(&mut self, x: &[f64], y: f64, w: f64) {
|
||||
self.x.extend(x.iter().map(|&v| v as f32));
|
||||
self.y.push(y as f32);
|
||||
self.w.push(w as f32);
|
||||
}
|
||||
pub fn n(&self) -> usize { self.y.len() }
|
||||
}
|
||||
|
||||
pub fn dot(b: &[f64], x: &[f64]) -> f64 { b.iter().zip(x).map(|(a, b)| a * b).sum() }
|
||||
|
||||
/// Gradient and Hessian (upper triangle filled, then mirrored) of a GLM whose per-row
|
||||
/// contributions are (g, h) = f(eta, y): grad += w g x, hess += w h x x'.
|
||||
fn grad_hess(d: &Design, beta: &[f64], f: &(dyn Fn(f64, f64) -> (f64, f64) + Sync)) -> (Vec<f64>, Vec<f64>) {
|
||||
let p = d.p;
|
||||
let threads = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(4).min(8);
|
||||
let chunk = d.n().div_ceil(threads).max(1);
|
||||
let parts: Vec<(Vec<f64>, Vec<f64>)> = std::thread::scope(|s| {
|
||||
let handles: Vec<_> = (0..threads).map(|t| {
|
||||
s.spawn(move || {
|
||||
let (mut g, mut h) = (vec![0.0; p], vec![0.0; p * p]);
|
||||
let lo = (t * chunk).min(d.n());
|
||||
let hi = ((t + 1) * chunk).min(d.n());
|
||||
let mut xi = vec![0.0; p];
|
||||
for i in lo..hi {
|
||||
for (k, v) in d.x[i * p..(i + 1) * p].iter().enumerate() { xi[k] = *v as f64; }
|
||||
let (gi, hi_) = f(dot(beta, &xi), d.y[i] as f64);
|
||||
let w = d.w[i] as f64;
|
||||
let (gw, hw) = (w * gi, w * hi_);
|
||||
for j in 0..p {
|
||||
if xi[j] == 0.0 { continue; }
|
||||
g[j] += gw * xi[j];
|
||||
let hj = hw * xi[j];
|
||||
let row = &mut h[j * p..(j + 1) * p];
|
||||
for k in j..p { row[k] += hj * xi[k]; }
|
||||
}
|
||||
}
|
||||
(g, h)
|
||||
})
|
||||
}).collect();
|
||||
handles.into_iter().map(|h| h.join().unwrap()).collect()
|
||||
});
|
||||
let (mut g, mut h) = (vec![0.0; p], vec![0.0; p * p]);
|
||||
for (pg, ph) in parts {
|
||||
for j in 0..p { g[j] += pg[j]; }
|
||||
for j in 0..p * p { h[j] += ph[j]; }
|
||||
}
|
||||
for j in 0..p { for k in 0..j { h[j * p + k] = h[k * p + j]; } }
|
||||
(g, h)
|
||||
}
|
||||
|
||||
/// Solves a x = b for symmetric positive definite a (overwritten) by Cholesky.
|
||||
fn solve(a: &mut [f64], b: &mut [f64], p: usize) -> bool {
|
||||
for j in 0..p {
|
||||
let mut s = a[j * p + j];
|
||||
for k in 0..j { s -= a[j * p + k] * a[j * p + k]; }
|
||||
if s <= 1e-12 { return false; }
|
||||
let l = s.sqrt();
|
||||
a[j * p + j] = l;
|
||||
for i in j + 1..p {
|
||||
let mut s = a[i * p + j];
|
||||
for k in 0..j { s -= a[i * p + k] * a[j * p + k]; }
|
||||
a[i * p + j] = s / l;
|
||||
}
|
||||
}
|
||||
for i in 0..p {
|
||||
let mut s = b[i];
|
||||
for k in 0..i { s -= a[i * p + k] * b[k]; }
|
||||
b[i] = s / a[i * p + i];
|
||||
}
|
||||
for i in (0..p).rev() {
|
||||
let mut s = b[i];
|
||||
for k in i + 1..p { s -= a[k * p + i] * b[k]; }
|
||||
b[i] = s / a[i * p + i];
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// One damped Newton step with a ridge on every coefficient but the intercept (column 0).
|
||||
fn newton(d: &Design, beta: &mut [f64], ridge: f64, f: &(dyn Fn(f64, f64) -> (f64, f64) + Sync)) -> f64 {
|
||||
let p = d.p;
|
||||
let (mut g, mut h) = grad_hess(d, beta, f);
|
||||
for j in 1..p { g[j] -= ridge * beta[j]; h[j * p + j] += ridge; }
|
||||
h[0] += 1e-9;
|
||||
if !solve(&mut h, &mut g, p) { return 0.0; }
|
||||
let step = g.iter().map(|v| v.abs()).fold(0.0, f64::max);
|
||||
let damp = if step > 5.0 { 5.0 / step } else { 1.0 };
|
||||
for j in 0..p { beta[j] += damp * g[j]; }
|
||||
step * damp
|
||||
}
|
||||
|
||||
pub fn sigmoid(e: f64) -> f64 { 1.0 / (1.0 + (-e).exp()) }
|
||||
|
||||
/// Logistic regression; y in {0, 1}.
|
||||
pub fn logit(d: &Design, ridge: f64) -> Vec<f64> {
|
||||
let mut beta = vec![0.0; d.p];
|
||||
let (sw, sy) = d.w.iter().zip(&d.y).fold((0.0, 0.0), |(a, b), (w, y)| (a + *w as f64, b + (*w * *y) as f64));
|
||||
let m = (sy / sw).clamp(1e-6, 1.0 - 1e-6);
|
||||
beta[0] = (m / (1.0 - m)).ln();
|
||||
let f = |e: f64, y: f64| { let mu = sigmoid(e.clamp(-30.0, 30.0)); (y - mu, (mu * (1.0 - mu)).max(1e-10)) };
|
||||
for _ in 0..30 {
|
||||
if newton(d, &mut beta, ridge, &f) < 1e-6 { break; }
|
||||
}
|
||||
beta
|
||||
}
|
||||
|
||||
/// Negative binomial regression, variance mu + alpha mu²; alpha by the Cameron–Trivedi moment
|
||||
/// estimator, updated between Newton steps.
|
||||
pub fn nb(d: &Design, ridge: f64) -> (Vec<f64>, f64) {
|
||||
let mut beta = vec![0.0; d.p];
|
||||
let (sw, sy) = d.w.iter().zip(&d.y).fold((0.0, 0.0), |(a, b), (w, y)| (a + *w as f64, b + (*w * *y) as f64));
|
||||
beta[0] = (sy / sw).max(1e-6).ln();
|
||||
let mut alpha = 0.5;
|
||||
for it in 0..40 {
|
||||
let a = alpha;
|
||||
let f = move |e: f64, y: f64| { let mu = e.clamp(-30.0, 12.0).exp(); ((y - mu) / (1.0 + a * mu), mu / (1.0 + a * mu)) };
|
||||
let step = newton(d, &mut beta, ridge, &f);
|
||||
if it >= 1 { alpha = moment_alpha(d, &beta); }
|
||||
if step < 1e-6 && (alpha - a).abs() < 1e-4 { break; }
|
||||
}
|
||||
// The moment estimator is a fast start; the final alpha (and the coefficients given it)
|
||||
// maximise the likelihood, which matters for the tail probabilities P(Y >= tau).
|
||||
for _ in 0..3 {
|
||||
let a0 = alpha;
|
||||
alpha = ml_alpha(d, &beta);
|
||||
let a = alpha;
|
||||
let f = move |e: f64, y: f64| { let mu = e.clamp(-30.0, 12.0).exp(); ((y - mu) / (1.0 + a * mu), mu / (1.0 + a * mu)) };
|
||||
for _ in 0..10 { if newton(d, &mut beta, ridge, &f) < 1e-6 { break; } }
|
||||
if (alpha - a0).abs() < 1e-3 * a0 { break; }
|
||||
}
|
||||
(beta, alpha)
|
||||
}
|
||||
|
||||
/// ln Γ(x) for x > 0 (Lanczos, g = 7).
|
||||
pub fn lgamma(x: f64) -> f64 {
|
||||
const G: [f64; 9] = [0.999_999_999_999_809_9, 676.520_368_121_885_1, -1_259.139_216_722_402_8, 771.323_428_777_653_1,
|
||||
-176.615_029_162_140_6, 12.507_343_278_686_905, -0.138_571_095_265_720_12, 9.984_369_578_019_572e-6, 1.505_632_735_149_311_6e-7];
|
||||
if x < 0.5 { return (std::f64::consts::PI / (std::f64::consts::PI * x).sin()).ln() - lgamma(1.0 - x); }
|
||||
let x = x - 1.0;
|
||||
let mut a = G[0];
|
||||
let t = x + 7.5;
|
||||
for (i, g) in G.iter().enumerate().skip(1) { a += g / (x + i as f64); }
|
||||
0.5 * (2.0 * std::f64::consts::PI).ln() + (x + 0.5) * t.ln() - t + a.ln()
|
||||
}
|
||||
|
||||
/// NB2 log-likelihood of y given mean mu (without the ln y! term, constant in alpha).
|
||||
fn nb_ll(y: f64, mu: f64, alpha: f64) -> f64 {
|
||||
let r = 1.0 / alpha;
|
||||
lgamma(y + r) - lgamma(r) + r * (r / (r + mu)).ln() + y * (mu / (r + mu)).ln()
|
||||
}
|
||||
|
||||
/// The alpha maximising the likelihood for fixed coefficients (golden-section search on ln alpha).
|
||||
fn ml_alpha(d: &Design, beta: &[f64]) -> f64 {
|
||||
let p = d.p;
|
||||
let mut xi = vec![0.0; p];
|
||||
let pts: Vec<(f64, f64, f64)> = (0..d.n()).map(|i| {
|
||||
for (k, v) in d.x[i * p..(i + 1) * p].iter().enumerate() { xi[k] = *v as f64; }
|
||||
(d.y[i] as f64, dot(beta, &xi).clamp(-30.0, 12.0).exp(), d.w[i] as f64)
|
||||
}).collect();
|
||||
let ll = |la: f64| { let a = la.exp(); pts.iter().map(|&(y, mu, w)| w * nb_ll(y, mu, a)).sum::<f64>() };
|
||||
let (mut lo, mut hi) = ((1e-4f64).ln(), (50.0f64).ln());
|
||||
let phi = 0.618_033_988_75;
|
||||
let (mut x1, mut x2) = (hi - phi * (hi - lo), lo + phi * (hi - lo));
|
||||
let (mut f1, mut f2) = (ll(x1), ll(x2));
|
||||
for _ in 0..40 {
|
||||
if f1 > f2 { hi = x2; x2 = x1; f2 = f1; x1 = hi - phi * (hi - lo); f1 = ll(x1); }
|
||||
else { lo = x1; x1 = x2; f1 = f2; x2 = lo + phi * (hi - lo); f2 = ll(x2); }
|
||||
}
|
||||
((lo + hi) / 2.0).exp()
|
||||
}
|
||||
|
||||
fn moment_alpha(d: &Design, beta: &[f64]) -> f64 {
|
||||
let p = d.p;
|
||||
let (mut num, mut den) = (0.0, 0.0);
|
||||
let mut xi = vec![0.0; p];
|
||||
for i in 0..d.n() {
|
||||
for (k, v) in d.x[i * p..(i + 1) * p].iter().enumerate() { xi[k] = *v as f64; }
|
||||
let mu = dot(beta, &xi).clamp(-30.0, 12.0).exp();
|
||||
let (y, w) = (d.y[i] as f64, d.w[i] as f64);
|
||||
num += w * ((y - mu).powi(2) - y);
|
||||
den += w * mu * mu;
|
||||
}
|
||||
(num / den.max(1e-12)).clamp(1e-4, 50.0)
|
||||
}
|
||||
|
||||
/// P(Y >= k) for Y ~ NB2(mu, alpha).
|
||||
pub fn nb_tail(mu: f64, alpha: f64, k: u32) -> f64 {
|
||||
if k == 0 { return 1.0; }
|
||||
let r = 1.0 / alpha;
|
||||
let q = mu / (r + mu); // "success" probability of each extra count
|
||||
let mut pmf = (r * (r / (r + mu)).ln()).exp();
|
||||
let mut cdf = pmf;
|
||||
for j in 0..k - 1 {
|
||||
pmf *= (j as f64 + r) / (j as f64 + 1.0) * q;
|
||||
cdf += pmf;
|
||||
}
|
||||
(1.0 - cdf).clamp(1e-9, 1.0 - 1e-9)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn recovers_poisson_and_logit_coefficients() {
|
||||
let mut nbd = Design::new(2);
|
||||
let mut lgd = Design::new(2);
|
||||
let mut s = 7u64;
|
||||
let mut u = || { s ^= s << 13; s ^= s >> 7; s ^= s << 17; (s >> 11) as f64 / (1u64 << 53) as f64 };
|
||||
for _ in 0..40000 {
|
||||
let x = u() * 2.0 - 1.0;
|
||||
let mu = (0.5 + 1.0 * x).exp();
|
||||
// Poisson by inversion
|
||||
let (mut k, mut pk, v) = (0u32, (-mu).exp(), u());
|
||||
let mut c = pk;
|
||||
while v > c { k += 1; pk *= mu / k as f64; c += pk; }
|
||||
nbd.push(&[1.0, x], k as f64, 1.0);
|
||||
lgd.push(&[1.0, x], (u() < sigmoid(-0.3 + 2.0 * x)) as i32 as f64, 1.0);
|
||||
}
|
||||
let (b, alpha) = nb(&nbd, 0.0);
|
||||
assert!((b[0] - 0.5).abs() < 0.05 && (b[1] - 1.0).abs() < 0.05, "{b:?}");
|
||||
assert!(alpha < 0.05, "alpha {alpha}");
|
||||
let b = logit(&lgd, 0.0);
|
||||
assert!((b[0] + 0.3).abs() < 0.06 && (b[1] - 2.0).abs() < 0.1, "{b:?}");
|
||||
assert!((lgamma(5.0) - 24f64.ln()).abs() < 1e-10 && (lgamma(0.5) - std::f64::consts::PI.sqrt().ln()).abs() < 1e-10);
|
||||
assert!((nb_tail(3.0, 1e-4, 1) - (1.0 - (-3.0f64).exp())).abs() < 1e-3);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
//! Artist form, first version (doc: "Kalman filter on log plays"): a local-level model on
|
||||
//! z = ln(1 + plays) per artist × channel. The level random-walks with variance q, each week is
|
||||
//! observed with noise r; q and r are fitted per channel by maximum likelihood over a grid,
|
||||
//! pooled across the artists who were played at all in the training weeks.
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct LocalLevel { pub q: f64, pub r: f64 }
|
||||
|
||||
/// Filtered level and its variance after the last observation, and the log-likelihood of the
|
||||
/// one-step-ahead predictions (the first observation only initialises).
|
||||
pub fn filter(z: &[f64], m: LocalLevel) -> (f64, f64, f64) {
|
||||
let (mut level, mut var, mut ll) = (z[0], m.r, 0.0);
|
||||
for &obs in &z[1..] {
|
||||
let pv = var + m.q;
|
||||
let f = pv + m.r;
|
||||
let e = obs - level;
|
||||
ll += -0.5 * ((2.0 * std::f64::consts::PI * f).ln() + e * e / f);
|
||||
let k = pv / f;
|
||||
level += k * e;
|
||||
var = (1.0 - k) * pv;
|
||||
}
|
||||
(level, var, ll)
|
||||
}
|
||||
|
||||
/// Best (q, r) on a log grid for a set of series.
|
||||
pub fn fit<'a>(series: impl Iterator<Item = &'a [f64]> + Clone) -> LocalLevel {
|
||||
let grid: Vec<f64> = (0..14).map(|i| 0.002 * 2f64.powi(i)).collect();
|
||||
let mut best = (f64::NEG_INFINITY, LocalLevel { q: 0.05, r: 0.3 });
|
||||
for &q in &grid {
|
||||
for &r in &grid {
|
||||
let m = LocalLevel { q, r };
|
||||
let ll: f64 = series.clone().map(|z| filter(z, m).2).sum();
|
||||
if ll > best.0 { best = (ll, m); }
|
||||
}
|
||||
}
|
||||
best.1
|
||||
}
|
||||
|
||||
/// Forecast h weeks ahead: mean and variance of z.
|
||||
pub fn forecast(level: f64, var: f64, m: LocalLevel, h: usize) -> (f64, f64) {
|
||||
(level, var + h as f64 * m.q + m.r)
|
||||
}
|
||||
|
||||
/// Standard normal upper tail.
|
||||
pub fn norm_sf(x: f64) -> f64 { 0.5 * erfc(x / std::f64::consts::SQRT_2) }
|
||||
|
||||
/// Complementary error function (Numerical Recipes erfcc, |error| < 1.2e-7).
|
||||
fn erfc(x: f64) -> f64 {
|
||||
let z = x.abs();
|
||||
let t = 1.0 / (1.0 + 0.5 * z);
|
||||
let r = t * (-z * z - 1.26551223 + t * (1.00002368 + t * (0.37409196 + t * (0.09678418 + t * (-0.18628806
|
||||
+ t * (0.27886807 + t * (-1.13520398 + t * (1.48851587 + t * (-0.82215223 + t * 0.17087277))))))))).exp();
|
||||
if x >= 0.0 { r } else { 2.0 - r }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn normal_tail() {
|
||||
assert!((norm_sf(0.0) - 0.5).abs() < 1e-7);
|
||||
assert!((norm_sf(1.959964) - 0.025).abs() < 1e-6);
|
||||
}
|
||||
#[test]
|
||||
fn finds_noise_levels() {
|
||||
let mut s = 11u64;
|
||||
let mut n = || { let mut t = 0.0; for _ in 0..12 { s ^= s << 13; s ^= s >> 7; s ^= s << 17; t += (s >> 11) as f64 / (1u64 << 53) as f64; } t - 6.0 };
|
||||
let series: Vec<Vec<f64>> = (0..300).map(|_| {
|
||||
let mut x = 1.0;
|
||||
(0..50).map(|_| { x += 0.3f64.sqrt() * n(); x + 0.5f64.sqrt() * n() }).collect()
|
||||
}).collect();
|
||||
let m = fit(series.iter().map(|v| v.as_slice()));
|
||||
assert!(m.q > 0.12 && m.q < 0.7 && m.r > 0.25 && m.r < 1.0, "{m:?}");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
//! Models (docs/MODELS.md). This first layer is the doc's "model we can build now": artist ×
|
||||
//! DR channel × week, trained on the DR artist pages' 52 weeks, with IFPI chart history.
|
||||
//!
|
||||
//! airplay model panel what the panel holds
|
||||
//! airplay model backtest [--h 1,4,8] [--refit 1] [--out docs/BACKTEST.md]
|
||||
//! airplay model rank [--h 4] [--max-h 12] [--top 50]
|
||||
|
||||
pub mod backtest;
|
||||
pub mod common;
|
||||
pub mod features;
|
||||
pub mod glm;
|
||||
pub mod kalman;
|
||||
pub mod panel;
|
||||
pub mod rank;
|
||||
|
||||
use crate::db::Db;
|
||||
use anyhow::Result;
|
||||
use panel::{is_p4, Panel, C, CHANNELS};
|
||||
|
||||
fn arg(args: &[String], name: &str) -> Option<String> {
|
||||
args.iter().position(|a| a == name).and_then(|i| args.get(i + 1)).cloned()
|
||||
}
|
||||
|
||||
pub fn run(db: &Db, args: &[String]) -> Result<()> {
|
||||
let p = Panel::load(db)?;
|
||||
match args.get(2).map(String::as_str) {
|
||||
Some("panel") => {
|
||||
println!("{} artists × {} channels × {} complete weeks ({} to {})", p.a(), C, p.w(), p.weeks[0], p.weeks[p.w() - 1]);
|
||||
let tau = common::thresholds(&p, p.w());
|
||||
for c in 0..C {
|
||||
let plays: f64 = (0..p.a()).flat_map(|a| (0..p.w()).map(move |w| (a, w))).map(|(a, w)| p.y(a, c, w)).sum();
|
||||
let rot = (0..p.a()).flat_map(|a| (0..p.w()).map(move |w| (a, w))).filter(|&(a, w)| p.y(a, c, w) >= tau[c] as f64).count();
|
||||
println!(" {:12} plays {:>8} τ {:>2} artist-weeks in rotation {:>6} ({:.1} artists per week)", CHANNELS[c], plays, tau[c], rot, rot as f64 / p.w() as f64);
|
||||
}
|
||||
let last = p.w() - 1;
|
||||
let charted = (0..p.a()).filter(|&a| p.chart(a, last).t40_hits > 0 || p.chart(a, last).air_hits > 0).count();
|
||||
let danish = p.artists.iter().filter(|a| a.danish).count();
|
||||
println!("artists matched to IFPI chart history: {charted}; Danish: {danish}");
|
||||
}
|
||||
Some("backtest") => {
|
||||
let horizons: Vec<usize> = arg(args, "--h").unwrap_or_else(|| "1,4,8".into()).split(',').filter_map(|s| s.trim().parse().ok()).collect();
|
||||
let refit: usize = arg(args, "--refit").and_then(|s| s.parse().ok()).unwrap_or(1).max(1);
|
||||
let report = backtest::run(&p, &horizons, refit)?;
|
||||
rank::create_tables(db)?;
|
||||
let at = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
for (m, h, metric, v) in &report.rows {
|
||||
db.conn.execute("INSERT OR REPLACE INTO backtest_scores VALUES (?1, ?2, ?3, ?4, ?5)", rusqlite::params![at, m, *h as i64, metric, v])?;
|
||||
}
|
||||
let out = arg(args, "--out").unwrap_or_else(|| "docs/BACKTEST.md".into());
|
||||
std::fs::write(&out, &report.markdown)?;
|
||||
print!("{}", report.markdown);
|
||||
eprintln!("written to {out}");
|
||||
}
|
||||
Some("rank") => {
|
||||
let h: usize = arg(args, "--h").and_then(|s| s.parse().ok()).unwrap_or(4);
|
||||
let max_h: usize = arg(args, "--max-h").and_then(|s| s.parse().ok()).unwrap_or(12).max(h);
|
||||
let top: usize = arg(args, "--top").and_then(|s| s.parse().ok()).unwrap_or(50);
|
||||
let mut lines = rank::run(db, &p, max_h, h)?;
|
||||
lines.sort_by(|a, b| b.p_any.total_cmp(&a.p_any).then(b.mean.total_cmp(&a.mean)));
|
||||
println!("Week of {} (h = {h}), model {}", p.weeks[p.w() - 1] + chrono::Duration::weeks(h as i64), rank::VERSION);
|
||||
println!("{:>3} {:28} {:>6} {:>6} {:>6} {:>6} {:>6} {:>6} {:>16}", "#", "artist", "any", "P3", "P4", "P5", "P6", "P8", "plays (80%)");
|
||||
for (i, l) in lines.iter().take(top).enumerate() {
|
||||
let p4 = (0..C).filter(|&c| is_p4(c)).map(|c| l.p_channel[c]).fold(0.0, f64::max);
|
||||
println!("{:>3} {:28} {:>6.2} {:>6.2} {:>6.2} {:>6.2} {:>6.2} {:>6.2} {:>5.0} ({:.0}–{:.0})", i + 1,
|
||||
p.artists[l.artist].name.chars().take(28).collect::<String>(), l.p_any, l.p_channel[0], p4,
|
||||
l.p_channel[11], l.p_channel[12], l.p_channel[13], l.mean, l.lo, l.hi);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
eprintln!("usage: airplay model panel | backtest [--h 1,4,8] [--refit 1] [--out FILE] | rank [--h 4] [--max-h 12] [--top 50]");
|
||||
std::process::exit(2);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
//! The artist × DR channel × week panel the first models train on (doc: "The model we can build
|
||||
//! now"). Plays come from the DR artist pages (52 Monday-to-Sunday weeks per artist); the
|
||||
//! week still running when the pages were read is left out. IFPI chart history is joined to
|
||||
//! the DR artists by name, as known at the time: chart week k counts as known from the end of
|
||||
//! week k + 1 (Hitlisten publishes with about a week's delay).
|
||||
|
||||
use crate::db::Db;
|
||||
use anyhow::{bail, Result};
|
||||
use chrono::{Duration, NaiveDate, Weekday};
|
||||
use serde_json::Value;
|
||||
use std::collections::{BTreeSet, HashMap};
|
||||
|
||||
/// The DR music channels with one play count per artist and week (the P4 total is left out:
|
||||
/// it double counts the ten regional P4s; P1 and P2 are talk and classical).
|
||||
pub const CHANNELS: [&str; 14] = [
|
||||
"p3", "p4kbh", "p4sjaelland", "p4fyn", "p4syd", "p4trekanten", "p4vest", "p4aarhus", "p4nord",
|
||||
"p4bornholm", "p4esbjerg", "p5", "p6beat", "p8jazz",
|
||||
];
|
||||
pub const C: usize = CHANNELS.len();
|
||||
|
||||
pub fn is_p4(c: usize) -> bool { CHANNELS[c].starts_with("p4") }
|
||||
|
||||
pub struct Artist {
|
||||
pub id: String,
|
||||
pub name: String,
|
||||
pub danish: bool,
|
||||
pub group: bool,
|
||||
pub female: bool,
|
||||
pub debut: Option<i32>,
|
||||
}
|
||||
|
||||
/// An artist's chart status as known at the end of a DR week.
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct Chart {
|
||||
/// Best position on the latest Track Top-40 known (0 = not on it).
|
||||
pub t40_rank: u8,
|
||||
/// Best position on the latest Airplay Top-20 known (0 = not on it).
|
||||
pub air_rank: u8,
|
||||
/// Weeks since the artist's latest new entry on either chart (u16::MAX = never).
|
||||
pub weeks_since_entry: u16,
|
||||
/// Distinct songs on the Track Top-40 / Airplay Top-20 so far.
|
||||
pub t40_hits: u16,
|
||||
pub air_hits: u16,
|
||||
}
|
||||
|
||||
pub struct Panel {
|
||||
/// Monday of each complete week, oldest first.
|
||||
pub weeks: Vec<NaiveDate>,
|
||||
pub artists: Vec<Artist>,
|
||||
y: Vec<f32>,
|
||||
chart: Vec<Chart>,
|
||||
}
|
||||
|
||||
impl Panel {
|
||||
pub fn w(&self) -> usize { self.weeks.len() }
|
||||
pub fn a(&self) -> usize { self.artists.len() }
|
||||
#[inline]
|
||||
pub fn y(&self, a: usize, c: usize, w: usize) -> f64 { self.y[(a * C + c) * self.weeks.len() + w] as f64 }
|
||||
#[inline]
|
||||
pub fn chart(&self, a: usize, w: usize) -> Chart { self.chart[a * self.weeks.len() + w] }
|
||||
|
||||
pub fn load(db: &Db) -> Result<Panel> {
|
||||
// Weeks: every Monday in the artist pages whose week had ended before the earliest page was read.
|
||||
let first_read: String = db.conn.query_row("SELECT min(fetched_at) FROM dr_artists", [], |r| r.get(0))?;
|
||||
let read_day = NaiveDate::parse_from_str(&first_read[..10], "%Y-%m-%d")?;
|
||||
let mut weeks: Vec<NaiveDate> = db.conn.prepare("SELECT DISTINCT week_start FROM dr_artist_weeks ORDER BY week_start")?
|
||||
.query_map([], |r| r.get::<_, String>(0))?
|
||||
.filter_map(|s| NaiveDate::parse_from_str(&s.ok()?, "%Y-%m-%d").ok())
|
||||
.filter(|d| *d + Duration::days(7) <= read_day)
|
||||
.collect();
|
||||
weeks.dedup();
|
||||
if weeks.len() < 30 { bail!("only {} complete weeks of DR artist history; run `airplay collect --only dr --backfill`", weeks.len()); }
|
||||
let week_of: HashMap<NaiveDate, usize> = weeks.iter().enumerate().map(|(i, d)| (*d, i)).collect();
|
||||
|
||||
let mut artists = Vec::new();
|
||||
let mut names: HashMap<String, Vec<usize>> = HashMap::new();
|
||||
let mut st = db.conn.prepare("SELECT artist_id, name, raw FROM dr_artists ORDER BY artist_id")?;
|
||||
let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?)))?;
|
||||
for r in rows {
|
||||
let (id, name, raw) = r?;
|
||||
let v: Value = serde_json::from_str(&raw).unwrap_or(Value::Null);
|
||||
let a = &v["artist"];
|
||||
let idx = artists.len();
|
||||
let mut keys = BTreeSet::from([key(&name)]);
|
||||
for n in a["names"].as_array().into_iter().flatten() {
|
||||
if let Some(s) = n["name"].as_str() { keys.insert(key(s)); }
|
||||
}
|
||||
for k in keys { if !k.is_empty() { names.entry(k).or_default().push(idx); } }
|
||||
artists.push(Artist {
|
||||
id,
|
||||
name,
|
||||
danish: a["country"].as_str() == Some("Danmark"),
|
||||
group: a["type"].as_str().is_some_and(|t| t != "person"),
|
||||
female: a["gender"].as_str() == Some("kvinde"),
|
||||
debut: a["lifespan"]["beginYear"].as_i64().map(|y| y as i32),
|
||||
});
|
||||
}
|
||||
let id_of: HashMap<String, usize> = artists.iter().enumerate().map(|(i, a)| (a.id.clone(), i)).collect();
|
||||
let ch_of: HashMap<&str, usize> = CHANNELS.iter().enumerate().map(|(i, c)| (*c, i)).collect();
|
||||
|
||||
let nw = weeks.len();
|
||||
let mut y = vec![0f32; artists.len() * C * nw];
|
||||
let mut st = db.conn.prepare("SELECT artist_id, week_start, channel, plays FROM dr_artist_weeks")?;
|
||||
let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, i64>(3)?)))?;
|
||||
for r in rows {
|
||||
let (id, ws, ch, plays) = r?;
|
||||
let (Some(&a), Some(&c)) = (id_of.get(&id), ch_of.get(ch.as_str())) else { continue };
|
||||
let Ok(d) = NaiveDate::parse_from_str(&ws, "%Y-%m-%d") else { continue };
|
||||
let Some(&w) = week_of.get(&d) else { continue };
|
||||
y[(a * C + c) * nw + w] = plays as f32;
|
||||
}
|
||||
|
||||
let chart = chart_states(db, &weeks, &names, artists.len())?;
|
||||
Ok(Panel { weeks, artists, y, chart })
|
||||
}
|
||||
}
|
||||
|
||||
/// Lower-case letters and digits (æ, ø, å kept), for matching names.
|
||||
pub fn key(s: &str) -> String {
|
||||
s.to_lowercase().chars().filter(|c| c.is_alphanumeric()).collect()
|
||||
}
|
||||
|
||||
/// The artists named in a chart's artist field ("ANNIKA FEAT. LAMIN" -> annika, lamin).
|
||||
fn chart_artists(s: &str, names: &HashMap<String, Vec<usize>>) -> Vec<usize> {
|
||||
let mut found: Vec<usize> = names.get(&key(s)).cloned().unwrap_or_default();
|
||||
let mut parts = vec![s.to_lowercase()];
|
||||
for sep in [" feat. ", " feat ", " ft. ", " ft ", " featuring ", " & ", ", ", " x ", " og ", " med ", " vs. ", " vs ", " + ", " / "] {
|
||||
parts = parts.iter().flat_map(|p| p.split(sep).map(str::to_string).collect::<Vec<_>>()).collect();
|
||||
}
|
||||
if parts.len() > 1 {
|
||||
for p in parts { if let Some(v) = names.get(&key(&p)) { found.extend(v); } }
|
||||
}
|
||||
found.sort_unstable();
|
||||
found.dedup();
|
||||
found
|
||||
}
|
||||
|
||||
fn chart_states(db: &Db, weeks: &[NaiveDate], names: &HashMap<String, Vec<usize>>, n_artists: usize) -> Result<Vec<Chart>> {
|
||||
struct Entry { monday: NaiveDate, t40: bool, rank: u8, new: bool, song: String }
|
||||
let mut by_artist: Vec<Vec<Entry>> = (0..n_artists).map(|_| Vec::new()).collect();
|
||||
let mut mondays: [BTreeSet<NaiveDate>; 2] = [BTreeSet::new(), BTreeSet::new()];
|
||||
let mut st = db.conn.prepare(
|
||||
"SELECT chart, year, week, position, coalesce(last_week, ''), weeks_on_chart, artist, title
|
||||
FROM hitlisten_charts WHERE chart IN ('t40', 'airplay')")?;
|
||||
let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i32>(1)?, r.get::<_, u32>(2)?, r.get::<_, i64>(3)?,
|
||||
r.get::<_, String>(4)?, r.get::<_, Option<i64>>(5)?, r.get::<_, String>(6)?, r.get::<_, String>(7)?)))?;
|
||||
for r in rows {
|
||||
let (chart, year, week, pos, last, weeks_on, artist, title) = r?;
|
||||
let Some(monday) = NaiveDate::from_isoywd_opt(year, week, Weekday::Mon) else { continue };
|
||||
let t40 = chart == "t40";
|
||||
mondays[t40 as usize].insert(monday);
|
||||
let new = weeks_on == Some(1) || last.trim().eq_ignore_ascii_case("NY");
|
||||
for a in chart_artists(&artist, names) {
|
||||
by_artist[a].push(Entry { monday, t40, rank: pos.clamp(0, 255) as u8, new, song: key(&title) });
|
||||
}
|
||||
}
|
||||
let nw = weeks.len();
|
||||
let mut out = Vec::with_capacity(n_artists * nw);
|
||||
// The latest chart week of each kind known at the end of each DR week.
|
||||
let latest = |t40: bool, cutoff: NaiveDate| mondays[t40 as usize].range(..=cutoff).next_back().copied();
|
||||
let cutoffs: Vec<NaiveDate> = weeks.iter().map(|d| *d - Duration::days(7)).collect();
|
||||
let latest_t40: Vec<Option<NaiveDate>> = cutoffs.iter().map(|c| latest(true, *c)).collect();
|
||||
let latest_air: Vec<Option<NaiveDate>> = cutoffs.iter().map(|c| latest(false, *c)).collect();
|
||||
for entries in &by_artist {
|
||||
for w in 0..nw {
|
||||
let cutoff = cutoffs[w];
|
||||
let mut s = Chart { t40_rank: 0, air_rank: 0, weeks_since_entry: u16::MAX, t40_hits: 0, air_hits: 0 };
|
||||
let (mut t40_songs, mut air_songs) = (BTreeSet::new(), BTreeSet::new());
|
||||
for e in entries.iter().filter(|e| e.monday <= cutoff) {
|
||||
if e.t40 { t40_songs.insert(&e.song); } else { air_songs.insert(&e.song); }
|
||||
let best = |cur: u8| if cur == 0 { e.rank } else { cur.min(e.rank) };
|
||||
if e.t40 && Some(e.monday) == latest_t40[w] { s.t40_rank = best(s.t40_rank); }
|
||||
if !e.t40 && Some(e.monday) == latest_air[w] { s.air_rank = best(s.air_rank); }
|
||||
if e.new {
|
||||
let since = ((cutoff - e.monday).num_days() / 7).clamp(0, u16::MAX as i64 - 1) as u16;
|
||||
s.weeks_since_entry = s.weeks_since_entry.min(since);
|
||||
}
|
||||
}
|
||||
s.t40_hits = t40_songs.len() as u16;
|
||||
s.air_hits = air_songs.len() as u16;
|
||||
out.push(s);
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
//! The ranking, version 0 (doc: "The ranking calculation"). For every artist, horizon h = 1–12
|
||||
//! weeks after the latest complete week and DR channel: P(in rotation), expected plays, and,
|
||||
//! across channels, P(in rotation on at least one) and total plays with an 80% interval.
|
||||
//!
|
||||
//! The artist's state is drawn once per simulation run and shared by all channels (a common
|
||||
//! gamma frailty, so each channel's marginal stays the fitted NB), so channels aren't treated as
|
||||
//! independent. Not yet conditioned on a release at t + h: that needs the release-effect curve
|
||||
//! from release dates (Deezer backfill). Until then this ranks expected airplay given current form.
|
||||
|
||||
use super::common::{thresholds, Direct, Rng};
|
||||
use super::features::P;
|
||||
use super::panel::{Panel, C, CHANNELS};
|
||||
use crate::db::Db;
|
||||
use anyhow::Result;
|
||||
use chrono::{Duration, Utc};
|
||||
use rusqlite::params;
|
||||
|
||||
pub const VERSION: &str = "dr-nb-v0";
|
||||
|
||||
pub struct Line {
|
||||
pub artist: usize,
|
||||
pub p_channel: [f64; C],
|
||||
pub mean_channel: [f64; C],
|
||||
pub p_any: f64,
|
||||
pub mean: f64,
|
||||
pub lo: f64,
|
||||
pub hi: f64,
|
||||
}
|
||||
|
||||
pub fn create_tables(db: &Db) -> Result<()> {
|
||||
db.conn.execute_batch(
|
||||
"CREATE TABLE IF NOT EXISTS forecasts (
|
||||
made_at TEXT NOT NULL, model_version TEXT NOT NULL,
|
||||
target TEXT NOT NULL, -- 'rotation' (prob, mean plays) per channel, or 'any' across channels
|
||||
key TEXT NOT NULL, -- '{dr artist id}:{channel}' or '{dr artist id}:any'
|
||||
horizon INTEGER NOT NULL, -- weeks after origin_week
|
||||
origin_week TEXT NOT NULL, target_week TEXT NOT NULL,
|
||||
prob REAL, mean REAL, lo REAL, hi REAL,
|
||||
PRIMARY KEY (made_at, model_version, target, key, horizon)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS backtest_scores (
|
||||
run_at TEXT NOT NULL, model TEXT NOT NULL, horizon INTEGER NOT NULL, metric TEXT NOT NULL, value REAL,
|
||||
PRIMARY KEY (run_at, model, horizon, metric)
|
||||
);",
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn forecast(p: &Panel, h: usize, sims: usize) -> Vec<Line> {
|
||||
let t0 = p.w() - 1;
|
||||
let tau = thresholds(p, p.w());
|
||||
let d = Direct::fit(p, h, t0, &tau, false);
|
||||
let mut x = vec![0.0; P];
|
||||
let mut rng = Rng(0x5eed ^ h as u64);
|
||||
let shape = 1.0 / d.alpha;
|
||||
(0..p.a()).map(|a| {
|
||||
let mut mean_channel = [0.0; C];
|
||||
let mut p_channel = [0.0; C];
|
||||
for c in 0..C {
|
||||
let (mu, pr, _) = d.predict(p, a, c, t0, tau[c], &mut x);
|
||||
mean_channel[c] = mu;
|
||||
p_channel[c] = pr;
|
||||
}
|
||||
let mut totals = Vec::with_capacity(sims);
|
||||
let mut any = 0usize;
|
||||
for _ in 0..sims {
|
||||
let g = rng.gamma(shape) / shape;
|
||||
let (mut tot, mut hit) = (0.0, false);
|
||||
for c in 0..C {
|
||||
let y = rng.poisson(mean_channel[c] * g);
|
||||
tot += y;
|
||||
hit |= y >= tau[c] as f64;
|
||||
}
|
||||
any += hit as usize;
|
||||
totals.push(tot);
|
||||
}
|
||||
totals.sort_by(f64::total_cmp);
|
||||
let q = |f: f64| totals[((sims as f64 - 1.0) * f).round() as usize];
|
||||
Line { artist: a, p_channel, mean_channel, p_any: any as f64 / sims as f64, mean: mean_channel.iter().sum(), lo: q(0.1), hi: q(0.9) }
|
||||
}).collect()
|
||||
}
|
||||
|
||||
/// Forecasts for h = 1..=max_h, stored in `forecasts`; returns the lines of `show_h`.
|
||||
pub fn run(db: &Db, p: &Panel, max_h: usize, show_h: usize) -> Result<Vec<Line>> {
|
||||
create_tables(db)?;
|
||||
let made = Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
let origin = p.weeks[p.w() - 1];
|
||||
let mut shown = Vec::new();
|
||||
for h in 1..=max_h {
|
||||
let lines = forecast(p, h, 2000);
|
||||
let target = (origin + Duration::weeks(h as i64)).to_string();
|
||||
let tx = db.conn.unchecked_transaction()?;
|
||||
{
|
||||
let mut ins = tx.prepare("INSERT OR REPLACE INTO forecasts VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)")?;
|
||||
for l in &lines {
|
||||
let id = &p.artists[l.artist].id;
|
||||
ins.execute(params![made, VERSION, "any", format!("{id}:any"), h as i64, origin.to_string(), target, l.p_any, l.mean, l.lo, l.hi])?;
|
||||
for c in 0..C {
|
||||
ins.execute(params![made, VERSION, "rotation", format!("{id}:{}", CHANNELS[c]), h as i64, origin.to_string(), target,
|
||||
l.p_channel[c], l.mean_channel[c], Option::<f64>::None, Option::<f64>::None])?;
|
||||
}
|
||||
}
|
||||
}
|
||||
tx.commit()?;
|
||||
eprintln!("rank: h={h} stored ({} artists)", lines.len());
|
||||
if h == show_h { shown = lines; }
|
||||
}
|
||||
Ok(shown)
|
||||
}
|
||||
Reference in New Issue
Block a user