Knowable pool (artists enter only from their first release/play/chart/list/NMF), pooled out-of-sample recalibration (Platt vs isotonic; the site uses pooled Platt per channel, isotonic any channel), release-timing baseline, 8-week bootstrap blocks, surge-rule precision/recall vs the lists with a constant grid (docs/ADDCHECK.md); backtests restructured to score all variants with calibration tables
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+248
-107
@@ -19,6 +19,7 @@
|
||||
use super::common::{mix, Rng};
|
||||
use super::features::{self, LAGS, P};
|
||||
use super::glm::{self, Design};
|
||||
use super::calib::Calib;
|
||||
use super::panel::{group, Panel, C, GROUPS};
|
||||
use anyhow::Result;
|
||||
use std::fmt::Write as _;
|
||||
@@ -96,6 +97,22 @@ pub fn released_in_week(p: &Panel, a: usize, w: usize) -> bool {
|
||||
dates.iter().any(|d| *d >= start && *d < start + chrono::Duration::days(7))
|
||||
}
|
||||
|
||||
/// Weeks since the artist's latest release as of week t, in bins: <6, 6–11, 12–25, 26–51, 52+, none.
|
||||
pub fn since_bin(p: &Panel, a: usize, t: usize) -> usize {
|
||||
let Some(dates) = p.releases[a].as_ref() else { return 5 };
|
||||
let end_t = p.weeks[t] + chrono::Duration::days(6);
|
||||
match dates.iter().rev().find(|d| **d <= end_t).map(|d| (end_t - *d).num_days() / 7) {
|
||||
None => 5, Some(w) if w < 6 => 0, Some(w) if w < 12 => 1, Some(w) if w < 26 => 2, Some(w) if w < 52 => 3, Some(_) => 4,
|
||||
}
|
||||
}
|
||||
|
||||
/// Does the artist release something in the window weeks t+h+1 ..= t+h+win?
|
||||
pub fn releases_in(p: &Panel, a: usize, t: usize, h: usize, win: usize) -> bool {
|
||||
let Some(dates) = p.releases[a].as_ref() else { return false };
|
||||
let (w0, w1) = window(p, t, h, win);
|
||||
dates.iter().any(|d| *d >= w0 && *d < w1)
|
||||
}
|
||||
|
||||
/// A clean release event in week w: a release that week and none in the 6 weeks before it, so an
|
||||
/// add that follows can't be the previous song's (DR's counts are per artist, not per song).
|
||||
pub const CLEAN_WEEKS: usize = 6;
|
||||
@@ -242,8 +259,15 @@ pub struct Rot {
|
||||
pub static ADD_RULE: std::sync::OnceLock<String> = std::sync::OnceLock::new();
|
||||
fn surge_rule() -> bool { ADD_RULE.get().is_none_or(|r| r != "onset") }
|
||||
|
||||
/// Surge-rule constants: [floor (× τ), jump over the baseline (× τ), jump (× baseline), held the
|
||||
/// next week (× τ and × baseline)]. Tuned against DR's lists (`airplay model addcheck --tune`).
|
||||
pub const SURGE_DEFAULT: [f64; 4] = [0.6, 0.5, 0.5, 0.25];
|
||||
pub static SURGE: std::sync::OnceLock<[f64; 4]> = std::sync::OnceLock::new();
|
||||
|
||||
impl Rot {
|
||||
pub fn new(p: &Panel, tau: &[u32; C]) -> Rot {
|
||||
pub fn new(p: &Panel, tau: &[u32; C]) -> Rot { Self::new_with(p, tau, *SURGE.get().unwrap_or(&SURGE_DEFAULT)) }
|
||||
|
||||
pub fn new_with(p: &Panel, tau: &[u32; C], k: [f64; 4]) -> Rot {
|
||||
let nw = p.w();
|
||||
let mut rot = vec![false; p.a() * C * nw];
|
||||
let mut any = vec![false; p.a() * nw];
|
||||
@@ -266,7 +290,7 @@ impl Rot {
|
||||
b.sort_by(f64::total_cmp);
|
||||
let base = b[4];
|
||||
let (y0, y1) = (p.y(a, c, w), p.y(a, c, w + 1));
|
||||
y0 >= (0.6 * t).max(base + (0.5 * t).max(0.5 * base)) && y1 >= (0.6 * t).max(base + (0.25 * t).max(0.25 * base))
|
||||
y0 >= (k[0] * t).max(base + (k[1] * t).max(k[2] * base)) && y1 >= (k[0] * t).max(base + (k[3] * t).max(k[3] * base))
|
||||
};
|
||||
if (onset || jump) && (1..=GAP).all(|k| !adds[(a * C + c) * nw + w - k]) { adds[(a * C + c) * nw + w] = true; }
|
||||
}
|
||||
@@ -366,6 +390,9 @@ pub struct AddModel {
|
||||
/// enough history to fit it honestly.
|
||||
pub platt_channel: (f64, f64),
|
||||
pub platt_any: (f64, f64),
|
||||
/// Recalibration fitted on pooled out-of-sample forecasts (replaces the Platt pair when set).
|
||||
pub calib_channel: Option<Calib>,
|
||||
pub calib_any: Option<Calib>,
|
||||
}
|
||||
|
||||
/// Recent origins used to fit the recalibration.
|
||||
@@ -393,8 +420,39 @@ impl AddModel {
|
||||
/// the artist's state the week before (earlier releases and NMF placements count as history;
|
||||
/// the new song's own NMF placement is the scenario variable).
|
||||
pub fn fit_events(p: &Panel, r: &Rot, last_week: usize) -> AddModel { Self::fit_with(p, r, 0, last_week, true, W) }
|
||||
/// The site's release-event model (8-week window, pooled recalibration).
|
||||
pub fn fit_events_site(p: &Panel, r: &Rot, last_week: usize) -> AddModel { Self::fit_events_pooled(p, r, last_week, W) }
|
||||
/// The release-event model for a window of `win` weeks.
|
||||
pub fn fit_events_w(p: &Panel, r: &Rot, last_week: usize, win: usize) -> AddModel { Self::fit_with(p, r, 0, last_week, true, win) }
|
||||
pub fn fit_events_w(p: &Panel, r: &Rot, last_week: usize, win: usize) -> AddModel { Self::fit_events_pooled(p, r, last_week, win) }
|
||||
|
||||
/// The release-event model recalibrated on pooled out-of-sample forecasts: a rolling refit (every
|
||||
/// 4 weeks) predicts each later week's clean releases from releases whose outcome was already
|
||||
/// known; Platt per station and isotonic for any station are fitted on all those forecasts
|
||||
/// (the combination that scored best in the backtest), then attached to the model fitted on
|
||||
/// everything known by `last_week`.
|
||||
pub fn fit_events_pooled(p: &Panel, r: &Rot, last_week: usize, win: usize) -> AddModel {
|
||||
let (mut pc_pairs, mut pa_pairs) = (Vec::new(), Vec::new());
|
||||
let first = LAGS - 1 + win + 1 + 8;
|
||||
let last = last_week.saturating_sub(win + 1);
|
||||
let mut model: Option<AddModel> = None;
|
||||
for t0 in first..=last {
|
||||
if (t0 - first) % 4 == 0 || model.is_none() { model = Some(Self::fit_raw(p, r, 0, t0, true, win)); }
|
||||
let m = model.as_ref().unwrap();
|
||||
for a in 0..p.a() {
|
||||
if !p.known(a, t0) || !clean_release(p, a, t0 + 1) { continue; }
|
||||
let q = p_nmf(p, a, t0);
|
||||
let sc = |b: bool| Scenario::new(true, b).window(win);
|
||||
let (rcn, ran) = m.predict_raw(p, r, a, t0, 0, sc(true));
|
||||
let (rcr, rar) = m.predict_raw(p, r, a, t0, 0, sc(false));
|
||||
pa_pairs.push((q * ran + (1.0 - q) * rar, r.add_any_in(a, t0, 0, win)));
|
||||
for c in 0..C { pc_pairs.push((q * rcn[c] + (1.0 - q) * rcr[c], r.add_in(a, c, t0, 0, win))); }
|
||||
}
|
||||
}
|
||||
let mut m = Self::fit_raw(p, r, 0, last_week, true, win);
|
||||
m.calib_channel = Some(Calib::platt(&pc_pairs));
|
||||
m.calib_any = Some(Calib::isotonic(&pa_pairs));
|
||||
m
|
||||
}
|
||||
|
||||
fn fit_with(p: &Panel, r: &Rot, h: usize, last_week: usize, events: bool, win: usize) -> AddModel {
|
||||
let mut m = Self::fit_raw(p, r, h, last_week, events, win);
|
||||
@@ -407,6 +465,7 @@ impl AddModel {
|
||||
let (mut dc, mut da) = (Design::new(2), Design::new(2));
|
||||
for t in c0..=hi {
|
||||
for a in 0..p.a() {
|
||||
if !p.known(a, t) { continue; }
|
||||
if events && !clean_release(p, a, t + 1) { continue; }
|
||||
let (pc, pa) = inner.predict(p, r, a, t, Scenario::actual(win), None);
|
||||
for c in 0..C {
|
||||
@@ -431,6 +490,8 @@ impl AddModel {
|
||||
let sc = Scenario::actual(win);
|
||||
for t in origins_w(h, last_week, win) {
|
||||
for a in 0..p.a() {
|
||||
// Only artists knowable at the origin (no survivorship from later discovery).
|
||||
if !p.known(a, t) { continue; }
|
||||
if events && !clean_release(p, a, t + 1) { continue; }
|
||||
for c in 0..C {
|
||||
// Channels where the artist hasn't been heard for 12 weeks: keep a quarter, weighted 4.
|
||||
@@ -443,7 +504,7 @@ impl AddModel {
|
||||
da.push(&xa, r.add_any_in(a, t, h, win) as i32 as f64, 1.0);
|
||||
}
|
||||
}
|
||||
AddModel { h, win, channel: glm::logit(&dc, 1.0), any: glm::logit(&da, 1.0), platt_channel: (0.0, 1.0), platt_any: (0.0, 1.0) }
|
||||
AddModel { h, win, channel: glm::logit(&dc, 1.0), any: glm::logit(&da, 1.0), platt_channel: (0.0, 1.0), platt_any: (0.0, 1.0), calib_channel: None, calib_any: None }
|
||||
}
|
||||
|
||||
/// (P(add) per channel, P(add on any channel)) from origin t.
|
||||
@@ -455,13 +516,24 @@ impl AddModel {
|
||||
/// As `predict`, for a window starting h weeks after t (the release-event model: a release in
|
||||
/// week t+h+1, with the time-since features moved forward by `sc.shift`).
|
||||
pub fn predict_at(&self, p: &Panel, r: &Rot, a: usize, t: usize, h: usize, sc: Scenario, chart: Option<&dyn Fn(&mut [f64], [usize; 4])>) -> ([f64; C], f64) {
|
||||
self.predict_full(p, r, a, t, h, sc, chart, false)
|
||||
}
|
||||
|
||||
/// The logits' own probabilities, without recalibration or the any-channel cap.
|
||||
pub fn predict_raw(&self, p: &Panel, r: &Rot, a: usize, t: usize, h: usize, sc: Scenario) -> ([f64; C], f64) {
|
||||
self.predict_full(p, r, a, t, h, sc, None, true)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn predict_full(&self, p: &Panel, r: &Rot, a: usize, t: usize, h: usize, sc: Scenario, chart: Option<&dyn Fn(&mut [f64], [usize; 4])>, raw: bool) -> ([f64; C], f64) {
|
||||
let sc = sc.window(self.win);
|
||||
let mut x = vec![0.0; PC];
|
||||
let mut pc = [0.0; C];
|
||||
for c in 0..C {
|
||||
row_channel(p, r, a, c, t, h, sc, &mut x);
|
||||
if let Some(f) = chart { f(&mut x, CHART_IDX_CHANNEL); }
|
||||
pc[c] = platt(self.platt_channel, glm::sigmoid(glm::dot(&self.channel, &x)));
|
||||
let p0 = glm::sigmoid(glm::dot(&self.channel, &x)).clamp(1e-6, 1.0 - 1e-6);
|
||||
pc[c] = if raw { p0 } else { self.calib_channel.as_ref().map(|k| k.apply(p0)).unwrap_or_else(|| platt(self.platt_channel, p0)) };
|
||||
}
|
||||
let mut xa = vec![0.0; PA];
|
||||
row_any(p, r, a, t, h, sc, &mut xa);
|
||||
@@ -471,7 +543,10 @@ impl AddModel {
|
||||
let mut best = [0.0f64; GROUPS];
|
||||
for c in 0..C { best[group(c)] = best[group(c)].max(pc[c]); }
|
||||
let cap = 1.0 - best.iter().map(|p| 1.0 - p).product::<f64>();
|
||||
(pc, platt(self.platt_any, glm::sigmoid(glm::dot(&self.any, &xa))).min(cap.max(1e-6)))
|
||||
let p0 = glm::sigmoid(glm::dot(&self.any, &xa)).clamp(1e-6, 1.0 - 1e-6);
|
||||
if raw { return (pc, p0); }
|
||||
let pa = self.calib_any.as_ref().map(|k| k.apply(p0)).unwrap_or_else(|| platt(self.platt_any, p0));
|
||||
(pc, pa.min(cap.max(1e-6)))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -499,7 +574,8 @@ fn average_precision(mut v: Vec<(f64, bool)>) -> f64 {
|
||||
fn skill(model: &[f64], base: &[f64]) -> (f64, f64, f64) {
|
||||
let point = 1.0 - model.iter().sum::<f64>() / base.iter().sum::<f64>();
|
||||
let n = model.len();
|
||||
let block = 3.min(n);
|
||||
// Blocks of 8 weeks: neighbouring origins share most of an 8-week target window.
|
||||
let block = 8.min(n);
|
||||
let mut rng = Rng(7);
|
||||
let mut d: Vec<f64> = (0..2000).map(|_| {
|
||||
let (mut lm, mut lb, mut k) = (0.0, 0.0, 0);
|
||||
@@ -510,33 +586,122 @@ fn skill(model: &[f64], base: &[f64]) -> (f64, f64, f64) {
|
||||
(point, d[100], d[1899])
|
||||
}
|
||||
|
||||
/// Scores of several models over walk-forward origins.
|
||||
struct Scores {
|
||||
names: Vec<&'static str>,
|
||||
wk: Vec<Vec<f64>>, // per origin, per model: summed log-loss
|
||||
pairs: Vec<Vec<(f64, bool)>>, // per model
|
||||
calib: Vec<[[f64; 3]; 10]>, // per model: [sum p, hits, n] per decile
|
||||
n: f64,
|
||||
hits: f64,
|
||||
}
|
||||
|
||||
impl Scores {
|
||||
fn new(names: Vec<&'static str>) -> Scores {
|
||||
let k = names.len();
|
||||
Scores { names, wk: Vec::new(), pairs: vec![Vec::new(); k], calib: vec![[[0.0; 3]; 10]; k], n: 0.0, hits: 0.0 }
|
||||
}
|
||||
fn start_origin(&mut self) { self.wk.push(vec![0.0; self.names.len()]); }
|
||||
fn add(&mut self, preds: &[f64], y: bool) {
|
||||
let w = self.wk.last_mut().unwrap();
|
||||
for (k, p) in preds.iter().enumerate() {
|
||||
let p = p.clamp(1e-6, 1.0 - 1e-6);
|
||||
w[k] += ll(p, y);
|
||||
self.pairs[k].push((p, y));
|
||||
let d = ((p * 10.0) as usize).min(9);
|
||||
self.calib[k][d][0] += p; self.calib[k][d][1] += y as i32 as f64; self.calib[k][d][2] += 1.0;
|
||||
}
|
||||
self.n += 1.0; self.hits += y as i32 as f64;
|
||||
}
|
||||
fn col(&self, k: usize) -> Vec<f64> { self.wk.iter().map(|w| w[k]).collect() }
|
||||
fn logloss(&self, k: usize) -> f64 { self.col(k).iter().sum::<f64>() / self.n.max(1.0) }
|
||||
fn prauc(&self, k: usize) -> f64 { average_precision(self.pairs[k].clone()) }
|
||||
fn calib_table(&self, md: &mut String, ks: &[usize]) -> Result<()> {
|
||||
write!(md, "| Predicted |")?;
|
||||
for &k in ks { write!(md, " {}: rows / mean predicted / observed |", self.names[k])?; }
|
||||
writeln!(md, "\n|---|{}", ks.iter().map(|_| "---|").collect::<String>())?;
|
||||
for d in 0..10 {
|
||||
if ks.iter().all(|&k| self.calib[k][d][2] == 0.0) { continue; }
|
||||
write!(md, "| {:.1}–{:.1} |", d as f64 / 10.0, (d + 1) as f64 / 10.0)?;
|
||||
for &k in ks {
|
||||
let c = self.calib[k][d];
|
||||
if c[2] > 0.0 { write!(md, " {} / {:.3} / {:.3} |", c[2], c[0] / c[2], c[1] / c[2])?; } else { write!(md, " — |")?; }
|
||||
}
|
||||
writeln!(md)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Pooled out-of-sample forecasts for recalibration: (origin, raw prediction, outcome).
|
||||
struct Pool(Vec<(usize, f64, bool)>);
|
||||
impl Pool {
|
||||
/// Platt and isotonic fitted on forecasts whose outcome was known by `t0` (origin + gap <= t0).
|
||||
fn fit(&self, t0: usize, gap: usize) -> (Calib, Calib) {
|
||||
let pairs: Vec<(f64, bool)> = self.0.iter().filter(|(o, _, _)| o + gap <= t0).map(|(_, p, y)| (*p, *y)).collect();
|
||||
(Calib::platt(&pairs), Calib::isotonic(&pairs))
|
||||
}
|
||||
}
|
||||
|
||||
/// The models scored in both backtests (index: meaning).
|
||||
const SCORED: [&str; 8] = ["rate by in-rotation-now", "rate by last-12-weeks level", "release timing only",
|
||||
"model, calibrated on the last 6 origins (pre-release)", "model, raw (pre-release)",
|
||||
"model + pooled Platt (pre-release)", "model + pooled isotonic (pre-release)", "model, NMF known (post-release)"];
|
||||
|
||||
fn table(md: &mut String, s_c: &Scores, s_a: &Scores, rows: &mut Vec<(String, usize, String, f64)>, h: usize, prefix: &str) -> Result<()> {
|
||||
writeln!(md, "| Model | Log-loss, per channel | PR-AUC, per channel | Log-loss, any channel | PR-AUC, any channel |\n|---|---|---|---|---|")?;
|
||||
for k in 0..SCORED.len() {
|
||||
let (lc, pc, la, pa) = (s_c.logloss(k), s_c.prauc(k), s_a.logloss(k), s_a.prauc(k));
|
||||
writeln!(md, "| {} | {lc:.4} | {pc:.3} | {la:.4} | {pa:.3} |", SCORED[k])?;
|
||||
for (m, v) in [("logloss_ch", lc), ("prauc_ch", pc), ("logloss_any", la), ("prauc_any", pa)] {
|
||||
rows.push((SCORED[k].to_string(), h, format!("{prefix}{m}"), v));
|
||||
}
|
||||
}
|
||||
writeln!(md, "\nSkill (1 − log-loss ÷ baseline log-loss; 90% interval from a bootstrap over blocks of 8 forecast weeks):\n")?;
|
||||
writeln!(md, "| Model | Per channel vs last-12-weeks | Per channel vs release timing | Any channel vs last-12-weeks | Any channel vs release timing |\n|---|---|---|---|---|")?;
|
||||
for k in [3usize, 5, 6] {
|
||||
let f = |s: &Scores, b: usize| { let x = skill(&s.col(k), &s.col(b)); format!("{:+.3} ({:+.3} to {:+.3})", x.0, x.1, x.2) };
|
||||
writeln!(md, "| {} | {} | {} | {} | {} |", SCORED[k], f(s_c, 1), f(s_c, 2), f(s_a, 1), f(s_a, 2))?;
|
||||
rows.push((SCORED[k].to_string(), h, format!("{prefix}skill_any_vs_level"), skill(&s_a.col(k), &s_a.col(1)).0));
|
||||
rows.push((SCORED[k].to_string(), h, format!("{prefix}skill_ch_vs_level"), skill(&s_c.col(k), &s_c.col(1)).0));
|
||||
}
|
||||
writeln!(md, "\nCalibration, any channel:\n")?;
|
||||
s_a.calib_table(md, &[3, 5, 6])?;
|
||||
writeln!(md, "\nCalibration, per channel:\n")?;
|
||||
s_c.calib_table(md, &[3, 5, 6])?;
|
||||
writeln!(md)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Release-timing-only rates: per station (and any) by whether the artist releases in the window
|
||||
/// and weeks since their last release. [key][hits, n]
|
||||
struct Timing { ch: Vec<[f64; 2]>, any: Vec<[f64; 2]> }
|
||||
impl Timing {
|
||||
fn new() -> Timing { Timing { ch: vec![[1.0, 50.0]; C * 12], any: vec![[1.0, 20.0]; 12] } }
|
||||
fn key(p: &Panel, a: usize, t: usize, h: usize, win: usize) -> usize { releases_in(p, a, t, h, win) as usize * 6 + since_bin(p, a, t) }
|
||||
}
|
||||
|
||||
/// Walk-forward: at each origin t0 the models see only windows that had closed by t0.
|
||||
pub fn backtest(p: &Panel, tau: &[u32; C], horizons: &[usize], start: usize, md: &mut String) -> Result<Vec<(String, usize, String, f64)>> {
|
||||
let r = Rot::new(p, tau);
|
||||
let mut rows = Vec::new();
|
||||
writeln!(md, "# Backtest: new adds\n")?;
|
||||
writeln!(md, "An add = the artist enters rotation (plays ≥ τ) after at least {GAP} weeks out of rotation: per channel, and on any DR channel after {GAP} weeks out of rotation everywhere. Target: an add within the {W} weeks that start h weeks after the forecast origin. Origins from week {}; each fit uses only windows closed by its origin.\n", start + 1)?;
|
||||
writeln!(md, "An add = the artist enters rotation (plays ≥ τ) after at least {GAP} weeks out of rotation, or their plays jump well above their own 8-week baseline and stay up; per channel, and on any DR channel. Target: an add within the {W} weeks that start h weeks after the forecast origin. Origins from week {}; each fit uses only windows closed (a week before) its origin; only artists knowable at the origin. Pre-release rows blend the window's NMF placement from the artist's history; release timing in the window is a scenario input.\n", start + 1)?;
|
||||
for &h in horizons {
|
||||
let last_origin = p.w() - 1 - h - W;
|
||||
if last_origin < start { continue; }
|
||||
// [model][week] log-loss sums; models: base(rot now), base(popularity bins), logit
|
||||
let names = ["rate by in-rotation-now", "rate by last-12-weeks level", "logit (pre-release: NMF from history)", "logit (post-release: NMF known)"];
|
||||
let mut wk_c: Vec<[f64; 4]> = Vec::new();
|
||||
let mut wk_a: Vec<[f64; 4]> = Vec::new();
|
||||
let mut pairs_c: [Vec<(f64, bool)>; 4] = Default::default();
|
||||
let mut pairs_a: [Vec<(f64, bool)>; 4] = Default::default();
|
||||
let mut calib = [[0.0f64; 3]; 10];
|
||||
let (mut nc, mut na, mut hits_c, mut hits_a) = (0.0, 0.0, 0.0, 0.0);
|
||||
if p.w() < start + h + W + 2 { continue; }
|
||||
let last_origin = p.w() - 1 - h - W - 1;
|
||||
let (mut s_c, mut s_a) = (Scores::new(SCORED.to_vec()), Scores::new(SCORED.to_vec()));
|
||||
let (mut pool_c, mut pool_a) = (Pool(Vec::new()), Pool(Vec::new()));
|
||||
let lvl = |m12: f64| ((m12.ln_1p() / 0.5) as usize).min(7);
|
||||
for t0 in start..=last_origin {
|
||||
let m = AddModel::fit(p, &r, h, t0);
|
||||
// Baselines from the same closed windows.
|
||||
let mut by_rot = [[1.0f64, 2.0]; 2];
|
||||
let mut by_lvl = [[1.0f64, 2.0]; 8];
|
||||
let mut any_rot = [[1.0f64, 2.0]; 2];
|
||||
let mut any_lvl = [[1.0f64, 2.0]; 8];
|
||||
let lvl = |m12: f64| ((m12.ln_1p() / 0.5) as usize).min(7);
|
||||
let (mut by_rot, mut by_lvl, mut any_rot, mut any_lvl) = ([[1.0f64, 2.0]; 2], [[1.0f64, 2.0]; 8], [[1.0f64, 2.0]; 2], [[1.0f64, 2.0]; 8]);
|
||||
let mut tm = Timing::new();
|
||||
for t in origins(h, t0) {
|
||||
for a in 0..p.a() {
|
||||
if !p.known(a, t) { continue; }
|
||||
let key = Timing::key(p, a, t, h, W);
|
||||
let mut tot = 0.0;
|
||||
for c in 0..C {
|
||||
let y = r.add_in_window(a, c, t, h) as i32 as f64;
|
||||
@@ -544,76 +709,59 @@ pub fn backtest(p: &Panel, tau: &[u32; C], horizons: &[usize], start: usize, md:
|
||||
by_rot[on][0] += y; by_rot[on][1] += 1.0;
|
||||
let m12 = (0..LAGS).map(|i| p.y(a, c, t - i)).sum::<f64>() / LAGS as f64;
|
||||
let b = lvl(m12); by_lvl[b][0] += y; by_lvl[b][1] += 1.0;
|
||||
tm.ch[c * 12 + key][0] += y; tm.ch[c * 12 + key][1] += 1.0;
|
||||
tot += m12;
|
||||
}
|
||||
let y = r.add_any_in_window(a, t, h) as i32 as f64;
|
||||
let on = r.any(a, t) as usize;
|
||||
any_rot[on][0] += y; any_rot[on][1] += 1.0;
|
||||
let b = lvl(tot); any_lvl[b][0] += y; any_lvl[b][1] += 1.0;
|
||||
tm.any[key][0] += y; tm.any[key][1] += 1.0;
|
||||
}
|
||||
}
|
||||
let (mut lc, mut la) = ([0.0; 4], [0.0; 4]);
|
||||
let gap = h + W + 1;
|
||||
let ((pl_c, iso_c), (pl_a, iso_a)) = (pool_c.fit(t0, gap), pool_a.fit(t0, gap));
|
||||
s_c.start_origin(); s_a.start_origin();
|
||||
for a in 0..p.a() {
|
||||
// Pre-release: the window's NMF placement isn't known yet; blend by the artist's NMF rate.
|
||||
if !p.known(a, t0) { continue; }
|
||||
let q = p_nmf(p, a, t0);
|
||||
let (pcn, pan) = m.predict(p, &r, a, t0, Scenario { nmf: Some(true), ..Scenario::ACTUAL }, None);
|
||||
let (pcr, par) = m.predict(p, &r, a, t0, Scenario { nmf: Some(false), ..Scenario::ACTUAL }, None);
|
||||
let pc: [f64; C] = std::array::from_fn(|c| q * pcn[c] + (1.0 - q) * pcr[c]);
|
||||
let pa = q * pan + (1.0 - q) * par;
|
||||
let nmf = |b: bool| Scenario { nmf: Some(b), ..Scenario::ACTUAL };
|
||||
let (pcn, pan) = m.predict(p, &r, a, t0, nmf(true), None);
|
||||
let (pcr, par) = m.predict(p, &r, a, t0, nmf(false), None);
|
||||
let (rcn, ran) = m.predict_raw(p, &r, a, t0, h, nmf(true));
|
||||
let (rcr, rar) = m.predict_raw(p, &r, a, t0, h, nmf(false));
|
||||
let (pcx, pax) = m.predict(p, &r, a, t0, Scenario::ACTUAL, None);
|
||||
let key = Timing::key(p, a, t0, h, W);
|
||||
let mut tot = 0.0;
|
||||
for c in 0..C {
|
||||
let y = r.add_in_window(a, c, t0, h);
|
||||
let on = r.on(a, c, t0) as usize;
|
||||
let m12 = (0..LAGS).map(|i| p.y(a, c, t0 - i)).sum::<f64>() / LAGS as f64;
|
||||
tot += m12;
|
||||
let preds = [by_rot[on][0] / by_rot[on][1], by_lvl[lvl(m12)][0] / by_lvl[lvl(m12)][1], pc[c], pcx[c]];
|
||||
for k in 0..4 { lc[k] += ll(preds[k], y); pairs_c[k].push((preds[k], y)); }
|
||||
let d = ((pc[c] * 10.0) as usize).min(9);
|
||||
calib[d][0] += pc[c]; calib[d][1] += y as i32 as f64; calib[d][2] += 1.0;
|
||||
nc += 1.0; hits_c += y as i32 as f64;
|
||||
let raw = q * rcn[c] + (1.0 - q) * rcr[c];
|
||||
let tmc = tm.ch[c * 12 + key];
|
||||
s_c.add(&[by_rot[on][0] / by_rot[on][1], by_lvl[lvl(m12)][0] / by_lvl[lvl(m12)][1], tmc[0] / tmc[1],
|
||||
q * pcn[c] + (1.0 - q) * pcr[c], raw, pl_c.apply(raw), iso_c.apply(raw), pcx[c]], y);
|
||||
pool_c.0.push((t0, raw, y));
|
||||
}
|
||||
let y = r.add_any_in_window(a, t0, h);
|
||||
let on = r.any(a, t0) as usize;
|
||||
let preds = [any_rot[on][0] / any_rot[on][1], any_lvl[lvl(tot)][0] / any_lvl[lvl(tot)][1], pa, pax];
|
||||
for k in 0..4 { la[k] += ll(preds[k], y); pairs_a[k].push((preds[k], y)); }
|
||||
na += 1.0; hits_a += y as i32 as f64;
|
||||
let raw = q * ran + (1.0 - q) * rar;
|
||||
let tma = tm.any[key];
|
||||
s_a.add(&[any_rot[on][0] / any_rot[on][1], any_lvl[lvl(tot)][0] / any_lvl[lvl(tot)][1], tma[0] / tma[1],
|
||||
q * pan + (1.0 - q) * par, raw, pl_a.apply(raw), iso_a.apply(raw), pax], y);
|
||||
pool_a.0.push((t0, raw, y));
|
||||
}
|
||||
wk_c.push(lc); wk_a.push(la);
|
||||
eprintln!("adds backtest h={h}: origin {} done", p.weeks[t0]);
|
||||
}
|
||||
let nwk = wk_c.len();
|
||||
writeln!(md, "## Window starting h = {h} weeks ahead ({nwk} forecast origins)\n")?;
|
||||
writeln!(md, "Base rates: per channel {:.2}% of artist-channels get an add in the window; on any channel {:.1}% of artists.\n", 100.0 * hits_c / nc, 100.0 * hits_a / na)?;
|
||||
writeln!(md, "| Model | Log-loss, per channel | PR-AUC, per channel | Log-loss, any channel | PR-AUC, any channel |\n|---|---|---|---|---|")?;
|
||||
for k in 0..4 {
|
||||
let (sc, sa) = (wk_c.iter().map(|w| w[k]).sum::<f64>() / nc, wk_a.iter().map(|w| w[k]).sum::<f64>() / na);
|
||||
let (ac, aa) = (average_precision(pairs_c[k].clone()), average_precision(pairs_a[k].clone()));
|
||||
for (metric, v) in [("add_logloss", sc), ("add_prauc", ac), ("addany_logloss", sa), ("addany_prauc", aa)] {
|
||||
rows.push((names[k].to_string(), h, metric.to_string(), v));
|
||||
}
|
||||
writeln!(md, "| {} | {sc:.4} | {ac:.3} | {sa:.4} | {aa:.3} |", names[k])?;
|
||||
}
|
||||
let col = |v: &Vec<[f64; 4]>, k: usize| v.iter().map(|w| w[k]).collect::<Vec<f64>>();
|
||||
let (s1, s2) = (skill(&col(&wk_c, 2), &col(&wk_c, 0)), skill(&col(&wk_c, 2), &col(&wk_c, 1)));
|
||||
let (s3, s4) = (skill(&col(&wk_a, 2), &col(&wk_a, 0)), skill(&col(&wk_a, 2), &col(&wk_a, 1)));
|
||||
rows.push(("logit".into(), h, "add_skill_vs_level".into(), s2.0));
|
||||
rows.push(("logit".into(), h, "addany_skill_vs_level".into(), s4.0));
|
||||
writeln!(md, "\nSkill of the logit (90% block-bootstrap interval over origins):\n")?;
|
||||
writeln!(md, "| | vs in-rotation-now | vs last-12-weeks level |\n|---|---|---|")?;
|
||||
writeln!(md, "| per channel | {:.3} ({:.3} to {:.3}) | {:.3} ({:.3} to {:.3}) |", s1.0, s1.1, s1.2, s2.0, s2.1, s2.2)?;
|
||||
writeln!(md, "| any channel | {:.3} ({:.3} to {:.3}) | {:.3} ({:.3} to {:.3}) |\n", s3.0, s3.1, s3.2, s4.0, s4.1, s4.2)?;
|
||||
writeln!(md, "Calibration, per channel:\n\n| Predicted | Rows | Mean predicted | Observed |\n|---|---|---|---|")?;
|
||||
for (d, c) in calib.iter().enumerate() {
|
||||
if c[2] > 0.0 { writeln!(md, "| {:.1}–{:.1} | {} | {:.3} | {:.3} |", d as f64 / 10.0, (d + 1) as f64 / 10.0, c[2], c[0] / c[2], c[1] / c[2])?; }
|
||||
}
|
||||
writeln!(md)?;
|
||||
if nwk < 8 { writeln!(md, "Fewer than 8 forecast origins: no verdict at this horizon (doc rule).\n")?; }
|
||||
writeln!(md, "## Window starting h = {h} weeks ahead ({} forecast origins)\n", s_c.wk.len())?;
|
||||
writeln!(md, "Base rates: per channel {:.2}% of artist-channels get an add in the window; on any channel {:.1}% of artists.\n", 100.0 * s_c.hits / s_c.n.max(1.0), 100.0 * s_a.hits / s_a.n.max(1.0))?;
|
||||
table(md, &s_c, &s_a, &mut rows, h, "add_")?;
|
||||
if s_c.wk.len() < 8 { writeln!(md, "Fewer than 8 forecast origins: no verdict at this horizon (doc rule).\n")?; }
|
||||
}
|
||||
let m = AddModel::fit(p, &r, 0, p.w() - 1);
|
||||
writeln!(md, "## Coefficients, window starting now (h = 0), all closed windows\n\n| Any-channel feature | Coefficient |\n|---|---|")?;
|
||||
for (n, b) in ANY_NAMES.iter().zip(&m.any) { writeln!(md, "| {n} | {b:+.3} |")?; }
|
||||
for (n, b) in REL_NAMES.iter().chain(WAVE_NAMES).chain(LONG_NAMES).chain(NMF_NAMES).chain(SEASON_NAMES).zip(&m.any[21..]) { writeln!(md, "| {n} | {b:+.3} |")?; }
|
||||
for (n, b) in ANY_NAMES.iter().chain(REL_NAMES).chain(WAVE_NAMES).chain(LONG_NAMES).chain(NMF_NAMES).chain(SEASON_NAMES).zip(&m.any) { writeln!(md, "| {n} | {b:+.3} |")?; }
|
||||
writeln!(md, "\n| Per-channel feature | Coefficient |\n|---|---|")?;
|
||||
for (n, b) in features::NAMES.iter().chain(EXTRA).chain(REL_NAMES).chain(WAVE_NAMES).chain(LONG_NAMES).chain(NMF_NAMES).chain(SEASON_NAMES).chain(AFF_NAMES).zip(&m.channel) { writeln!(md, "| {n} | {b:+.3} |")?; }
|
||||
writeln!(md)?;
|
||||
@@ -621,83 +769,76 @@ pub fn backtest(p: &Panel, tau: &[u32; C], horizons: &[usize], start: usize, md:
|
||||
}
|
||||
|
||||
/// Walk-forward test of the release-event model: at each origin t0 the model sees only events
|
||||
/// whose 8 weeks had closed, and scores the releases of week t0+1. The new song's NMF placement
|
||||
/// isn't known in advance, so it's blended from the artist's history (as on the site).
|
||||
/// whose window had closed (a week before t0), and scores the releases of week t0+1. Pre-release
|
||||
/// rows blend the new song's NMF placement from the artist's history (as on the site).
|
||||
pub fn backtest_events(p: &Panel, tau: &[u32; C], start: usize, md: &mut String) -> Result<Vec<(String, usize, String, f64)>> {
|
||||
let r = Rot::new(p, tau);
|
||||
let mut rows = Vec::new();
|
||||
let names = ["rate by in-rotation-now", "rate by last-12-weeks level", "event model (pre-release: NMF from history)", "event model (post-release: NMF known)"];
|
||||
let mut wk_any: Vec<[f64; 4]> = Vec::new();
|
||||
let mut wk_ch: Vec<[f64; 4]> = Vec::new();
|
||||
let mut pairs_any: [Vec<(f64, bool)>; 4] = Default::default();
|
||||
let mut pairs_ch: [Vec<(f64, bool)>; 4] = Default::default();
|
||||
let (mut n_ev, mut n_hit) = (0usize, 0usize);
|
||||
let (mut s_c, mut s_a) = (Scores::new(SCORED.to_vec()), Scores::new(SCORED.to_vec()));
|
||||
let (mut pool_c, mut pool_a) = (Pool(Vec::new()), Pool(Vec::new()));
|
||||
let lvl = |m: f64| ((m.ln_1p() / 0.5) as usize).min(7);
|
||||
let level = |a: usize, t: usize| (0..LAGS).map(|i| p.group_total(a, t - i, None)).sum::<f64>() / LAGS as f64;
|
||||
for t0 in start..p.w().saturating_sub(W) {
|
||||
if t0 + 1 >= p.w() { break; }
|
||||
for t0 in start..p.w().saturating_sub(W + 1) {
|
||||
let m = AddModel::fit_events(p, &r, t0);
|
||||
// baselines from the training events
|
||||
let (mut by_rot, mut by_lvl) = ([[1.0f64, 2.0]; 2], [[1.0f64, 2.0]; 8]);
|
||||
let (mut cr, mut cl) = ([[1.0f64, 2.0]; 2], [[1.0f64, 2.0]; 8]);
|
||||
// Baselines from the training events. Release timing only: by weeks since the previous release.
|
||||
let (mut by_rot, mut by_lvl, mut cr, mut cl) = ([[1.0f64, 2.0]; 2], [[1.0f64, 2.0]; 8], [[1.0f64, 2.0]; 2], [[1.0f64, 2.0]; 8]);
|
||||
let mut tm = Timing::new();
|
||||
for t in origins(0, t0) {
|
||||
for a in 0..p.a() {
|
||||
if !clean_release(p, a, t + 1) { continue; }
|
||||
if !p.known(a, t) || !clean_release(p, a, t + 1) { continue; }
|
||||
let key = since_bin(p, a, t);
|
||||
let y = r.add_any_in_window(a, t, 0) as i32 as f64;
|
||||
let on = r.any(a, t) as usize;
|
||||
by_rot[on][0] += y; by_rot[on][1] += 1.0;
|
||||
let b = lvl(level(a, t)); by_lvl[b][0] += y; by_lvl[b][1] += 1.0;
|
||||
tm.any[key][0] += y; tm.any[key][1] += 1.0;
|
||||
for c in 0..C {
|
||||
let yc = r.add_in_window(a, c, t, 0) as i32 as f64;
|
||||
let onc = r.on(a, c, t) as usize;
|
||||
cr[onc][0] += yc; cr[onc][1] += 1.0;
|
||||
let bc = lvl((0..LAGS).map(|i| p.y(a, c, t - i)).sum::<f64>() / LAGS as f64);
|
||||
cl[bc][0] += yc; cl[bc][1] += 1.0;
|
||||
tm.ch[c * 12 + key][0] += yc; tm.ch[c * 12 + key][1] += 1.0;
|
||||
}
|
||||
}
|
||||
}
|
||||
let (mut la, mut lc) = ([0.0; 4], [0.0; 4]);
|
||||
let mut any_events = false;
|
||||
let gap = W + 1;
|
||||
let ((pl_c, iso_c), (pl_a, iso_a)) = (pool_c.fit(t0, gap), pool_a.fit(t0, gap));
|
||||
let mut started = false;
|
||||
for a in 0..p.a() {
|
||||
if !clean_release(p, a, t0 + 1) { continue; }
|
||||
any_events = true;
|
||||
if !p.known(a, t0) || !clean_release(p, a, t0 + 1) { continue; }
|
||||
if !started { s_c.start_origin(); s_a.start_origin(); started = true; }
|
||||
let q = p_nmf(p, a, t0);
|
||||
let (pcn, pan) = m.predict(p, &r, a, t0, Scenario::new(true, true), None);
|
||||
let (pcr, par) = m.predict(p, &r, a, t0, Scenario::new(true, false), None);
|
||||
let pa = q * pan + (1.0 - q) * par;
|
||||
let (rcn, ran) = m.predict_raw(p, &r, a, t0, 0, Scenario::new(true, true));
|
||||
let (rcr, rar) = m.predict_raw(p, &r, a, t0, 0, Scenario::new(true, false));
|
||||
let (pcx, pax) = m.predict(p, &r, a, t0, Scenario::ACTUAL, None);
|
||||
let key = since_bin(p, a, t0);
|
||||
let y = r.add_any_in_window(a, t0, 0);
|
||||
let on = r.any(a, t0) as usize;
|
||||
let preds = [by_rot[on][0] / by_rot[on][1], by_lvl[lvl(level(a, t0))][0] / by_lvl[lvl(level(a, t0))][1], pa, pax];
|
||||
for k in 0..4 { la[k] += ll(preds[k], y); pairs_any[k].push((preds[k], y)); }
|
||||
n_ev += 1; n_hit += y as usize;
|
||||
let raw = q * ran + (1.0 - q) * rar;
|
||||
let tma = tm.any[key];
|
||||
s_a.add(&[by_rot[on][0] / by_rot[on][1], by_lvl[lvl(level(a, t0))][0] / by_lvl[lvl(level(a, t0))][1], tma[0] / tma[1],
|
||||
q * pan + (1.0 - q) * par, raw, pl_a.apply(raw), iso_a.apply(raw), pax], y);
|
||||
pool_a.0.push((t0, raw, y));
|
||||
for c in 0..C {
|
||||
let pc = q * pcn[c] + (1.0 - q) * pcr[c];
|
||||
let yc = r.add_in_window(a, c, t0, 0);
|
||||
let onc = r.on(a, c, t0) as usize;
|
||||
let bc = lvl((0..LAGS).map(|i| p.y(a, c, t0 - i)).sum::<f64>() / LAGS as f64);
|
||||
let pr = [cr[onc][0] / cr[onc][1], cl[bc][0] / cl[bc][1], pc, pcx[c]];
|
||||
for k in 0..4 { lc[k] += ll(pr[k], yc); pairs_ch[k].push((pr[k], yc)); }
|
||||
let rawc = q * rcn[c] + (1.0 - q) * rcr[c];
|
||||
let tmc = tm.ch[c * 12 + key];
|
||||
s_c.add(&[cr[onc][0] / cr[onc][1], cl[bc][0] / cl[bc][1], tmc[0] / tmc[1],
|
||||
q * pcn[c] + (1.0 - q) * pcr[c], rawc, pl_c.apply(rawc), iso_c.apply(rawc), pcx[c]], yc);
|
||||
pool_c.0.push((t0, rawc, yc));
|
||||
}
|
||||
}
|
||||
if any_events { wk_any.push(la); wk_ch.push(lc); }
|
||||
eprintln!("event backtest: origin {} done", p.weeks[t0]);
|
||||
}
|
||||
writeln!(md, "# Backtest: if the artist releases a new song, will DR add it?\n")?;
|
||||
writeln!(md, "Events: an artist in the panel releases something (Deezer) in week t0+1, with no other release in the {CLEAN_WEEKS} weeks before (so an add can't be the previous song's); target: a DR add (entering rotation after {GAP}+ weeks out, or plays jumping well above the artist's own 8-week baseline and staying up) within the {W} weeks from the release week. The model knows the artist's state the week before; the new song's NMF Denmark placement is blended from the artist's history. {n_ev} release events scored, {n_hit} ({:.1}%) got an add on some DR channel.\n",
|
||||
100.0 * n_hit as f64 / n_ev.max(1) as f64)?;
|
||||
writeln!(md, "| Model | Log-loss, any channel | PR-AUC, any channel | Log-loss, per channel | PR-AUC, per channel |\n|---|---|---|---|---|")?;
|
||||
let nch = (n_ev * C).max(1) as f64;
|
||||
for k in 0..4 {
|
||||
let (a, c) = (wk_any.iter().map(|w| w[k]).sum::<f64>() / n_ev.max(1) as f64, wk_ch.iter().map(|w| w[k]).sum::<f64>() / nch);
|
||||
let (pa, pc) = (average_precision(pairs_any[k].clone()), average_precision(pairs_ch[k].clone()));
|
||||
writeln!(md, "| {} | {a:.4} | {pa:.3} | {c:.4} | {pc:.3} |", names[k])?;
|
||||
for (metric, v) in [("event_logloss_any", a), ("event_prauc_any", pa), ("event_logloss_ch", c), ("event_prauc_ch", pc)] { rows.push((names[k].to_string(), 0, metric.to_string(), v)); }
|
||||
}
|
||||
let col = |v: &Vec<[f64; 4]>, k: usize| v.iter().map(|w| w[k]).collect::<Vec<f64>>();
|
||||
let (s1, s2) = (skill(&col(&wk_any, 2), &col(&wk_any, 0)), skill(&col(&wk_any, 2), &col(&wk_any, 1)));
|
||||
rows.push(("event model".into(), 0, "event_skill_any_vs_level".into(), s2.0));
|
||||
writeln!(md, "\nSkill of the event model, any channel: {:.3} ({:.3} to {:.3}) vs in-rotation-now, {:.3} ({:.3} to {:.3}) vs last-12-weeks level.\n", s1.0, s1.1, s1.2, s2.0, s2.1, s2.2)?;
|
||||
writeln!(md, "Events: an artist knowable at the time releases something (Deezer) in week t0+1, with no other release in the {CLEAN_WEEKS} weeks before (so an add can't be the previous song's); target: a DR add (entering rotation after {GAP}+ weeks out, or plays jumping well above the artist's own 8-week baseline and staying up) within the {W} weeks from the release week. The model knows the artist's state the week before; pre-release rows blend the new song's NMF Denmark placement from the artist's history. \"Release timing only\" = the add rate by weeks since the artist's previous release. {} release events scored, {} ({:.1}%) got an add on some DR channel.\n",
|
||||
s_a.n, s_a.hits, 100.0 * s_a.hits / s_a.n.max(1.0))?;
|
||||
table(md, &s_c, &s_a, &mut rows, 0, "event_")?;
|
||||
let m = AddModel::fit_events(p, &r, p.w() - 1);
|
||||
writeln!(md, "| Any-channel feature (all events) | Coefficient |\n|---|---|")?;
|
||||
for (n, b) in ANY_NAMES.iter().chain(REL_NAMES).chain(WAVE_NAMES).chain(LONG_NAMES).chain(NMF_NAMES).chain(SEASON_NAMES).zip(&m.any) { writeln!(md, "| {n} | {b:+.3} |")?; }
|
||||
|
||||
Reference in New Issue
Block a user