Old DR artist pages from the Wayback Machine (53 weeks of plays per capture, 2019-2024) into dr_artist_history; leak fixes (weeks since last add counts only adds up to t-1; training windows close a week before the origin) and a leak test (model leakcheck) that rebuilds every feature from data cut at the origin; backtests split pre-release (NMF from history) and post-release (NMF known); site shows the new-song backtest
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+39
-28
@@ -42,7 +42,9 @@ pub const WAVE_NAMES: &[&str] = &["DR: weeks since last add 13-25", "DR: last ad
|
||||
/// and the IFPI wave position.
|
||||
pub fn wave_feats(p: &Panel, r: &Rot, a: usize, t: usize, shift: usize) -> [f64; WAVE] {
|
||||
if off("wave") { return [0.0; WAVE]; }
|
||||
let since = (0..=t).find(|&i| r.add_any(a, t - i)).map(|s| s + shift);
|
||||
// An add in week w is only confirmed in week w + 1 (the surge rule reads the following week),
|
||||
// so as of week t only adds up to t - 1 are known.
|
||||
let since = (1..=t).find(|&i| r.add_any(a, t - i)).map(|s| s + shift);
|
||||
let ch = p.chart(a, t);
|
||||
let f = |b: bool| b as i32 as f64;
|
||||
let air = ch.air_since as f64 + shift as f64;
|
||||
@@ -374,7 +376,9 @@ fn platt(ab: (f64, f64), p: f64) -> f64 { glm::sigmoid(ab.0 + ab.1 * logit(p)).c
|
||||
|
||||
fn origins(h: usize, last_week: usize) -> std::ops::RangeInclusive<usize> { origins_w(h, last_week, W) }
|
||||
fn origins_w(h: usize, last_week: usize, win: usize) -> std::ops::RangeInclusive<usize> {
|
||||
let hi = last_week.saturating_sub(h + win);
|
||||
// The label's last week needs the following week too (surge rule): the window must close at
|
||||
// least one week before `last_week`.
|
||||
let hi = last_week.saturating_sub(h + win + 1);
|
||||
LAGS - 1..=hi
|
||||
}
|
||||
|
||||
@@ -394,7 +398,7 @@ impl AddModel {
|
||||
|
||||
fn fit_with(p: &Panel, r: &Rot, h: usize, last_week: usize, events: bool, win: usize) -> AddModel {
|
||||
let mut m = Self::fit_raw(p, r, h, last_week, events, win);
|
||||
let hi = last_week.saturating_sub(h + win);
|
||||
let hi = last_week.saturating_sub(h + win + 1);
|
||||
if hi + 1 >= CAL_ORIGINS + LAGS {
|
||||
let c0 = hi + 1 - CAL_ORIGINS;
|
||||
// the inner model may only use windows that closed before the first calibration origin
|
||||
@@ -516,11 +520,11 @@ pub fn backtest(p: &Panel, tau: &[u32; C], horizons: &[usize], start: usize, md:
|
||||
let last_origin = p.w() - 1 - h - W;
|
||||
if last_origin < start { continue; }
|
||||
// [model][week] log-loss sums; models: base(rot now), base(popularity bins), logit
|
||||
let names = ["rate by in-rotation-now", "rate by last-12-weeks level", "logit"];
|
||||
let mut wk_c: Vec<[f64; 3]> = Vec::new();
|
||||
let mut wk_a: Vec<[f64; 3]> = Vec::new();
|
||||
let mut pairs_c: [Vec<(f64, bool)>; 3] = Default::default();
|
||||
let mut pairs_a: [Vec<(f64, bool)>; 3] = Default::default();
|
||||
let names = ["rate by in-rotation-now", "rate by last-12-weeks level", "logit (pre-release: NMF from history)", "logit (post-release: NMF known)"];
|
||||
let mut wk_c: Vec<[f64; 4]> = Vec::new();
|
||||
let mut wk_a: Vec<[f64; 4]> = Vec::new();
|
||||
let mut pairs_c: [Vec<(f64, bool)>; 4] = Default::default();
|
||||
let mut pairs_a: [Vec<(f64, bool)>; 4] = Default::default();
|
||||
let mut calib = [[0.0f64; 3]; 10];
|
||||
let (mut nc, mut na, mut hits_c, mut hits_a) = (0.0, 0.0, 0.0, 0.0);
|
||||
for t0 in start..=last_origin {
|
||||
@@ -548,25 +552,31 @@ pub fn backtest(p: &Panel, tau: &[u32; C], horizons: &[usize], start: usize, md:
|
||||
let b = lvl(tot); any_lvl[b][0] += y; any_lvl[b][1] += 1.0;
|
||||
}
|
||||
}
|
||||
let (mut lc, mut la) = ([0.0; 3], [0.0; 3]);
|
||||
let (mut lc, mut la) = ([0.0; 4], [0.0; 4]);
|
||||
for a in 0..p.a() {
|
||||
let (pc, pa) = m.predict(p, &r, a, t0, Scenario::ACTUAL, None);
|
||||
// Pre-release: the window's NMF placement isn't known yet; blend by the artist's NMF rate.
|
||||
let q = p_nmf(p, a, t0);
|
||||
let (pcn, pan) = m.predict(p, &r, a, t0, Scenario { nmf: Some(true), ..Scenario::ACTUAL }, None);
|
||||
let (pcr, par) = m.predict(p, &r, a, t0, Scenario { nmf: Some(false), ..Scenario::ACTUAL }, None);
|
||||
let pc: [f64; C] = std::array::from_fn(|c| q * pcn[c] + (1.0 - q) * pcr[c]);
|
||||
let pa = q * pan + (1.0 - q) * par;
|
||||
let (pcx, pax) = m.predict(p, &r, a, t0, Scenario::ACTUAL, None);
|
||||
let mut tot = 0.0;
|
||||
for c in 0..C {
|
||||
let y = r.add_in_window(a, c, t0, h);
|
||||
let on = r.on(a, c, t0) as usize;
|
||||
let m12 = (0..LAGS).map(|i| p.y(a, c, t0 - i)).sum::<f64>() / LAGS as f64;
|
||||
tot += m12;
|
||||
let preds = [by_rot[on][0] / by_rot[on][1], by_lvl[lvl(m12)][0] / by_lvl[lvl(m12)][1], pc[c]];
|
||||
for k in 0..3 { lc[k] += ll(preds[k], y); pairs_c[k].push((preds[k], y)); }
|
||||
let preds = [by_rot[on][0] / by_rot[on][1], by_lvl[lvl(m12)][0] / by_lvl[lvl(m12)][1], pc[c], pcx[c]];
|
||||
for k in 0..4 { lc[k] += ll(preds[k], y); pairs_c[k].push((preds[k], y)); }
|
||||
let d = ((pc[c] * 10.0) as usize).min(9);
|
||||
calib[d][0] += pc[c]; calib[d][1] += y as i32 as f64; calib[d][2] += 1.0;
|
||||
nc += 1.0; hits_c += y as i32 as f64;
|
||||
}
|
||||
let y = r.add_any_in_window(a, t0, h);
|
||||
let on = r.any(a, t0) as usize;
|
||||
let preds = [any_rot[on][0] / any_rot[on][1], any_lvl[lvl(tot)][0] / any_lvl[lvl(tot)][1], pa];
|
||||
for k in 0..3 { la[k] += ll(preds[k], y); pairs_a[k].push((preds[k], y)); }
|
||||
let preds = [any_rot[on][0] / any_rot[on][1], any_lvl[lvl(tot)][0] / any_lvl[lvl(tot)][1], pa, pax];
|
||||
for k in 0..4 { la[k] += ll(preds[k], y); pairs_a[k].push((preds[k], y)); }
|
||||
na += 1.0; hits_a += y as i32 as f64;
|
||||
}
|
||||
wk_c.push(lc); wk_a.push(la);
|
||||
@@ -576,7 +586,7 @@ pub fn backtest(p: &Panel, tau: &[u32; C], horizons: &[usize], start: usize, md:
|
||||
writeln!(md, "## Window starting h = {h} weeks ahead ({nwk} forecast origins)\n")?;
|
||||
writeln!(md, "Base rates: per channel {:.2}% of artist-channels get an add in the window; on any channel {:.1}% of artists.\n", 100.0 * hits_c / nc, 100.0 * hits_a / na)?;
|
||||
writeln!(md, "| Model | Log-loss, per channel | PR-AUC, per channel | Log-loss, any channel | PR-AUC, any channel |\n|---|---|---|---|---|")?;
|
||||
for k in 0..3 {
|
||||
for k in 0..4 {
|
||||
let (sc, sa) = (wk_c.iter().map(|w| w[k]).sum::<f64>() / nc, wk_a.iter().map(|w| w[k]).sum::<f64>() / na);
|
||||
let (ac, aa) = (average_precision(pairs_c[k].clone()), average_precision(pairs_a[k].clone()));
|
||||
for (metric, v) in [("add_logloss", sc), ("add_prauc", ac), ("addany_logloss", sa), ("addany_prauc", aa)] {
|
||||
@@ -584,7 +594,7 @@ pub fn backtest(p: &Panel, tau: &[u32; C], horizons: &[usize], start: usize, md:
|
||||
}
|
||||
writeln!(md, "| {} | {sc:.4} | {ac:.3} | {sa:.4} | {aa:.3} |", names[k])?;
|
||||
}
|
||||
let col = |v: &Vec<[f64; 3]>, k: usize| v.iter().map(|w| w[k]).collect::<Vec<f64>>();
|
||||
let col = |v: &Vec<[f64; 4]>, k: usize| v.iter().map(|w| w[k]).collect::<Vec<f64>>();
|
||||
let (s1, s2) = (skill(&col(&wk_c, 2), &col(&wk_c, 0)), skill(&col(&wk_c, 2), &col(&wk_c, 1)));
|
||||
let (s3, s4) = (skill(&col(&wk_a, 2), &col(&wk_a, 0)), skill(&col(&wk_a, 2), &col(&wk_a, 1)));
|
||||
rows.push(("logit".into(), h, "add_skill_vs_level".into(), s2.0));
|
||||
@@ -616,11 +626,11 @@ pub fn backtest(p: &Panel, tau: &[u32; C], horizons: &[usize], start: usize, md:
|
||||
pub fn backtest_events(p: &Panel, tau: &[u32; C], start: usize, md: &mut String) -> Result<Vec<(String, usize, String, f64)>> {
|
||||
let r = Rot::new(p, tau);
|
||||
let mut rows = Vec::new();
|
||||
let names = ["rate by in-rotation-now", "rate by last-12-weeks level", "event model"];
|
||||
let mut wk_any: Vec<[f64; 3]> = Vec::new();
|
||||
let mut wk_ch: Vec<[f64; 3]> = Vec::new();
|
||||
let mut pairs_any: [Vec<(f64, bool)>; 3] = Default::default();
|
||||
let mut pairs_ch: [Vec<(f64, bool)>; 3] = Default::default();
|
||||
let names = ["rate by in-rotation-now", "rate by last-12-weeks level", "event model (pre-release: NMF from history)", "event model (post-release: NMF known)"];
|
||||
let mut wk_any: Vec<[f64; 4]> = Vec::new();
|
||||
let mut wk_ch: Vec<[f64; 4]> = Vec::new();
|
||||
let mut pairs_any: [Vec<(f64, bool)>; 4] = Default::default();
|
||||
let mut pairs_ch: [Vec<(f64, bool)>; 4] = Default::default();
|
||||
let (mut n_ev, mut n_hit) = (0usize, 0usize);
|
||||
let lvl = |m: f64| ((m.ln_1p() / 0.5) as usize).min(7);
|
||||
let level = |a: usize, t: usize| (0..LAGS).map(|i| p.group_total(a, t - i, None)).sum::<f64>() / LAGS as f64;
|
||||
@@ -646,7 +656,7 @@ pub fn backtest_events(p: &Panel, tau: &[u32; C], start: usize, md: &mut String)
|
||||
}
|
||||
}
|
||||
}
|
||||
let (mut la, mut lc) = ([0.0; 3], [0.0; 3]);
|
||||
let (mut la, mut lc) = ([0.0; 4], [0.0; 4]);
|
||||
let mut any_events = false;
|
||||
for a in 0..p.a() {
|
||||
if !clean_release(p, a, t0 + 1) { continue; }
|
||||
@@ -655,18 +665,19 @@ pub fn backtest_events(p: &Panel, tau: &[u32; C], start: usize, md: &mut String)
|
||||
let (pcn, pan) = m.predict(p, &r, a, t0, Scenario::new(true, true), None);
|
||||
let (pcr, par) = m.predict(p, &r, a, t0, Scenario::new(true, false), None);
|
||||
let pa = q * pan + (1.0 - q) * par;
|
||||
let (pcx, pax) = m.predict(p, &r, a, t0, Scenario::ACTUAL, None);
|
||||
let y = r.add_any_in_window(a, t0, 0);
|
||||
let on = r.any(a, t0) as usize;
|
||||
let preds = [by_rot[on][0] / by_rot[on][1], by_lvl[lvl(level(a, t0))][0] / by_lvl[lvl(level(a, t0))][1], pa];
|
||||
for k in 0..3 { la[k] += ll(preds[k], y); pairs_any[k].push((preds[k], y)); }
|
||||
let preds = [by_rot[on][0] / by_rot[on][1], by_lvl[lvl(level(a, t0))][0] / by_lvl[lvl(level(a, t0))][1], pa, pax];
|
||||
for k in 0..4 { la[k] += ll(preds[k], y); pairs_any[k].push((preds[k], y)); }
|
||||
n_ev += 1; n_hit += y as usize;
|
||||
for c in 0..C {
|
||||
let pc = q * pcn[c] + (1.0 - q) * pcr[c];
|
||||
let yc = r.add_in_window(a, c, t0, 0);
|
||||
let onc = r.on(a, c, t0) as usize;
|
||||
let bc = lvl((0..LAGS).map(|i| p.y(a, c, t0 - i)).sum::<f64>() / LAGS as f64);
|
||||
let pr = [cr[onc][0] / cr[onc][1], cl[bc][0] / cl[bc][1], pc];
|
||||
for k in 0..3 { lc[k] += ll(pr[k], yc); pairs_ch[k].push((pr[k], yc)); }
|
||||
let pr = [cr[onc][0] / cr[onc][1], cl[bc][0] / cl[bc][1], pc, pcx[c]];
|
||||
for k in 0..4 { lc[k] += ll(pr[k], yc); pairs_ch[k].push((pr[k], yc)); }
|
||||
}
|
||||
}
|
||||
if any_events { wk_any.push(la); wk_ch.push(lc); }
|
||||
@@ -677,13 +688,13 @@ pub fn backtest_events(p: &Panel, tau: &[u32; C], start: usize, md: &mut String)
|
||||
100.0 * n_hit as f64 / n_ev.max(1) as f64)?;
|
||||
writeln!(md, "| Model | Log-loss, any channel | PR-AUC, any channel | Log-loss, per channel | PR-AUC, per channel |\n|---|---|---|---|---|")?;
|
||||
let nch = (n_ev * C).max(1) as f64;
|
||||
for k in 0..3 {
|
||||
for k in 0..4 {
|
||||
let (a, c) = (wk_any.iter().map(|w| w[k]).sum::<f64>() / n_ev.max(1) as f64, wk_ch.iter().map(|w| w[k]).sum::<f64>() / nch);
|
||||
let (pa, pc) = (average_precision(pairs_any[k].clone()), average_precision(pairs_ch[k].clone()));
|
||||
writeln!(md, "| {} | {a:.4} | {pa:.3} | {c:.4} | {pc:.3} |", names[k])?;
|
||||
for (metric, v) in [("event_logloss_any", a), ("event_prauc_any", pa), ("event_logloss_ch", c), ("event_prauc_ch", pc)] { rows.push((names[k].to_string(), 0, metric.to_string(), v)); }
|
||||
}
|
||||
let col = |v: &Vec<[f64; 3]>, k: usize| v.iter().map(|w| w[k]).collect::<Vec<f64>>();
|
||||
let col = |v: &Vec<[f64; 4]>, k: usize| v.iter().map(|w| w[k]).collect::<Vec<f64>>();
|
||||
let (s1, s2) = (skill(&col(&wk_any, 2), &col(&wk_any, 0)), skill(&col(&wk_any, 2), &col(&wk_any, 1)));
|
||||
rows.push(("event model".into(), 0, "event_skill_any_vs_level".into(), s2.0));
|
||||
writeln!(md, "\nSkill of the event model, any channel: {:.3} ({:.3} to {:.3}) vs in-rotation-now, {:.3} ({:.3} to {:.3}) vs last-12-weeks level.\n", s1.0, s1.1, s1.2, s2.0, s2.1, s2.2)?;
|
||||
|
||||
Reference in New Issue
Block a user