diff --git a/docs/HISTORY.md b/docs/HISTORY.md index 9dc90d6..07532e9 100644 --- a/docs/HISTORY.md +++ b/docs/HISTORY.md @@ -62,3 +62,21 @@ memory down. Switching it on for the site was reverted after the service hit its Also found: per-channel probabilities above ~0.2 are overconfident (40-50% predicted, ~8% observed), mostly P4 regions for artists already in rotation (Mumle: 44% on P4 vs 6% any channel). A floor making "any" >= the best station group made "any" worse (0.0664 -> 0.0690) until those are recalibrated. + +## Calibration, competition and slots (2026-11-20), history on + +- **Site**: history on (`AIRPLAY_HISTORY=1` for airplay-web and airplay-collect); fitting peaks at ~2.5 GB, + so both services have `MemoryMax=3584M` (drop-in history.conf). +- **Per-channel calibration by station group x in rotation (isotonic)**: no gain (release events per-channel + log loss 0.0150 vs 0.0148 pooled Platt); the 50-60% bin still came true 0 of 16 times. P4's regions repeat + the same few events, so the tail cannot be learned. +- **Cap each channel at the any-channel chance**: release events per-channel PR-AUC 0.105 -> 0.165, log loss + 0.0148 -> 0.0143, "any" unchanged; general model neutral to slightly worse (PR-AUC 0.285 -> 0.267). + Applied to the release-event model only (the site's "if they release" numbers: Mumle 44% on P4 vs 6% any). +- **Competition scaling** (each week, scale a channel's chances to its usual number of adds): worse everywhere + (any 0.0736 -> 0.0760). Over 8-week windows actual adds rise with predicted supply (1.4-1.7 per predicted + add, correlation 0.72-0.82 at h=4): no fixed capacity to share. Rejected. +- **Slots per channel** (Poisson deviance on 26 weeks May-Nov 2026, five main channels): flat recent average + 232, week-of-year average 833, recent level x shrunk calendar blocks 217. Blocks: summer holiday (weeks 28-31: + P3 0.66, P4 0.77, P5 0.83), New Year (weeks 1-2), December (not trustworthy until song-level data separates + Christmas music). Not yet in the model. diff --git a/src/model/adds.rs b/src/model/adds.rs index ccc3fc3..69ebcbf 100644 --- a/src/model/adds.rs +++ b/src/model/adds.rs @@ -640,7 +640,15 @@ impl AddModel { // is an add on some channel. Off: the per-channel numbers are overconfident above ~0.2, and // the floor made "any" worse (log loss 0.0664 -> 0.0690) until they are recalibrated. let floor = if std::env::var("AIRPLAY_ANY_FLOOR").is_ok_and(|v| v == "1") { best.iter().copied().fold(0.0, f64::max) } else { 0.0 }; - (pc, pa.max(floor).min(cap.max(1e-6))) + let pa = pa.max(floor).min(cap.max(1e-6)); + // The release-event model (the only one with pooled calibration): each channel at most the + // any-channel chance. Its per-channel odds for artists already in rotation were overconfident + // (40-60% predicted, ~0-10% observed: P4's regions repeat the same few events); capped, the + // per-channel PR-AUC went 0.105 -> 0.165 with "any" unchanged. AIRPLAY_CAP_CH=0 turns it off. + if self.calib_any.is_some() && std::env::var("AIRPLAY_CAP_CH").map_or(true, |v| v != "0") { + for v in pc.iter_mut() { *v = v.min(pa); } + } + (pc, pa) } } @@ -755,9 +763,26 @@ impl Pool { } /// The models scored in both backtests (index: meaning). -const SCORED: [&str; 8] = ["rate by in-rotation-now", "rate by last-12-weeks level", "release timing only", +const SCORED: [&str; 10] = ["rate by in-rotation-now", "rate by last-12-weeks level", "release timing only", "model, calibrated on the last 6 origins (pre-release)", "model, raw (pre-release)", - "model + pooled Platt (pre-release)", "model + pooled isotonic (pre-release)", "model, NMF known (post-release)"]; + "model + pooled Platt (pre-release)", "model + pooled isotonic (pre-release)", "model, NMF known (post-release)", + "model + isotonic by station group x in rotation; any >= best channel (pre-release)", + "the same + competition scaling to the channel's usual adds (pre-release)"]; + +/// Calibration strata: station group x already in rotation on the channel. +fn stratum(c: usize, on: bool) -> usize { group(c) * 2 + on as usize } + +/// Pooled out-of-sample forecasts with their stratum, for a calibration per stratum. +struct StratPool(Vec<(usize, f64, bool, usize)>); +impl StratPool { + /// Isotonic per stratum on forecasts known by t0; strata with fewer than 10 adds use `fallback`. + fn fit(&self, t0: usize, gap: usize, fallback: &Calib) -> Vec { + (0..GROUPS * 2).map(|k| { + let pairs: Vec<(f64, bool)> = self.0.iter().filter(|(o, _, _, s)| *s == k && o + gap <= t0).map(|(_, p, y, _)| (*p, *y)).collect(); + if pairs.iter().filter(|x| x.1).count() >= 10 { Calib::isotonic(&pairs) } else { fallback.clone() } + }).collect() + } +} fn table(md: &mut String, s_c: &Scores, s_a: &Scores, rows: &mut Vec<(String, usize, String, f64)>, h: usize, prefix: &str) -> Result<()> { writeln!(md, "| Model | Log-loss, per channel | PR-AUC, per channel | ROC-AUC, per channel | Log-loss, any channel | PR-AUC, any channel | ROC-AUC, any channel |\n|---|---|---|---|---|---|---|")?; @@ -771,16 +796,16 @@ fn table(md: &mut String, s_c: &Scores, s_a: &Scores, rows: &mut Vec<(String, us } writeln!(md, "\nSkill (1 − log-loss ÷ baseline log-loss; 90% interval from a bootstrap over blocks of 8 forecast weeks):\n")?; writeln!(md, "| Model | Per channel vs last-12-weeks | Per channel vs release timing | Any channel vs last-12-weeks | Any channel vs release timing |\n|---|---|---|---|---|")?; - for k in [3usize, 5, 6] { + for k in [3usize, 5, 6, 8, 9] { let f = |s: &Scores, b: usize| { let x = skill(&s.col(k), &s.col(b)); format!("{:+.3} ({:+.3} to {:+.3})", x.0, x.1, x.2) }; writeln!(md, "| {} | {} | {} | {} | {} |", SCORED[k], f(s_c, 1), f(s_c, 2), f(s_a, 1), f(s_a, 2))?; rows.push((SCORED[k].to_string(), h, format!("{prefix}skill_any_vs_level"), skill(&s_a.col(k), &s_a.col(1)).0)); rows.push((SCORED[k].to_string(), h, format!("{prefix}skill_ch_vs_level"), skill(&s_c.col(k), &s_c.col(1)).0)); } writeln!(md, "\nCalibration, any channel:\n")?; - s_a.calib_table(md, &[3, 5, 6])?; + s_a.calib_table(md, &[5, 8])?; writeln!(md, "\nCalibration, per channel:\n")?; - s_c.calib_table(md, &[3, 5, 6])?; + s_c.calib_table(md, &[5, 8])?; writeln!(md)?; Ok(()) } @@ -804,19 +829,25 @@ pub fn backtest(p: &Panel, tau: &[u32; C], horizons: &[usize], start: usize, md: let last_origin = (p.w() - 1 - h - W - 1).min(*TEST_END.get().unwrap_or(&usize::MAX)); let (mut s_c, mut s_a) = (Scores::new(SCORED.to_vec()), Scores::new(SCORED.to_vec())); let (mut pool_c, mut pool_a) = (Pool(Vec::new()), Pool(Vec::new())); + let mut spool = StratPool(Vec::new()); + // Competition check: per origin and channel, (predicted total, actual adds, usual adds). + let mut comp: Vec<(usize, f64, f64, f64)> = Vec::new(); let lvl = |m12: f64| ((m12.ln_1p() / 0.5) as usize).min(7); for t0 in start..=last_origin { let m = AddModel::fit(p, &r, h, t0); // Baselines from the same closed windows. let (mut by_rot, mut by_lvl, mut any_rot, mut any_lvl) = ([[1.0f64, 2.0]; 2], [[1.0f64, 2.0]; 8], [[1.0f64, 2.0]; 2], [[1.0f64, 2.0]; 8]); let mut tm = Timing::new(); + let mut per_t = vec![[0.0f64; C + 1]; p.w()]; for t in origins(h, t0) { for a in 0..p.a() { if !p.known(a, t) { continue; } let key = Timing::key(p, a, t, h, W); let mut tot = 0.0; + per_t[t][C] += r.add_any_in_window(a, t, h) as i32 as f64; for c in 0..C { let y = r.add_in_window(a, c, t, h) as i32 as f64; + per_t[t][c] += y; let on = r.on(a, c, t) as usize; by_rot[on][0] += y; by_rot[on][1] += 1.0; let m12 = (0..LAGS).map(|i| p.y(a, c, t - i)).sum::() / LAGS as f64; @@ -833,7 +864,18 @@ pub fn backtest(p: &Panel, tau: &[u32; C], horizons: &[usize], start: usize, md: } let gap = h + W + 1; let ((pl_c, iso_c), (pl_a, iso_a)) = (pool_c.fit(t0, gap), pool_a.fit(t0, gap)); + let strat = spool.fit(t0, gap, &iso_c); + // The channel's usual number of adds per window: the last 52 training origins. + let tr = origins(h, t0); + let (thi, tlo) = (*tr.end(), (*tr.end()).saturating_sub(51).max(*tr.start())); + let usual: Vec = (0..=C).map(|c| (tlo..=thi).map(|t| per_t[t][c]).sum::() / (thi - tlo + 1) as f64).collect(); s_c.start_origin(); s_a.start_origin(); + // First pass: every artist's forecasts; second pass: scores (the competition variant + // needs the channel's predicted total). + let mut ch_rows: Vec<([f64; 8], f64, bool, usize)> = Vec::new(); + let mut any_rows: Vec<([f64; 8], f64, bool)> = Vec::new(); + let mut total = vec![0.0f64; C + 1]; + let mut actual = vec![0.0f64; C + 1]; for a in 0..p.a() { if !p.known(a, t0) { continue; } let q = p_nmf(p, a, t0); @@ -852,23 +894,59 @@ pub fn backtest(p: &Panel, tau: &[u32; C], horizons: &[usize], start: usize, md: tot += m12; let raw = q * rcn[c] + (1.0 - q) * rcr[c]; let tmc = tm.ch[c * 12 + key]; - s_c.add(&[by_rot[on][0] / by_rot[on][1], by_lvl[lvl(m12)][0] / by_lvl[lvl(m12)][1], tmc[0] / tmc[1], - q * pcn[c] + (1.0 - q) * pcr[c], raw, pl_c.apply(raw), iso_c.apply(raw), pcx[c]], y); + let k = stratum(c, on == 1); + let ps = if std::env::var("AIRPLAY_CAP_CH").is_ok_and(|v| v == "1") { pl_c.apply(raw) } else { strat.get(k).map(|cal| cal.apply(raw)).unwrap_or(raw) }; + ch_rows.push(([by_rot[on][0] / by_rot[on][1], by_lvl[lvl(m12)][0] / by_lvl[lvl(m12)][1], tmc[0] / tmc[1], + q * pcn[c] + (1.0 - q) * pcr[c], raw, pl_c.apply(raw), iso_c.apply(raw), pcx[c]], ps, y, c)); + actual[c] += y as i32 as f64; pool_c.0.push((t0, raw, y)); + spool.0.push((t0, raw, y, k)); } + let best = ch_rows[ch_rows.len() - C..].iter().map(|r| r.1).fold(0.0, f64::max); let y = r.add_any_in_window(a, t0, h); let on = r.any(a, t0) as usize; let raw = q * ran + (1.0 - q) * rar; let tma = tm.any[key]; - s_a.add(&[any_rot[on][0] / any_rot[on][1], any_lvl[lvl(tot)][0] / any_lvl[lvl(tot)][1], tma[0] / tma[1], - q * pan + (1.0 - q) * par, raw, pl_a.apply(raw), iso_a.apply(raw), pax], y); + let cap = std::env::var("AIRPLAY_CAP_CH").is_ok_and(|v| v == "1"); + let pf = if cap { iso_a.apply(raw) } else { iso_a.apply(raw).max(best) }; + // With the cap, this artist's channel chances are capped at the any-channel chance. + if cap { let n = ch_rows.len(); for row in ch_rows[n - C..].iter_mut() { row.1 = row.1.min(pf); } } + any_rows.push(([any_rot[on][0] / any_rot[on][1], any_lvl[lvl(tot)][0] / any_lvl[lvl(tot)][1], tma[0] / tma[1], + q * pan + (1.0 - q) * par, raw, pl_a.apply(raw), iso_a.apply(raw), pax], pf, y)); + actual[C] += y as i32 as f64; pool_a.0.push((t0, raw, y)); } + for (_, ps, _, c) in &ch_rows { total[*c] += ps; } + for (_, pf, _) in &any_rows { total[C] += pf; } + let scale: Vec = (0..=C).map(|c| if total[c] > 0.0 && usual[c] > 0.0 { usual[c] / total[c] } else { 1.0 }).collect(); + for (v, ps, y, c) in &ch_rows { + let mut x = v.to_vec(); x.push(*ps); x.push((ps * scale[*c]).clamp(1e-6, 0.999)); + s_c.add(&x, *y); + } + for (v, pf, y) in &any_rows { + let mut x = v.to_vec(); x.push(*pf); x.push((pf * scale[C]).clamp(1e-6, 0.999)); + s_a.add(&x, *y); + } + for c in 0..=C { comp.push((c, total[c], actual[c], usual[c])); } eprintln!("adds backtest h={h}: origin {} done", p.weeks[t0]); } writeln!(md, "## Window starting h = {h} weeks ahead ({} forecast origins)\n", s_c.wk.len())?; writeln!(md, "Base rates: per channel {:.2}% of artist-channels get an add in the window; on any channel {:.1}% of artists.\n", 100.0 * s_c.hits / s_c.n.max(1.0), 100.0 * s_a.hits / s_a.n.max(1.0))?; table(md, &s_c, &s_a, &mut rows, h, "add_")?; + // Is there a cap? Regress actual adds on the predicted total, per channel (each centred on + // its own mean): slope ~1 = adds follow supply (no cap), ~0 = a fixed number of slots. + let mut md2 = String::new(); + for (label, sel) in [("per channel", (0..C).collect::>()), ("any channel", vec![C])] { + let (mut sxy, mut sxx, mut syy) = (0.0, 0.0, 0.0); + for &c in &sel { + let v: Vec<&(usize, f64, f64, f64)> = comp.iter().filter(|x| x.0 == c).collect(); + if v.len() < 3 { continue; } + let (mx, my) = (v.iter().map(|x| x.1).sum::() / v.len() as f64, v.iter().map(|x| x.2).sum::() / v.len() as f64); + for x in v { sxy += (x.1 - mx) * (x.2 - my); sxx += (x.1 - mx).powi(2); syy += (x.2 - my).powi(2); } + } + if sxx > 0.0 && syy > 0.0 { let _ = writeln!(md2, "- {label}: actual adds rise {:.2} per predicted add (correlation {:.2}) across forecast weeks", sxy / sxx, sxy / (sxx * syy).sqrt()); } + } + writeln!(md, "Competition check (does a week with more predicted adds get more actual adds?):\n\n{md2}")?; if s_c.wk.len() < 8 { writeln!(md, "Fewer than 8 forecast origins: no verdict at this horizon (doc rule).\n")?; } } let m = AddModel::fit(p, &r, 0, p.w() - 1); @@ -892,6 +970,7 @@ pub fn backtest_events(p: &Panel, tau: &[u32; C], start: usize, win: usize, md: let level = |a: usize, t: usize| (0..LAGS).map(|i| p.group_total(a, t - i, None)).sum::() / LAGS as f64; // Release events by quiet artists (out of rotation 26 weeks): former favourites (4+ weeks on DR 2019-24) or not. let mut favs: Vec<(f64, f64, bool)> = Vec::new(); + let mut spool = StratPool(Vec::new()); let mut quiet_other: Vec<(f64, f64, bool)> = Vec::new(); for t0 in start..p.w().saturating_sub(win + 1).min(TEST_END.get().map_or(usize::MAX, |e| e + 1)) { let m = AddModel::fit_events_win(p, &r, t0, win); @@ -919,6 +998,7 @@ pub fn backtest_events(p: &Panel, tau: &[u32; C], start: usize, win: usize, md: } let gap = win + 1; let ((pl_c, iso_c), (pl_a, iso_a)) = (pool_c.fit(t0, gap), pool_a.fit(t0, gap)); + let strat = spool.fit(t0, gap, &iso_c); let mut started = false; for a in 0..p.a() { if !p.known(a, t0) || !clean_release(p, a, t0 + 1) { continue; } @@ -934,8 +1014,13 @@ pub fn backtest_events(p: &Panel, tau: &[u32; C], start: usize, win: usize, md: let on = r.any(a, t0) as usize; let raw = q * ran + (1.0 - q) * rar; let tma = tm.any[key]; + let cap = std::env::var("AIRPLAY_CAP_CH").is_ok_and(|v| v == "1"); + let ps: Vec = (0..C).map(|c| { let rawc = q * rcn[c] + (1.0 - q) * rcr[c]; + // AIRPLAY_CAP_CH=1: the other direction of coherence, each channel capped at the any-channel chance. + if cap { pl_c.apply(rawc).min(iso_a.apply(raw)) } else { strat.get(stratum(c, r.on(a, c, t0))).map(|k| k.apply(rawc)).unwrap_or(rawc) } }).collect(); + let pf = if cap { iso_a.apply(raw) } else { iso_a.apply(raw).max(ps.iter().copied().fold(0.0, f64::max)) }; s_a.add(&[by_rot[on][0] / by_rot[on][1], by_lvl[lvl(level(a, t0))][0] / by_lvl[lvl(level(a, t0))][1], tma[0] / tma[1], - q * pan + (1.0 - q) * par, raw, pl_a.apply(raw), iso_a.apply(raw), pax], y); + q * pan + (1.0 - q) * par, raw, pl_a.apply(raw), iso_a.apply(raw), pax, pf, pf], y); pool_a.0.push((t0, raw, y)); if quiet(&r, a, t0) { if p.fav_before(a, t0) >= 4 { favs.push((raw, iso_a.apply(raw), y)); } else { quiet_other.push((raw, iso_a.apply(raw), y)); } } for c in 0..C { @@ -945,8 +1030,9 @@ pub fn backtest_events(p: &Panel, tau: &[u32; C], start: usize, win: usize, md: let rawc = q * rcn[c] + (1.0 - q) * rcr[c]; let tmc = tm.ch[c * 12 + key]; s_c.add(&[cr[onc][0] / cr[onc][1], cl[bc][0] / cl[bc][1], tmc[0] / tmc[1], - q * pcn[c] + (1.0 - q) * pcr[c], rawc, pl_c.apply(rawc), iso_c.apply(rawc), pcx[c]], yc); + q * pcn[c] + (1.0 - q) * pcr[c], rawc, pl_c.apply(rawc), iso_c.apply(rawc), pcx[c], ps[c], ps[c]], yc); pool_c.0.push((t0, rawc, yc)); + spool.0.push((t0, rawc, yc, stratum(c, onc == 1))); } } eprintln!("event backtest: origin {} done", p.weeks[t0]);