Release-event model: channels capped at the any-channel chance; stratified calibration and competition scaling tested (rejected); slots analysis; docs
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -62,3 +62,21 @@ memory down. Switching it on for the site was reverted after the service hit its
|
||||
Also found: per-channel probabilities above ~0.2 are overconfident (40-50% predicted, ~8% observed),
|
||||
mostly P4 regions for artists already in rotation (Mumle: 44% on P4 vs 6% any channel). A floor making
|
||||
"any" >= the best station group made "any" worse (0.0664 -> 0.0690) until those are recalibrated.
|
||||
|
||||
## Calibration, competition and slots (2026-11-20), history on
|
||||
|
||||
- **Site**: history on (`AIRPLAY_HISTORY=1` for airplay-web and airplay-collect); fitting peaks at ~2.5 GB,
|
||||
so both services have `MemoryMax=3584M` (drop-in history.conf).
|
||||
- **Per-channel calibration by station group x in rotation (isotonic)**: no gain (release events per-channel
|
||||
log loss 0.0150 vs 0.0148 pooled Platt); the 50-60% bin still came true 0 of 16 times. P4's regions repeat
|
||||
the same few events, so the tail cannot be learned.
|
||||
- **Cap each channel at the any-channel chance**: release events per-channel PR-AUC 0.105 -> 0.165, log loss
|
||||
0.0148 -> 0.0143, "any" unchanged; general model neutral to slightly worse (PR-AUC 0.285 -> 0.267).
|
||||
Applied to the release-event model only (the site's "if they release" numbers: Mumle 44% on P4 vs 6% any).
|
||||
- **Competition scaling** (each week, scale a channel's chances to its usual number of adds): worse everywhere
|
||||
(any 0.0736 -> 0.0760). Over 8-week windows actual adds rise with predicted supply (1.4-1.7 per predicted
|
||||
add, correlation 0.72-0.82 at h=4): no fixed capacity to share. Rejected.
|
||||
- **Slots per channel** (Poisson deviance on 26 weeks May-Nov 2026, five main channels): flat recent average
|
||||
232, week-of-year average 833, recent level x shrunk calendar blocks 217. Blocks: summer holiday (weeks 28-31:
|
||||
P3 0.66, P4 0.77, P5 0.83), New Year (weeks 1-2), December (not trustworthy until song-level data separates
|
||||
Christmas music). Not yet in the model.
|
||||
|
||||
+98
-12
@@ -640,7 +640,15 @@ impl AddModel {
|
||||
// is an add on some channel. Off: the per-channel numbers are overconfident above ~0.2, and
|
||||
// the floor made "any" worse (log loss 0.0664 -> 0.0690) until they are recalibrated.
|
||||
let floor = if std::env::var("AIRPLAY_ANY_FLOOR").is_ok_and(|v| v == "1") { best.iter().copied().fold(0.0, f64::max) } else { 0.0 };
|
||||
(pc, pa.max(floor).min(cap.max(1e-6)))
|
||||
let pa = pa.max(floor).min(cap.max(1e-6));
|
||||
// The release-event model (the only one with pooled calibration): each channel at most the
|
||||
// any-channel chance. Its per-channel odds for artists already in rotation were overconfident
|
||||
// (40-60% predicted, ~0-10% observed: P4's regions repeat the same few events); capped, the
|
||||
// per-channel PR-AUC went 0.105 -> 0.165 with "any" unchanged. AIRPLAY_CAP_CH=0 turns it off.
|
||||
if self.calib_any.is_some() && std::env::var("AIRPLAY_CAP_CH").map_or(true, |v| v != "0") {
|
||||
for v in pc.iter_mut() { *v = v.min(pa); }
|
||||
}
|
||||
(pc, pa)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -755,9 +763,26 @@ impl Pool {
|
||||
}
|
||||
|
||||
/// The models scored in both backtests (index: meaning).
|
||||
const SCORED: [&str; 8] = ["rate by in-rotation-now", "rate by last-12-weeks level", "release timing only",
|
||||
const SCORED: [&str; 10] = ["rate by in-rotation-now", "rate by last-12-weeks level", "release timing only",
|
||||
"model, calibrated on the last 6 origins (pre-release)", "model, raw (pre-release)",
|
||||
"model + pooled Platt (pre-release)", "model + pooled isotonic (pre-release)", "model, NMF known (post-release)"];
|
||||
"model + pooled Platt (pre-release)", "model + pooled isotonic (pre-release)", "model, NMF known (post-release)",
|
||||
"model + isotonic by station group x in rotation; any >= best channel (pre-release)",
|
||||
"the same + competition scaling to the channel's usual adds (pre-release)"];
|
||||
|
||||
/// Calibration strata: station group x already in rotation on the channel.
|
||||
fn stratum(c: usize, on: bool) -> usize { group(c) * 2 + on as usize }
|
||||
|
||||
/// Pooled out-of-sample forecasts with their stratum, for a calibration per stratum.
|
||||
struct StratPool(Vec<(usize, f64, bool, usize)>);
|
||||
impl StratPool {
|
||||
/// Isotonic per stratum on forecasts known by t0; strata with fewer than 10 adds use `fallback`.
|
||||
fn fit(&self, t0: usize, gap: usize, fallback: &Calib) -> Vec<Calib> {
|
||||
(0..GROUPS * 2).map(|k| {
|
||||
let pairs: Vec<(f64, bool)> = self.0.iter().filter(|(o, _, _, s)| *s == k && o + gap <= t0).map(|(_, p, y, _)| (*p, *y)).collect();
|
||||
if pairs.iter().filter(|x| x.1).count() >= 10 { Calib::isotonic(&pairs) } else { fallback.clone() }
|
||||
}).collect()
|
||||
}
|
||||
}
|
||||
|
||||
fn table(md: &mut String, s_c: &Scores, s_a: &Scores, rows: &mut Vec<(String, usize, String, f64)>, h: usize, prefix: &str) -> Result<()> {
|
||||
writeln!(md, "| Model | Log-loss, per channel | PR-AUC, per channel | ROC-AUC, per channel | Log-loss, any channel | PR-AUC, any channel | ROC-AUC, any channel |\n|---|---|---|---|---|---|---|")?;
|
||||
@@ -771,16 +796,16 @@ fn table(md: &mut String, s_c: &Scores, s_a: &Scores, rows: &mut Vec<(String, us
|
||||
}
|
||||
writeln!(md, "\nSkill (1 − log-loss ÷ baseline log-loss; 90% interval from a bootstrap over blocks of 8 forecast weeks):\n")?;
|
||||
writeln!(md, "| Model | Per channel vs last-12-weeks | Per channel vs release timing | Any channel vs last-12-weeks | Any channel vs release timing |\n|---|---|---|---|---|")?;
|
||||
for k in [3usize, 5, 6] {
|
||||
for k in [3usize, 5, 6, 8, 9] {
|
||||
let f = |s: &Scores, b: usize| { let x = skill(&s.col(k), &s.col(b)); format!("{:+.3} ({:+.3} to {:+.3})", x.0, x.1, x.2) };
|
||||
writeln!(md, "| {} | {} | {} | {} | {} |", SCORED[k], f(s_c, 1), f(s_c, 2), f(s_a, 1), f(s_a, 2))?;
|
||||
rows.push((SCORED[k].to_string(), h, format!("{prefix}skill_any_vs_level"), skill(&s_a.col(k), &s_a.col(1)).0));
|
||||
rows.push((SCORED[k].to_string(), h, format!("{prefix}skill_ch_vs_level"), skill(&s_c.col(k), &s_c.col(1)).0));
|
||||
}
|
||||
writeln!(md, "\nCalibration, any channel:\n")?;
|
||||
s_a.calib_table(md, &[3, 5, 6])?;
|
||||
s_a.calib_table(md, &[5, 8])?;
|
||||
writeln!(md, "\nCalibration, per channel:\n")?;
|
||||
s_c.calib_table(md, &[3, 5, 6])?;
|
||||
s_c.calib_table(md, &[5, 8])?;
|
||||
writeln!(md)?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -804,19 +829,25 @@ pub fn backtest(p: &Panel, tau: &[u32; C], horizons: &[usize], start: usize, md:
|
||||
let last_origin = (p.w() - 1 - h - W - 1).min(*TEST_END.get().unwrap_or(&usize::MAX));
|
||||
let (mut s_c, mut s_a) = (Scores::new(SCORED.to_vec()), Scores::new(SCORED.to_vec()));
|
||||
let (mut pool_c, mut pool_a) = (Pool(Vec::new()), Pool(Vec::new()));
|
||||
let mut spool = StratPool(Vec::new());
|
||||
// Competition check: per origin and channel, (predicted total, actual adds, usual adds).
|
||||
let mut comp: Vec<(usize, f64, f64, f64)> = Vec::new();
|
||||
let lvl = |m12: f64| ((m12.ln_1p() / 0.5) as usize).min(7);
|
||||
for t0 in start..=last_origin {
|
||||
let m = AddModel::fit(p, &r, h, t0);
|
||||
// Baselines from the same closed windows.
|
||||
let (mut by_rot, mut by_lvl, mut any_rot, mut any_lvl) = ([[1.0f64, 2.0]; 2], [[1.0f64, 2.0]; 8], [[1.0f64, 2.0]; 2], [[1.0f64, 2.0]; 8]);
|
||||
let mut tm = Timing::new();
|
||||
let mut per_t = vec![[0.0f64; C + 1]; p.w()];
|
||||
for t in origins(h, t0) {
|
||||
for a in 0..p.a() {
|
||||
if !p.known(a, t) { continue; }
|
||||
let key = Timing::key(p, a, t, h, W);
|
||||
let mut tot = 0.0;
|
||||
per_t[t][C] += r.add_any_in_window(a, t, h) as i32 as f64;
|
||||
for c in 0..C {
|
||||
let y = r.add_in_window(a, c, t, h) as i32 as f64;
|
||||
per_t[t][c] += y;
|
||||
let on = r.on(a, c, t) as usize;
|
||||
by_rot[on][0] += y; by_rot[on][1] += 1.0;
|
||||
let m12 = (0..LAGS).map(|i| p.y(a, c, t - i)).sum::<f64>() / LAGS as f64;
|
||||
@@ -833,7 +864,18 @@ pub fn backtest(p: &Panel, tau: &[u32; C], horizons: &[usize], start: usize, md:
|
||||
}
|
||||
let gap = h + W + 1;
|
||||
let ((pl_c, iso_c), (pl_a, iso_a)) = (pool_c.fit(t0, gap), pool_a.fit(t0, gap));
|
||||
let strat = spool.fit(t0, gap, &iso_c);
|
||||
// The channel's usual number of adds per window: the last 52 training origins.
|
||||
let tr = origins(h, t0);
|
||||
let (thi, tlo) = (*tr.end(), (*tr.end()).saturating_sub(51).max(*tr.start()));
|
||||
let usual: Vec<f64> = (0..=C).map(|c| (tlo..=thi).map(|t| per_t[t][c]).sum::<f64>() / (thi - tlo + 1) as f64).collect();
|
||||
s_c.start_origin(); s_a.start_origin();
|
||||
// First pass: every artist's forecasts; second pass: scores (the competition variant
|
||||
// needs the channel's predicted total).
|
||||
let mut ch_rows: Vec<([f64; 8], f64, bool, usize)> = Vec::new();
|
||||
let mut any_rows: Vec<([f64; 8], f64, bool)> = Vec::new();
|
||||
let mut total = vec![0.0f64; C + 1];
|
||||
let mut actual = vec![0.0f64; C + 1];
|
||||
for a in 0..p.a() {
|
||||
if !p.known(a, t0) { continue; }
|
||||
let q = p_nmf(p, a, t0);
|
||||
@@ -852,23 +894,59 @@ pub fn backtest(p: &Panel, tau: &[u32; C], horizons: &[usize], start: usize, md:
|
||||
tot += m12;
|
||||
let raw = q * rcn[c] + (1.0 - q) * rcr[c];
|
||||
let tmc = tm.ch[c * 12 + key];
|
||||
s_c.add(&[by_rot[on][0] / by_rot[on][1], by_lvl[lvl(m12)][0] / by_lvl[lvl(m12)][1], tmc[0] / tmc[1],
|
||||
q * pcn[c] + (1.0 - q) * pcr[c], raw, pl_c.apply(raw), iso_c.apply(raw), pcx[c]], y);
|
||||
let k = stratum(c, on == 1);
|
||||
let ps = if std::env::var("AIRPLAY_CAP_CH").is_ok_and(|v| v == "1") { pl_c.apply(raw) } else { strat.get(k).map(|cal| cal.apply(raw)).unwrap_or(raw) };
|
||||
ch_rows.push(([by_rot[on][0] / by_rot[on][1], by_lvl[lvl(m12)][0] / by_lvl[lvl(m12)][1], tmc[0] / tmc[1],
|
||||
q * pcn[c] + (1.0 - q) * pcr[c], raw, pl_c.apply(raw), iso_c.apply(raw), pcx[c]], ps, y, c));
|
||||
actual[c] += y as i32 as f64;
|
||||
pool_c.0.push((t0, raw, y));
|
||||
spool.0.push((t0, raw, y, k));
|
||||
}
|
||||
let best = ch_rows[ch_rows.len() - C..].iter().map(|r| r.1).fold(0.0, f64::max);
|
||||
let y = r.add_any_in_window(a, t0, h);
|
||||
let on = r.any(a, t0) as usize;
|
||||
let raw = q * ran + (1.0 - q) * rar;
|
||||
let tma = tm.any[key];
|
||||
s_a.add(&[any_rot[on][0] / any_rot[on][1], any_lvl[lvl(tot)][0] / any_lvl[lvl(tot)][1], tma[0] / tma[1],
|
||||
q * pan + (1.0 - q) * par, raw, pl_a.apply(raw), iso_a.apply(raw), pax], y);
|
||||
let cap = std::env::var("AIRPLAY_CAP_CH").is_ok_and(|v| v == "1");
|
||||
let pf = if cap { iso_a.apply(raw) } else { iso_a.apply(raw).max(best) };
|
||||
// With the cap, this artist's channel chances are capped at the any-channel chance.
|
||||
if cap { let n = ch_rows.len(); for row in ch_rows[n - C..].iter_mut() { row.1 = row.1.min(pf); } }
|
||||
any_rows.push(([any_rot[on][0] / any_rot[on][1], any_lvl[lvl(tot)][0] / any_lvl[lvl(tot)][1], tma[0] / tma[1],
|
||||
q * pan + (1.0 - q) * par, raw, pl_a.apply(raw), iso_a.apply(raw), pax], pf, y));
|
||||
actual[C] += y as i32 as f64;
|
||||
pool_a.0.push((t0, raw, y));
|
||||
}
|
||||
for (_, ps, _, c) in &ch_rows { total[*c] += ps; }
|
||||
for (_, pf, _) in &any_rows { total[C] += pf; }
|
||||
let scale: Vec<f64> = (0..=C).map(|c| if total[c] > 0.0 && usual[c] > 0.0 { usual[c] / total[c] } else { 1.0 }).collect();
|
||||
for (v, ps, y, c) in &ch_rows {
|
||||
let mut x = v.to_vec(); x.push(*ps); x.push((ps * scale[*c]).clamp(1e-6, 0.999));
|
||||
s_c.add(&x, *y);
|
||||
}
|
||||
for (v, pf, y) in &any_rows {
|
||||
let mut x = v.to_vec(); x.push(*pf); x.push((pf * scale[C]).clamp(1e-6, 0.999));
|
||||
s_a.add(&x, *y);
|
||||
}
|
||||
for c in 0..=C { comp.push((c, total[c], actual[c], usual[c])); }
|
||||
eprintln!("adds backtest h={h}: origin {} done", p.weeks[t0]);
|
||||
}
|
||||
writeln!(md, "## Window starting h = {h} weeks ahead ({} forecast origins)\n", s_c.wk.len())?;
|
||||
writeln!(md, "Base rates: per channel {:.2}% of artist-channels get an add in the window; on any channel {:.1}% of artists.\n", 100.0 * s_c.hits / s_c.n.max(1.0), 100.0 * s_a.hits / s_a.n.max(1.0))?;
|
||||
table(md, &s_c, &s_a, &mut rows, h, "add_")?;
|
||||
// Is there a cap? Regress actual adds on the predicted total, per channel (each centred on
|
||||
// its own mean): slope ~1 = adds follow supply (no cap), ~0 = a fixed number of slots.
|
||||
let mut md2 = String::new();
|
||||
for (label, sel) in [("per channel", (0..C).collect::<Vec<_>>()), ("any channel", vec![C])] {
|
||||
let (mut sxy, mut sxx, mut syy) = (0.0, 0.0, 0.0);
|
||||
for &c in &sel {
|
||||
let v: Vec<&(usize, f64, f64, f64)> = comp.iter().filter(|x| x.0 == c).collect();
|
||||
if v.len() < 3 { continue; }
|
||||
let (mx, my) = (v.iter().map(|x| x.1).sum::<f64>() / v.len() as f64, v.iter().map(|x| x.2).sum::<f64>() / v.len() as f64);
|
||||
for x in v { sxy += (x.1 - mx) * (x.2 - my); sxx += (x.1 - mx).powi(2); syy += (x.2 - my).powi(2); }
|
||||
}
|
||||
if sxx > 0.0 && syy > 0.0 { let _ = writeln!(md2, "- {label}: actual adds rise {:.2} per predicted add (correlation {:.2}) across forecast weeks", sxy / sxx, sxy / (sxx * syy).sqrt()); }
|
||||
}
|
||||
writeln!(md, "Competition check (does a week with more predicted adds get more actual adds?):\n\n{md2}")?;
|
||||
if s_c.wk.len() < 8 { writeln!(md, "Fewer than 8 forecast origins: no verdict at this horizon (doc rule).\n")?; }
|
||||
}
|
||||
let m = AddModel::fit(p, &r, 0, p.w() - 1);
|
||||
@@ -892,6 +970,7 @@ pub fn backtest_events(p: &Panel, tau: &[u32; C], start: usize, win: usize, md:
|
||||
let level = |a: usize, t: usize| (0..LAGS).map(|i| p.group_total(a, t - i, None)).sum::<f64>() / LAGS as f64;
|
||||
// Release events by quiet artists (out of rotation 26 weeks): former favourites (4+ weeks on DR 2019-24) or not.
|
||||
let mut favs: Vec<(f64, f64, bool)> = Vec::new();
|
||||
let mut spool = StratPool(Vec::new());
|
||||
let mut quiet_other: Vec<(f64, f64, bool)> = Vec::new();
|
||||
for t0 in start..p.w().saturating_sub(win + 1).min(TEST_END.get().map_or(usize::MAX, |e| e + 1)) {
|
||||
let m = AddModel::fit_events_win(p, &r, t0, win);
|
||||
@@ -919,6 +998,7 @@ pub fn backtest_events(p: &Panel, tau: &[u32; C], start: usize, win: usize, md:
|
||||
}
|
||||
let gap = win + 1;
|
||||
let ((pl_c, iso_c), (pl_a, iso_a)) = (pool_c.fit(t0, gap), pool_a.fit(t0, gap));
|
||||
let strat = spool.fit(t0, gap, &iso_c);
|
||||
let mut started = false;
|
||||
for a in 0..p.a() {
|
||||
if !p.known(a, t0) || !clean_release(p, a, t0 + 1) { continue; }
|
||||
@@ -934,8 +1014,13 @@ pub fn backtest_events(p: &Panel, tau: &[u32; C], start: usize, win: usize, md:
|
||||
let on = r.any(a, t0) as usize;
|
||||
let raw = q * ran + (1.0 - q) * rar;
|
||||
let tma = tm.any[key];
|
||||
let cap = std::env::var("AIRPLAY_CAP_CH").is_ok_and(|v| v == "1");
|
||||
let ps: Vec<f64> = (0..C).map(|c| { let rawc = q * rcn[c] + (1.0 - q) * rcr[c];
|
||||
// AIRPLAY_CAP_CH=1: the other direction of coherence, each channel capped at the any-channel chance.
|
||||
if cap { pl_c.apply(rawc).min(iso_a.apply(raw)) } else { strat.get(stratum(c, r.on(a, c, t0))).map(|k| k.apply(rawc)).unwrap_or(rawc) } }).collect();
|
||||
let pf = if cap { iso_a.apply(raw) } else { iso_a.apply(raw).max(ps.iter().copied().fold(0.0, f64::max)) };
|
||||
s_a.add(&[by_rot[on][0] / by_rot[on][1], by_lvl[lvl(level(a, t0))][0] / by_lvl[lvl(level(a, t0))][1], tma[0] / tma[1],
|
||||
q * pan + (1.0 - q) * par, raw, pl_a.apply(raw), iso_a.apply(raw), pax], y);
|
||||
q * pan + (1.0 - q) * par, raw, pl_a.apply(raw), iso_a.apply(raw), pax, pf, pf], y);
|
||||
pool_a.0.push((t0, raw, y));
|
||||
if quiet(&r, a, t0) { if p.fav_before(a, t0) >= 4 { favs.push((raw, iso_a.apply(raw), y)); } else { quiet_other.push((raw, iso_a.apply(raw), y)); } }
|
||||
for c in 0..C {
|
||||
@@ -945,8 +1030,9 @@ pub fn backtest_events(p: &Panel, tau: &[u32; C], start: usize, win: usize, md:
|
||||
let rawc = q * rcn[c] + (1.0 - q) * rcr[c];
|
||||
let tmc = tm.ch[c * 12 + key];
|
||||
s_c.add(&[cr[onc][0] / cr[onc][1], cl[bc][0] / cl[bc][1], tmc[0] / tmc[1],
|
||||
q * pcn[c] + (1.0 - q) * pcr[c], rawc, pl_c.apply(rawc), iso_c.apply(rawc), pcx[c]], yc);
|
||||
q * pcn[c] + (1.0 - q) * pcr[c], rawc, pl_c.apply(rawc), iso_c.apply(rawc), pcx[c], ps[c], ps[c]], yc);
|
||||
pool_c.0.push((t0, rawc, yc));
|
||||
spool.0.push((t0, rawc, yc, stratum(c, onc == 1)));
|
||||
}
|
||||
}
|
||||
eprintln!("event backtest: origin {} done", p.weeks[t0]);
|
||||
|
||||
Reference in New Issue
Block a user