History: boundary fix (spurious adds, dropped rows), archive rows subsampled, both models use archive at 0.3; optional any>=channel floor; ROC; docs

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-20 00:53:36 +00:00
co-authored by Claude Opus 5.5
parent 30fed60407
commit a3b225fa48
3 changed files with 113 additions and 26 deletions
+27
View File
@@ -35,3 +35,30 @@ Findings (2026-11-19):
the model did worse than "recent plays" there. the model did worse than "recent plays" there.
- Not switched on for the site. Next: cap the history-dependent features at 52 weeks so archive and live - Not switched on for the site. Next: cap the history-dependent features at 52 weeks so archive and live
rows mean the same thing, then re-test. rows mean the same thing, then re-test.
## Correction (2026-11-20): the earlier "history hurts" verdict was a bug
At the boundary between archive and live weeks, artists without an archived copy looked silent in the
archive weeks: an artist already in rotation when the live data begins showed a spurious add, and the
"20 observed weeks before a row" rule dropped uncovered artists from the first 20 live weeks, so the
general model trained mostly on popular (archived) artists. Fixed: an add needs GAP (surge: 8) observed
weeks before it, and a row needs the 12 observed lag weeks the one-year panel needs; with archive weight 0
the history panel then reproduces the one-year model (up to the larger Christmas-artist list).
After the fix, recalibrated (Platt), same test weeks (May-Nov 2026):
| | 1 year | 2023-26, archive weight 0.3 |
|---|---|---|
| General model, any channel, log loss h=0 / h=4 | 0.0742 / 0.0789 | 0.0736 / 0.0782 |
| General model, per channel, log loss h=0 / h=4 | 0.0135 / 0.0160 | 0.0130 / 0.0152 |
| General model, per channel, PR-AUC h=0 / h=4 | 0.266 / 0.170 | 0.287 / 0.198 |
| Release events, any channel, log loss / ROC-AUC / PR-AUC | 0.0655 / 0.882 / 0.224 | 0.0644 / 0.895 / 0.211 |
| Release events, per channel, log loss | 0.0158 | 0.0151 |
History now wins on the coming period nearly everywhere; by the project rule (largest dataset, then the
best accuracy on the coming period) it should be on. Archive rows are subsampled (share = weight) to keep
memory down. Switching it on for the site was reverted after the service hit its 1.5 GB memory limit.
Also found: per-channel probabilities above ~0.2 are overconfident (40-50% predicted, ~8% observed),
mostly P4 regions for artists already in rotation (Mumle: 44% on P4 vs 6% any channel). A floor making
"any" >= the best station group made "any" worse (0.0664 -> 0.0690) until those are recalibrated.
+62 -14
View File
@@ -47,7 +47,8 @@ pub fn wave_feats(p: &Panel, r: &Rot, a: usize, t: usize, shift: usize) -> [f64;
if off("wave") { return [0.0; WAVE]; } if off("wave") { return [0.0; WAVE]; }
// An add in week w is only confirmed in week w + 1 (the surge rule reads the following week), // An add in week w is only confirmed in week w + 1 (the surge rule reads the following week),
// so as of week t only adds up to t - 1 are known. // so as of week t only adds up to t - 1 are known.
let since = (1..=t).find(|&i| r.add_any(a, t - i)).map(|s| s + shift); // At most a year back, so "no add yet" means the same with a longer panel (2023-) as with one year.
let since = (1..=t.min(52)).find(|&i| r.add_any(a, t - i)).map(|s| s + shift);
let ch = p.chart(a, t); let ch = p.chart(a, t);
let f = |b: bool| b as i32 as f64; let f = |b: bool| b as i32 as f64;
let air = ch.air_since as f64 + shift as f64; let air = ch.air_since as f64 + shift as f64;
@@ -177,7 +178,7 @@ pub fn long_feats(p: &Panel, r: &Rot, a: usize, t: usize, _h: usize, sc: Scenari
// The comeback case: a former favourite (archive pages 2019-24, before the panel) out of // The comeback case: a former favourite (archive pages 2019-24, before the panel) out of
// rotation for half a year. One bounded flag: a continuous favourite score next to its // rotation for half a year. One bounded flag: a continuous favourite score next to its
// interaction with "quiet" fitted two large opposing coefficients on one panel year. // interaction with "quiet" fitted two large opposing coefficients on one panel year.
let fav = !off("fav") && p.fav_weeks[a] >= 4.0 && quiet(r, a, t); let fav = !off("fav") && p.fav_before(a, t) >= 4 && quiet(r, a, t);
let end_t = p.weeks[t] + chrono::Duration::days(6); let end_t = p.weeks[t] + chrono::Duration::days(6);
// A list published on a Friday is known from then. // A list published on a Friday is known from then.
let before: Vec<&chrono::NaiveDate> = p.listed_on[a].iter().filter(|d| **d <= end_t).collect(); let before: Vec<&chrono::NaiveDate> = p.listed_on[a].iter().filter(|d| **d <= end_t).collect();
@@ -309,8 +310,10 @@ impl Rot {
let on = |w: usize| rot[(a * C + c) * nw + w]; let on = |w: usize| rot[(a * C + c) * nw + w];
let t = tau[c] as f64; let t = tau[c] as f64;
for w in GAP..nw { for w in GAP..nw {
let onset = on(w) && (1..=GAP).all(|k| !on(w - k)); // As at the panel's start: an add needs GAP observed weeks before it (8 for a surge).
let jump = surge && w >= 8 && w + 1 < nw && { let s0 = p.run_start(a, w);
let onset = w >= s0 + GAP && on(w) && (1..=GAP).all(|k| !on(w - k));
let jump = surge && w >= s0 + 8 && w + 1 < nw && p.observed(a, w + 1) && {
let mut b: Vec<f64> = (w - 8..w).map(|k| p.y(a, c, k)).collect(); let mut b: Vec<f64> = (w - 8..w).map(|k| p.y(a, c, k)).collect();
b.sort_by(f64::total_cmp); b.sort_by(f64::total_cmp);
let base = b[4]; let base = b[4];
@@ -321,7 +324,7 @@ impl Rot {
} }
} }
for w in GAP..nw { for w in GAP..nw {
let classic = any[a * nw + w] && (1..=GAP).all(|k| !any[a * nw + w - k]); let classic = w >= p.run_start(a, w) + GAP && any[a * nw + w] && (1..=GAP).all(|k| !any[a * nw + w - k]);
let some = if surge { (0..C).any(|c| adds[(a * C + c) * nw + w]) } else { classic }; let some = if surge { (0..C).any(|c| adds[(a * C + c) * nw + w]) } else { classic };
if some && (1..=GAP).all(|k| !adds_any[a * nw + w - k]) { adds_any[a * nw + w] = true; } if some && (1..=GAP).all(|k| !adds_any[a * nw + w - k]) { adds_any[a * nw + w] = true; }
} }
@@ -434,10 +437,35 @@ pub struct AddModel {
pub calib_any: Option<Calib>, pub calib_any: Option<Calib>,
} }
/// Training weight of an archive-era row (AIRPLAY_ARCH_W, default 1; live rows always 1). /// The intercept shift that makes the model's predictions fit the selected rows (one-dimensional
fn arch_w(p: &Panel, t: usize) -> f64 { /// Newton on the weighted log-likelihood, coefficients fixed).
static W: std::sync::OnceLock<f64> = std::sync::OnceLock::new(); fn intercept_shift(d: &Design, b: &[f64], keep: &[bool]) -> f64 {
if t >= p.live0 { 1.0 } else { *W.get_or_init(|| std::env::var("AIRPLAY_ARCH_W").ok().and_then(|v| v.parse().ok()).unwrap_or(1.0)) } let rows: Vec<(f64, f64, f64)> = (0..d.n()).filter(|&i| keep[i]).map(|i| {
let x: Vec<f64> = d.x[i * d.p..(i + 1) * d.p].iter().map(|v| *v as f64).collect();
(glm::dot(b, &x), d.y[i] as f64, d.w[i] as f64)
}).collect();
if rows.iter().filter(|r| r.1 > 0.5).count() < 5 { return 0.0; }
let mut s = 0.0;
for _ in 0..30 {
let (mut g, mut h) = (0.0, 0.0);
for (e, y, w) in &rows { let m = glm::sigmoid(e + s); g += w * (y - m); h += w * m * (1.0 - m); }
if h <= 0.0 { break; }
let step = (g / h).clamp(-2.0, 2.0);
s += step;
if step.abs() < 1e-6 { break; }
}
s
}
/// Training weight of an archive-era row (live rows always 1). The release-event model uses the
/// archive at 0.3 (AIRPLAY_ARCH_W), the general add model too (AIRPLAY_ARCH_W_ALL): after the
/// boundary fix both predict the coming period better than one year alone. See docs/HISTORY.md.
fn arch_w(p: &Panel, t: usize, events: bool) -> f64 {
static WE: std::sync::OnceLock<f64> = std::sync::OnceLock::new();
static WA: std::sync::OnceLock<f64> = std::sync::OnceLock::new();
if t >= p.live0 { return 1.0; }
if events { *WE.get_or_init(|| std::env::var("AIRPLAY_ARCH_W").ok().and_then(|v| v.parse().ok()).unwrap_or(0.3)) }
else { *WA.get_or_init(|| std::env::var("AIRPLAY_ARCH_W_ALL").ok().and_then(|v| v.parse().ok()).unwrap_or(0.3)) }
} }
/// Recent origins used to fit the recalibration. /// Recent origins used to fit the recalibration.
@@ -533,24 +561,40 @@ impl AddModel {
fn fit_raw(p: &Panel, r: &Rot, h: usize, last_week: usize, events: bool, win: usize) -> AddModel { fn fit_raw(p: &Panel, r: &Rot, h: usize, last_week: usize, events: bool, win: usize) -> AddModel {
let (mut dc, mut da) = (Design::new(PC), Design::new(PA)); let (mut dc, mut da) = (Design::new(PC), Design::new(PA));
let (mut x, mut xa) = (vec![0.0; PC], vec![0.0; PA]); let (mut x, mut xa) = (vec![0.0; PC], vec![0.0; PA]);
let (mut live_c, mut live_a): (Vec<bool>, Vec<bool>) = (Vec::new(), Vec::new());
let sc = Scenario::actual(win); let sc = Scenario::actual(win);
for t in origins_w(h, last_week, win) { for t in origins_w(h, last_week, win) {
for a in 0..p.a() { for a in 0..p.a() {
// Only artists knowable at the origin (no survivorship from later discovery). // Only artists knowable at the origin (no survivorship from later discovery).
if !p.known(a, t) { continue; } if !p.known(a, t) { continue; }
if events && !clean_release(p, a, t + 1) { continue; } if events && !clean_release(p, a, t + 1) { continue; }
// Archive rows: a deterministic random share = their weight, kept at weight 1. Same
// expected fit as weighting every row, at a fraction of the memory.
let aw = arch_w(p, t, events);
if aw == 0.0 { continue; }
if aw < 1.0 && (mix((a as u64) << 24 ^ t as u64 ^ 0xa4c) % 1000) as f64 >= aw * 1000.0 { continue; }
for c in 0..C { for c in 0..C {
// Channels where the artist hasn't been heard for 12 weeks: keep a quarter, weighted 4. // Channels where the artist hasn't been heard for 12 weeks: keep a quarter, weighted 4.
let quiet = (0..LAGS).all(|i| p.y(a, c, t - i) == 0.0); let quiet = (0..LAGS).all(|i| p.y(a, c, t - i) == 0.0);
let wgt = if quiet && !events { if mix((a as u64) << 32 ^ (c as u64) << 20 ^ t as u64 ^ 0xadd) % 4 != 0 { continue; } 4.0 } else { 1.0 }; let wgt = if quiet && !events { if mix((a as u64) << 32 ^ (c as u64) << 20 ^ t as u64 ^ 0xadd) % 4 != 0 { continue; } 4.0 } else { 1.0 };
row_channel(p, r, a, c, t, h, sc, &mut x); row_channel(p, r, a, c, t, h, sc, &mut x);
dc.push(&x, r.add_in(a, c, t, h, win) as i32 as f64, wgt * arch_w(p, t)); dc.push(&x, r.add_in(a, c, t, h, win) as i32 as f64, wgt);
live_c.push(t >= p.live0);
} }
row_any(p, r, a, t, h, sc, &mut xa); row_any(p, r, a, t, h, sc, &mut xa);
da.push(&xa, r.add_any_in(a, t, h, win) as i32 as f64, arch_w(p, t)); da.push(&xa, r.add_any_in(a, t, h, win) as i32 as f64, 1.0);
live_a.push(t >= p.live0);
} }
} }
AddModel { h, win, channel: glm::logit(&dc, 1.0), any: glm::logit(&da, 1.0), platt_channel: (0.0, 1.0), platt_any: (0.0, 1.0), calib_channel: None, calib_any: None } let (mut bc, mut ba) = (glm::logit(&dc, 1.0), glm::logit(&da, 1.0));
// Train on everything, set the level on the recent period: with history (live0 > 0) and
// AIRPLAY_RECENT_CAL=1, the intercept is re-fitted on the live-era training rows only, so
// the archive shapes every coefficient while today's base rate sets the level.
if p.live0 > 0 && std::env::var("AIRPLAY_RECENT_CAL").is_ok_and(|v| v == "1") {
bc[0] += intercept_shift(&dc, &bc, &live_c);
ba[0] += intercept_shift(&da, &ba, &live_a);
}
AddModel { h, win, channel: bc, any: ba, platt_channel: (0.0, 1.0), platt_any: (0.0, 1.0), calib_channel: None, calib_any: None }
} }
/// (P(add) per channel, P(add on any channel)) from origin t. /// (P(add) per channel, P(add on any channel)) from origin t.
@@ -592,7 +636,11 @@ impl AddModel {
let p0 = glm::sigmoid(glm::dot(&self.any, &xa)).clamp(1e-6, 1.0 - 1e-6); let p0 = glm::sigmoid(glm::dot(&self.any, &xa)).clamp(1e-6, 1.0 - 1e-6);
if raw { return (pc, p0); } if raw { return (pc, p0); }
let pa = self.calib_any.as_ref().map(|k| k.apply(p0)).unwrap_or_else(|| platt(self.platt_any, p0)); let pa = self.calib_any.as_ref().map(|k| k.apply(p0)).unwrap_or_else(|| platt(self.platt_any, p0));
(pc, pa.min(cap.max(1e-6))) // ...and (AIRPLAY_ANY_FLOOR=1) at least the best single station group: an add on one channel
// is an add on some channel. Off: the per-channel numbers are overconfident above ~0.2, and
// the floor made "any" worse (log loss 0.0664 -> 0.0690) until they are recalibrated.
let floor = if std::env::var("AIRPLAY_ANY_FLOOR").is_ok_and(|v| v == "1") { best.iter().copied().fold(0.0, f64::max) } else { 0.0 };
(pc, pa.max(floor).min(cap.max(1e-6)))
} }
} }
@@ -889,7 +937,7 @@ pub fn backtest_events(p: &Panel, tau: &[u32; C], start: usize, win: usize, md:
s_a.add(&[by_rot[on][0] / by_rot[on][1], by_lvl[lvl(level(a, t0))][0] / by_lvl[lvl(level(a, t0))][1], tma[0] / tma[1], s_a.add(&[by_rot[on][0] / by_rot[on][1], by_lvl[lvl(level(a, t0))][0] / by_lvl[lvl(level(a, t0))][1], tma[0] / tma[1],
q * pan + (1.0 - q) * par, raw, pl_a.apply(raw), iso_a.apply(raw), pax], y); q * pan + (1.0 - q) * par, raw, pl_a.apply(raw), iso_a.apply(raw), pax], y);
pool_a.0.push((t0, raw, y)); pool_a.0.push((t0, raw, y));
if quiet(&r, a, t0) { if p.fav_weeks[a] >= 4.0 { favs.push((raw, iso_a.apply(raw), y)); } else { quiet_other.push((raw, iso_a.apply(raw), y)); } } if quiet(&r, a, t0) { if p.fav_before(a, t0) >= 4 { favs.push((raw, iso_a.apply(raw), y)); } else { quiet_other.push((raw, iso_a.apply(raw), y)); } }
for c in 0..C { for c in 0..C {
let yc = r.add_in(a, c, t0, 0, win); let yc = r.add_in(a, c, t0, 0, win);
let onc = r.on(a, c, t0) as usize; let onc = r.on(a, c, t0) as usize;
+24 -12
View File
@@ -102,7 +102,7 @@ pub struct Panel {
/// in at least two different years (2019-). Their December re-entries aren't new adds. /// in at least two different years (2019-). Their December re-entries aren't new adds.
pub xmas: Vec<bool>, pub xmas: Vec<bool>,
/// Weeks with 20+ DR plays (all channels) on the archived artist pages, 2019-24 (before the panel). /// Weeks with 20+ DR plays (all channels) on the archived artist pages, 2019-24 (before the panel).
pub fav_weeks: Vec<f64>, pub fav_weeks: Vec<Vec<NaiveDate>>,
/// Season index per ISO week (1..=53): new songs reaching the Airplay Top-20 that week of the /// Season index per ISO week (1..=53): new songs reaching the Airplay Top-20 that week of the
/// year, relative to an average week, 2002–2024 (before the DR panel), smoothed over 5 weeks. /// year, relative to an average week, 2002–2024 (before the DR panel), smoothed over 5 weeks.
pub season: [f64; 54], pub season: [f64; 54],
@@ -120,13 +120,23 @@ impl Panel {
#[inline] #[inline]
pub fn known(&self, a: usize, t: usize) -> bool { self.first_known[a] <= t && self.observed_around(a, t) } pub fn known(&self, a: usize, t: usize) -> bool { self.first_known[a] <= t && self.observed_around(a, t) }
/// Is the artist observed in the 20 weeks to t (or since the panel began) and the 15 weeks /// Is the artist observed in the 12 weeks to t (the lags; as the one-year panel's first origin)
/// after it (or to the panel's end)? Always true in the live weeks; in the archive weeks it keeps /// and the 15 weeks after it (or to the panel's end)? Unobserved archive weeks must not count as
/// unobserved weeks from counting as silence, in features and in targets. /// silence, in features or in targets.
#[inline] #[inline]
pub fn observed_around(&self, a: usize, t: usize) -> bool { pub fn observed_around(&self, a: usize, t: usize) -> bool {
let i = a * self.weeks.len() + t; let i = a * self.weeks.len() + t;
self.back[i] as usize >= 20.min(t + 1) && self.fwd[i] as usize >= 15.min(self.weeks.len() - t) self.back[i] as usize >= 12 && self.fwd[i] as usize >= 15.min(self.weeks.len() - t)
}
/// The first week of the observed run that contains week w (0 in the one-year panel): weeks
/// before it are unknown, so history-based labels and features must not reach past it.
#[inline]
pub fn run_start(&self, a: usize, w: usize) -> usize { (w + 1).saturating_sub(self.back[a * self.weeks.len() + w] as usize) }
/// Former-favourite weeks (20+ DR plays, old pages 2019-24) that ended at least 26 weeks before
/// week t: the same count for today's rows, and no look-ahead for archive rows.
pub fn fav_before(&self, a: usize, t: usize) -> usize {
let cut = self.weeks[t] - Duration::weeks(26);
self.fav_weeks[a].partition_point(|d| *d < cut)
} }
#[inline] #[inline]
pub fn observed(&self, a: usize, w: usize) -> bool { self.back[a * self.weeks.len() + w] > 0 } pub fn observed(&self, a: usize, w: usize) -> bool { self.back[a * self.weeks.len() + w] > 0 }
@@ -251,7 +261,7 @@ impl Panel {
let (alltime, genre_fit) = load_affinity(db, &id_of, artists.len())?; let (alltime, genre_fit) = load_affinity(db, &id_of, artists.len())?;
let mut p = Panel { weeks, artists, y, chart, list_tau, releases, archive_rate, listed_on, nmf_dk, season, alltime, genre_fit, first_known: Vec::new(), fav_weeks: Vec::new(), let mut p = Panel { weeks, artists, y, chart, list_tau, releases, archive_rate, listed_on, nmf_dk, season, alltime, genre_fit, first_known: Vec::new(), fav_weeks: Vec::new(),
live0, back, fwd, xmas: Vec::new() }; live0, back, fwd, xmas: Vec::new() };
p.fav_weeks = load_fav_weeks(db, &id_of, p.a(), p.weeks[0])?; p.fav_weeks = load_fav_weeks(db, &id_of, p.a())?;
p.xmas = load_xmas(db, &id_of, &p)?; p.xmas = load_xmas(db, &id_of, &p)?;
p.first_known = first_known(db, &p, &id_of)?; p.first_known = first_known(db, &p, &id_of)?;
Ok(p) Ok(p)
@@ -306,15 +316,17 @@ fn load_xmas(db: &Db, id_of: &HashMap<String, usize>, p: &Panel) -> Result<Vec<b
}).collect()) }).collect())
} }
fn load_fav_weeks(db: &Db, id_of: &HashMap<String, usize>, n: usize, start: NaiveDate) -> Result<Vec<f64>> { /// The weeks with 20+ DR plays on the old artist pages (2019-24), per artist, sorted.
let mut out = vec![0.0; n]; fn load_fav_weeks(db: &Db, id_of: &HashMap<String, usize>, n: usize) -> Result<Vec<Vec<NaiveDate>>> {
let mut out = vec![Vec::new(); n];
let has: i64 = db.conn.query_row("SELECT count(*) FROM sqlite_master WHERE name = 'dr_artist_history'", [], |r| r.get(0))?; let has: i64 = db.conn.query_row("SELECT count(*) FROM sqlite_master WHERE name = 'dr_artist_history'", [], |r| r.get(0))?;
if has == 0 { return Ok(out); } if has == 0 { return Ok(out); }
for r in db.conn.prepare("SELECT artist_id, count(*) FROM dr_artist_history WHERE plays >= 20 AND week_start < ?1 GROUP BY artist_id")? for r in db.conn.prepare("SELECT artist_id, week_start FROM dr_artist_history WHERE plays >= 20")?
.query_map([start.to_string()], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?)))? { .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? {
let (id, k) = r?; let (id, w) = r?;
if let Some(&a) = id_of.get(&id) { out[a] = k as f64; } if let (Some(&a), Ok(d)) = (id_of.get(&id), NaiveDate::parse_from_str(&w, "%Y-%m-%d")) { out[a].push(d); }
} }
for v in out.iter_mut() { v.sort(); }
Ok(out) Ok(out)
} }