From 5971fec9a28dfbde7c7bb5402c51f49c8ecdf21a Mon Sep 17 00:00:00 2001 From: moritsbw-cpu Date: Fri, 20 Nov 2026 13:52:29 +0000 Subject: [PATCH] Next week page: relative certainty (vs an average artist), tail-corrected 1-week chances, track record shows only bands with 30+ forecasts Co-Authored-By: Claude Opus 5.5 --- src/model/adds.rs | 25 +++++++++++++++++------ src/model/calib.rs | 3 +++ src/model/rank.rs | 7 +++++-- src/web/index.html | 49 +++++++++++++++++++++++++++------------------- 4 files changed, 56 insertions(+), 28 deletions(-) diff --git a/src/model/adds.rs b/src/model/adds.rs index a29f514..87dc179 100644 --- a/src/model/adds.rs +++ b/src/model/adds.rs @@ -1059,6 +1059,19 @@ pub fn backtest_events(p: &Panel, tau: &[u32; C], start: usize, win: usize, md: Ok(rows) } +/// Above TAIL_K, a 1-week chance is only as far above it as `s` times the model's distance (one +/// factor, fitted by log-likelihood on forecasts above TAIL_K). The raw 1-week chances were right +/// below 5% and overconfident above it (13% predicted, 5.6% happened); Platt and isotonic maps, +/// dominated by the low range or overfitting the few high cases, did not fix it out of sample. +pub const TAIL_K: f64 = 0.05; +fn tail_shrink(pairs: &[(f64, bool)]) -> Calib { + let tail: Vec<&(f64, bool)> = pairs.iter().filter(|x| x.0 > TAIL_K).collect(); + if tail.iter().filter(|x| x.1).count() < 10 { return Calib::Identity; } + let loss = |s: f64| tail.iter().map(|(q, y)| ll((TAIL_K + (q - TAIL_K) * s).clamp(1e-6, 1.0 - 1e-6), *y)).sum::(); + let best = (0..=100).map(|i| i as f64 / 100.0).min_by(|a, b| loss(*a).total_cmp(&loss(*b))).unwrap_or(1.0); + Calib::Tail(TAIL_K, best) +} + /// Bins for the "next week" track record (upper edges). pub const NEXT_BINS: [f64; 7] = [0.05, 0.1, 0.2, 0.3, 0.5, 0.7, 1.0]; @@ -1076,7 +1089,7 @@ pub fn backtest_next(db: &crate::db::Db, p: &Panel, tau: &[u32; C], start: usize let mut origins = 0; let fit = |pool: &[(usize, f64, bool)], t0: usize| -> Calib { let pairs: Vec<(f64, bool)> = pool.iter().filter(|(o, _, _)| o + h + 2 <= t0).map(|(_, q, y)| (*q, *y)).collect(); - Calib::isotonic(&pairs) + tail_shrink(&pairs) }; for t0 in start..=last { if (t0 - start) % 2 == 0 || m.is_none() { m = Some(AddModel::fit_win(p, &r, h, t0, 1)); } @@ -1100,16 +1113,16 @@ pub fn backtest_next(db: &crate::db::Db, p: &Panel, tau: &[u32; C], start: usize eprintln!("next-week backtest: origin {} done", p.weeks[t0]); } // The maps the daily run applies: isotonic on every forecast in the test. - let all = |pool: &[(usize, f64, bool)]| Calib::isotonic(&pool.iter().map(|(_, q, y)| (*q, *y)).collect::>()); + let all = |pool: &[(usize, f64, bool)]| tail_shrink(&pool.iter().map(|(_, q, y)| (*q, *y)).collect::>()); db.conn.execute_batch("CREATE TABLE IF NOT EXISTS calibration_maps (name TEXT PRIMARY KEY, knots TEXT NOT NULL, made_at TEXT NOT NULL);")?; for (name, cal) in [("next_any", all(&pool_any)), ("next_ch", all(&pool_ch))] { - if let Calib::Iso(k) = cal { - db.conn.execute("INSERT OR REPLACE INTO calibration_maps VALUES (?1, ?2, ?3)", rusqlite::params![name, serde_json::to_string(&k)?, chrono::Utc::now().to_rfc3339()])?; - } + let v = match cal { Calib::Iso(k) => serde_json::json!({"iso": k}), Calib::Platt(a, b) => serde_json::json!({"platt": [a, b]}), + Calib::Tail(k, s) => serde_json::json!({"tail": [k, s]}), Calib::Identity => serde_json::json!({}) }; + db.conn.execute("INSERT OR REPLACE INTO calibration_maps VALUES (?1, ?2, ?3)", rusqlite::params![name, v.to_string(), chrono::Utc::now().to_rfc3339()])?; } let mut rows = Vec::new(); writeln!(md, "# Backtest: who is added next week\n")?; - writeln!(md, "A 1-week window starting {} weeks after the latest complete DR week (the live page's lag), {origins} forecast weeks from {}. Release chance blended in as on the site; channels capped at the any-channel chance. \"Calibrated\" = an isotonic map refitted each week on earlier weeks' forecasts only (what the site would have shown); the final map is stored for the daily run.\n", h + 1, p.weeks[start])?; + writeln!(md, "A 1-week window starting {} weeks after the latest complete DR week (the live page's lag), {origins} forecast weeks from {}. Release chance blended in as on the site; channels capped at the any-channel chance. \"Calibrated\" = above 5%, the distance from 5% shrunk by one factor fitted on earlier weeks' forecasts only (refitted each week) on earlier weeks' forecasts only (what the site would have shown); the final map is stored for the daily run.\n", h + 1, p.weeks[start])?; for (label, v) in [("raw_any", &raw_any), ("any", &cal_any), ("ch", &cal_ch)] { let n = v.len() as f64; let rate = v.iter().filter(|x| x.1).count() as f64 / n.max(1.0); diff --git a/src/model/calib.rs b/src/model/calib.rs index 10556e5..7f69d43 100644 --- a/src/model/calib.rs +++ b/src/model/calib.rs @@ -10,6 +10,8 @@ pub enum Calib { Platt(f64, f64), /// Knots (predicted, observed), increasing in both. Iso(Vec<(f64, f64)>), + /// Above k, the distance from k times s (the next-week page's tail correction). + Tail(f64, f64), } fn logit(p: f64) -> f64 { let p = p.clamp(1e-6, 1.0 - 1e-6); (p / (1.0 - p)).ln() } @@ -19,6 +21,7 @@ impl Calib { let out = match self { Calib::Identity => p, Calib::Platt(a, b) => glm::sigmoid(a + b * logit(p)), + Calib::Tail(k, s) => if p > *k { k + (p - k) * s } else { p }, Calib::Iso(k) => { if k.is_empty() { p } else if p <= k[0].0 { k[0].1 } diff --git a/src/model/rank.rs b/src/model/rank.rs index bd9881c..e2c5541 100644 --- a/src/model/rank.rs +++ b/src/model/rank.rs @@ -167,10 +167,13 @@ pub fn run(db: &Db, p: &Panel, max_h: usize, show_h: usize) -> Result> let next_monday = today + Duration::days(7 - chrono::Datelike::weekday(&today).num_days_from_monday() as i64); let h_next = ((next_monday - origin).num_days() / 7 - 1).max(0) as usize; let m = AddModel::fit_win(p, &rot, h_next, t0, 1); - // The isotonic maps from `airplay model nextweek` (the raw 1-week chances were overconfident above ~10%). + // The tail correction from `airplay model nextweek` (the raw 1-week chances were overconfident above 5%). let map = |name: &str| -> Option { let k: String = db.conn.query_row("SELECT knots FROM calibration_maps WHERE name = ?1", [name], |r| r.get(0)).ok()?; - Some(super::calib::Calib::Iso(serde_json::from_str(&k).ok()?)) + let v: serde_json::Value = serde_json::from_str(&k).ok()?; + if let Some(ks) = v["tail"].as_array() { return Some(super::calib::Calib::Tail(ks.first()?.as_f64()?, ks.get(1)?.as_f64()?)); } + if let Some(ab) = v["platt"].as_array() { return Some(super::calib::Calib::Platt(ab.first()?.as_f64()?, ab.get(1)?.as_f64()?)); } + Some(super::calib::Calib::Iso(serde_json::from_value(v["iso"].clone()).ok()?)) }; let (cal_any, cal_ch) = (map("next_any"), map("next_ch")); let tx = db.conn.unchecked_transaction()?; diff --git a/src/web/index.html b/src/web/index.html index 409b9e6..e405a15 100644 --- a/src/web/index.html +++ b/src/web/index.html @@ -131,10 +131,10 @@ svg text { fill: var(--muted); font-size: 11px; } .cert.c1 { background: color-mix(in srgb, var(--good) 12%, var(--panel)); color: var(--good); } .cert.c2 { background: var(--accent-soft); color: var(--accent); } .cert.c3, .cert.c4 { background: var(--bg); color: var(--muted); } -.dots { display: flex; align-items: center; gap: 4px; } -.dots span { width: 11px; height: 11px; border-radius: 50%; border: 1.5px solid var(--accent); } -.dots span.f { background: var(--accent); } -.dots em { font-style: normal; color: var(--muted); font-size: 13px; margin-left: 8px; } +.rel { display: grid; grid-template-columns: 1fr auto; gap: 4px 10px; align-items: center; font-size: 13px; color: var(--muted); } +.rel .track2 { height: 10px; border-radius: 5px; background: var(--line); overflow: hidden; } +.rel .track2 i { display: block; height: 100%; background: var(--accent); border-radius: 5px; } +.rel b { color: var(--ink); } .chs { display: flex; flex-wrap: wrap; gap: 6px; } .chs span { font-size: 12px; padding: 2px 8px; border-radius: 6px; background: color-mix(in srgb, var(--c) 14%, var(--panel)); color: var(--ink); } .live { background: var(--bg); border-radius: 10px; padding: 10px 12px; font-size: 13px; } @@ -498,7 +498,10 @@ function renderModel() { // ---------- next week ---------- let nw = null, nwFilter = 'all', nwGroup = 'any'; const GROUP_C = { P3: 'var(--p3)', P4: 'var(--p4)', P5: 'var(--p5)', P6: 'var(--p6)', P8: 'var(--p8)' }; -const certainty = p => p >= 0.7 ? [0, 'Very likely'] : p >= 0.4 ? [1, 'Likely'] : p >= 0.2 ? [2, 'Possible'] : p >= 0.1 ? [3, 'Long shot'] : [4, 'Outside chance']; +// A single week is a narrow window (an average artist: about 0.3%), so certainty is shown against +// that: how many times an average artist's chance. +let nwBase = 0.003; +const certainty = p => { const m = p / nwBase; return m >= 25 ? [0, 'Top pick'] : m >= 15 ? [1, 'Strong'] : m >= 8 ? [2, 'Contender'] : [3, 'Outside chance']; }; const oneIn = p => p >= 0.5 ? `${Math.round(p * 10)} in 10` : `1 in ${Math.max(2, Math.round(1 / p))}`; const fmtLong = s => new Date(s + 'T00:00:00Z').toLocaleDateString('en-GB', { day: 'numeric', month: 'long', timeZone: 'UTC' }); const plusDays = (s, d) => { const x = new Date(s + 'T00:00:00Z'); x.setUTCDate(x.getUTCDate() + d); return x.toISOString().slice(0, 10); }; @@ -522,21 +525,25 @@ function renderNext() { const end = plusDays(d.week, 6), t = d.track || {}; // Track record: bins of the walk-forward test of exactly this forecast. const bins = []; let lo = 0; - d.bins.forEach((hi, i) => { if (t[`any_bin${i}_n`]) bins.push({ lo, hi, n: t[`any_bin${i}_n`], pred: t[`any_bin${i}_pred`], obs: t[`any_bin${i}_obs`] }); lo = hi; }); - const showcase = bins.filter(b => b.lo >= 0.2).sort((a, b) => b.n - a.n)[0] || bins[bins.length - 1]; - const veryLikely = d.list.filter(x => x.p >= 0.7).length, likely = d.list.filter(x => x.p >= 0.4).length; + // Bands with fewer than 30 past forecasts say nothing yet; left out. + d.bins.forEach((hi, i) => { if (t[`any_bin${i}_n`] >= 30) bins.push({ lo, hi, n: t[`any_bin${i}_n`], pred: t[`any_bin${i}_pred`], obs: t[`any_bin${i}_obs`] }); lo = hi; }); + const pct1 = p => (p * 100).toFixed(p < 0.1 ? 1 : 0) + '%'; + // The track record shown up front: the highest band with enough forecasts to mean something. + const showcase = bins.filter(b => b.n >= 200 && b.lo > 0).sort((a, b) => b.lo - a.lo)[0] || bins[0]; + nwBase = t.any_rate || 0.003; + const top10 = d.list.slice(0, 10).reduce((s, x) => s + x.p, 0), pmax = Math.max(...d.list.map(x => x.p), 0.01); const groups = ['P3', 'P4', 'P5', 'P6', 'P8']; const enrich = x => { const gs = Object.entries(x.groups).map(([g, v]) => ({ group: g, ...v })).sort((a, b) => b.p - a.p); return { ...x, gs, best: gs[0] }; }; let list = d.list.map(enrich); if (nwFilter === 'new') list = list.filter(x => !x.seasonal); else if (nwFilter === 'seasonal') list = list.filter(x => x.seasonal); if (nwGroup !== 'any') list = list.filter(x => x.groups[nwGroup]).sort((a, b) => b.groups[nwGroup].p - a.groups[nwGroup].p); const card = x => { - const p = nwGroup === 'any' ? x.p : x.groups[nwGroup].p, [ci, cl] = certainty(p), filled = Math.round(p * 10); + const p = nwGroup === 'any' ? x.p : x.groups[nwGroup].p, [ci, cl] = certainty(p); const g = nwGroup === 'any' ? x.best : { group: nwGroup, ...x.groups[nwGroup] }; return `
${esc(x.name)}${x.danish ? 'Danish' : 'International'}${x.seasonal ? ' · Seasonal return' : ''}
${pct(p)}
${cl}
-
${Array.from({ length: 10 }, (_, i) => ``).join('')}${oneIn(p)}
+
${oneIn(p)} · ${Math.round(p / nwBase)}× an average artist
${x.gs.filter(v => v.p >= 0.01).map(v => `${v.group} ${pct(v.p)}`).join('') || 'no single station stands out'}
${liveLine(g)} ${x.song ? `
Playing now: ${esc(x.song.title)} (${x.song.plays} plays on DR this week)
` : ''} @@ -544,15 +551,17 @@ function renderNext() { }; const chart = () => { if (!bins.length) return '

No track record stored yet (`airplay model nextweek`).

'; - const W = 320, H = 240, P = 36, X = v => P + v * (W - P - 10), Y = v => H - P - v * (H - P - 10); + // Zoomed to the range the forecasts use (a single week: mostly under 15%). + const top = Math.min(1, Math.max(0.15, ...bins.map(b => Math.max(b.pred, b.obs) * 1.25))); + const W = 320, H = 240, P = 44, X = v => P + v / top * (W - P - 10), Y = v => H - P - v / top * (H - P - 10); const maxn = Math.max(...bins.map(b => b.n)); return ` - - - ${[0, .25, .5, .75, 1].map(v => `${v * 100}%${v * 100}%`).join('')} + + + ${[0, .25, .5, .75, 1].map(f => f * top).map(v => `${Math.round(v * 100)}%${Math.round(v * 100)}%`).join('')} we predicted - it happened - ${bins.map(b => `${Math.round(b.lo * 100)}-${Math.round(b.hi * 100)}%: predicted ${pct(b.pred)}, happened ${pct(b.obs)} (${b.n} forecasts)`).join('')} + it happened + ${bins.filter(b => b.pred <= top && b.obs <= top).map(b => `${Math.round(b.lo * 100)}-${Math.round(b.hi * 100)}%: predicted ${pct1(b.pred)}, happened ${pct1(b.obs)} (${b.n} forecasts)`).join('')} `; }; $('nw').innerHTML = ` @@ -562,8 +571,8 @@ function renderNext() {

The chance each artist enters rotation on a DR music channel that week: a new song, or plays jumping well above their usual level. Forecast made ${fmtLong(d.made_at.slice(0, 10))} from DR's artist pages up to the week of ${fmtLong(d.origin_week)}; the live lines use every DR play logged since ${fmtLong(plusDays(d.this_monday, -7))}.

Expected adds on DR
${Math.round(d.expected)}90% range ${Math.round(d.range[0])}-${Math.round(d.range[1])}
-
Likely or better
${likely}${veryLikely} very likely (70%+)
-
Track record
${showcase ? pct(showcase.obs) : '—'}${showcase ? `happened when we said ${Math.round(showcase.lo * 100)}-${Math.round(showcase.hi * 100)}%` : 'not tested yet'}
+
From our top 10
${top10.toFixed(1)}adds expected among the ten strongest candidates
+
Track record
${showcase ? (showcase.obs * 100).toFixed(1) + '%' : '—'}${showcase ? `came true when we said ${Math.round(showcase.lo * 100)}-${Math.round(showcase.hi * 100)}% (${Math.round(showcase.n).toLocaleString('en')} forecasts)` : 'not tested yet'}
Add board: most likely per station
@@ -578,8 +587,8 @@ function renderNext() {
How sure are we?
${chart()}

Each dot is a group of past forecasts made exactly like this one: a 1-week window, ${d.horizon + 1} weeks after the latest complete week of DR data, scored on what DR actually did (walk-forward, ${t.any_origins ? Math.round(t.any_origins) + ' forecast weeks' : 'past year'}, nothing from the future used). On the dashed line the percentages mean what they say. Bigger dots hold more forecasts.

-

${bins.map(b => `${Math.round(b.lo * 100)}-${Math.round(b.hi * 100)}%: happened ${pct(b.obs)} of ${Math.round(b.n).toLocaleString('en')} times`).join(' · ')}

-

Very likely 70%+ · Likely 40-70% · Possible 20-40% · Long shot 10-20% · Outside chance under 10%. "1 in 4" means: of four artists with this chance, about one gets added. Adds of old songs (Christmas, classics) are marked Seasonal return when we can tell; the forecast itself does not yet separate them.

+

${bins.map(b => `We said ${Math.round(b.lo * 100)}-${Math.round(b.hi * 100)}% (on average ${pct1(b.pred)}): it happened ${pct1(b.obs)} of ${Math.round(b.n).toLocaleString('en')} times`).join(' · ')}. Higher bands have too few past forecasts to show yet.

+

One week is a narrow window: an average artist on DR's books has about ${(nwBase * 100).toFixed(1)}% chance of being added in any given week, and even the strongest candidates rarely pass 1 in 10. So we grade against that: Top pick 25× an average artist or more · Strong 15-25× · Contender 8-15× · Outside chance below. "1 in 10" means: of ten artists with this chance, about one gets added that week. Adds of old songs (Christmas, classics) are marked Seasonal return when we can tell; the forecast itself does not yet separate them.

${t.any_roc ? `

Ranking quality on the same test: an artist who got added was ranked above one who did not ${Math.round(t.any_roc * 100)}% of the time (ROC-AUC ${t.any_roc.toFixed(3)}).

` : ''}
`; document.querySelectorAll('#nw [data-f]').forEach(b => b.onclick = () => { nwFilter = b.dataset.f; renderNext(); }); document.querySelectorAll('#nw [data-g]').forEach(b => b.onclick = () => { nwGroup = b.dataset.g; renderNext(); });