Default window 2 weeks (1-week option added) on the site, best week and what-if; new-song backtest takes the window (--w, default 2)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-16 12:32:28 +00:00
co-authored by Claude Opus 5.5
parent 37b3029e02
commit 0e511e1e0b
6 changed files with 114 additions and 107 deletions
+19 -18
View File
@@ -419,7 +419,8 @@ impl AddModel {
/// within 8 weeks?" Trained only on artist-weeks where the artist did release in week t+1, with
/// the artist's state the week before (earlier releases and NMF placements count as history;
/// the new song's own NMF placement is the scenario variable).
pub fn fit_events(p: &Panel, r: &Rot, last_week: usize) -> AddModel { Self::fit_with(p, r, 0, last_week, true, W) }
/// As `fit_events`, for a window of `win` weeks (6-origin recalibration; used in backtests).
pub fn fit_events_win(p: &Panel, r: &Rot, last_week: usize, win: usize) -> AddModel { Self::fit_with(p, r, 0, last_week, true, win) }
/// The site's release-event model (8-week window, pooled recalibration).
pub fn fit_events_site(p: &Panel, r: &Rot, last_week: usize) -> AddModel { Self::fit_events_pooled(p, r, last_week, W) }
/// The release-event model for a window of `win` weeks.
@@ -771,29 +772,29 @@ pub fn backtest(p: &Panel, tau: &[u32; C], horizons: &[usize], start: usize, md:
/// Walk-forward test of the release-event model: at each origin t0 the model sees only events
/// whose window had closed (a week before t0), and scores the releases of week t0+1. Pre-release
/// rows blend the new song's NMF placement from the artist's history (as on the site).
pub fn backtest_events(p: &Panel, tau: &[u32; C], start: usize, md: &mut String) -> Result<Vec<(String, usize, String, f64)>> {
pub fn backtest_events(p: &Panel, tau: &[u32; C], start: usize, win: usize, md: &mut String) -> Result<Vec<(String, usize, String, f64)>> {
let r = Rot::new(p, tau);
let mut rows = Vec::new();
let (mut s_c, mut s_a) = (Scores::new(SCORED.to_vec()), Scores::new(SCORED.to_vec()));
let (mut pool_c, mut pool_a) = (Pool(Vec::new()), Pool(Vec::new()));
let lvl = |m: f64| ((m.ln_1p() / 0.5) as usize).min(7);
let level = |a: usize, t: usize| (0..LAGS).map(|i| p.group_total(a, t - i, None)).sum::<f64>() / LAGS as f64;
for t0 in start..p.w().saturating_sub(W + 1) {
let m = AddModel::fit_events(p, &r, t0);
for t0 in start..p.w().saturating_sub(win + 1) {
let m = AddModel::fit_events_win(p, &r, t0, win);
// Baselines from the training events. Release timing only: by weeks since the previous release.
let (mut by_rot, mut by_lvl, mut cr, mut cl) = ([[1.0f64, 2.0]; 2], [[1.0f64, 2.0]; 8], [[1.0f64, 2.0]; 2], [[1.0f64, 2.0]; 8]);
let mut tm = Timing::new();
for t in origins(0, t0) {
for t in origins_w(0, t0, win) {
for a in 0..p.a() {
if !p.known(a, t) || !clean_release(p, a, t + 1) { continue; }
let key = since_bin(p, a, t);
let y = r.add_any_in_window(a, t, 0) as i32 as f64;
let y = r.add_any_in(a, t, 0, win) as i32 as f64;
let on = r.any(a, t) as usize;
by_rot[on][0] += y; by_rot[on][1] += 1.0;
let b = lvl(level(a, t)); by_lvl[b][0] += y; by_lvl[b][1] += 1.0;
tm.any[key][0] += y; tm.any[key][1] += 1.0;
for c in 0..C {
let yc = r.add_in_window(a, c, t, 0) as i32 as f64;
let yc = r.add_in(a, c, t, 0, win) as i32 as f64;
let onc = r.on(a, c, t) as usize;
cr[onc][0] += yc; cr[onc][1] += 1.0;
let bc = lvl((0..LAGS).map(|i| p.y(a, c, t - i)).sum::<f64>() / LAGS as f64);
@@ -802,20 +803,20 @@ pub fn backtest_events(p: &Panel, tau: &[u32; C], start: usize, md: &mut String)
}
}
}
let gap = W + 1;
let gap = win + 1;
let ((pl_c, iso_c), (pl_a, iso_a)) = (pool_c.fit(t0, gap), pool_a.fit(t0, gap));
let mut started = false;
for a in 0..p.a() {
if !p.known(a, t0) || !clean_release(p, a, t0 + 1) { continue; }
if !started { s_c.start_origin(); s_a.start_origin(); started = true; }
let q = p_nmf(p, a, t0);
let (pcn, pan) = m.predict(p, &r, a, t0, Scenario::new(true, true), None);
let (pcr, par) = m.predict(p, &r, a, t0, Scenario::new(true, false), None);
let (rcn, ran) = m.predict_raw(p, &r, a, t0, 0, Scenario::new(true, true));
let (rcr, rar) = m.predict_raw(p, &r, a, t0, 0, Scenario::new(true, false));
let (pcx, pax) = m.predict(p, &r, a, t0, Scenario::ACTUAL, None);
let (pcn, pan) = m.predict(p, &r, a, t0, Scenario::new(true, true).window(win), None);
let (pcr, par) = m.predict(p, &r, a, t0, Scenario::new(true, false).window(win), None);
let (rcn, ran) = m.predict_raw(p, &r, a, t0, 0, Scenario::new(true, true).window(win));
let (rcr, rar) = m.predict_raw(p, &r, a, t0, 0, Scenario::new(true, false).window(win));
let (pcx, pax) = m.predict(p, &r, a, t0, Scenario::actual(win), None);
let key = since_bin(p, a, t0);
let y = r.add_any_in_window(a, t0, 0);
let y = r.add_any_in(a, t0, 0, win);
let on = r.any(a, t0) as usize;
let raw = q * ran + (1.0 - q) * rar;
let tma = tm.any[key];
@@ -823,7 +824,7 @@ pub fn backtest_events(p: &Panel, tau: &[u32; C], start: usize, md: &mut String)
q * pan + (1.0 - q) * par, raw, pl_a.apply(raw), iso_a.apply(raw), pax], y);
pool_a.0.push((t0, raw, y));
for c in 0..C {
let yc = r.add_in_window(a, c, t0, 0);
let yc = r.add_in(a, c, t0, 0, win);
let onc = r.on(a, c, t0) as usize;
let bc = lvl((0..LAGS).map(|i| p.y(a, c, t0 - i)).sum::<f64>() / LAGS as f64);
let rawc = q * rcn[c] + (1.0 - q) * rcr[c];
@@ -836,10 +837,10 @@ pub fn backtest_events(p: &Panel, tau: &[u32; C], start: usize, md: &mut String)
eprintln!("event backtest: origin {} done", p.weeks[t0]);
}
writeln!(md, "# Backtest: if the artist releases a new song, will DR add it?\n")?;
writeln!(md, "Events: an artist knowable at the time releases something (Deezer) in week t0+1, with no other release in the {CLEAN_WEEKS} weeks before (so an add can't be the previous song's); target: a DR add (entering rotation after {GAP}+ weeks out, or plays jumping well above the artist's own 8-week baseline and staying up) within the {W} weeks from the release week. The model knows the artist's state the week before; pre-release rows blend the new song's NMF Denmark placement from the artist's history. \"Release timing only\" = the add rate by weeks since the artist's previous release. {} release events scored, {} ({:.1}%) got an add on some DR channel.\n",
writeln!(md, "Events: an artist knowable at the time releases something (Deezer) in week t0+1, with no other release in the {CLEAN_WEEKS} weeks before (so an add can't be the previous song's); target: a DR add (entering rotation after {GAP}+ weeks out, or plays jumping well above the artist's own 8-week baseline and staying up) within {win} week(s) from the release week. The model knows the artist's state the week before; pre-release rows blend the new song's NMF Denmark placement from the artist's history. \"Release timing only\" = the add rate by weeks since the artist's previous release. {} release events scored, {} ({:.1}%) got an add on some DR channel.\n",
s_a.n, s_a.hits, 100.0 * s_a.hits / s_a.n.max(1.0))?;
table(md, &s_c, &s_a, &mut rows, 0, "event_")?;
let m = AddModel::fit_events(p, &r, p.w() - 1);
table(md, &s_c, &s_a, &mut rows, win, "event_")?;
let m = AddModel::fit_events_win(p, &r, p.w() - 1, win);
writeln!(md, "| Any-channel feature (all events) | Coefficient |\n|---|---|")?;
for (n, b) in ANY_NAMES.iter().chain(REL_NAMES).chain(WAVE_NAMES).chain(LONG_NAMES).chain(NMF_NAMES).chain(SEASON_NAMES).zip(&m.any) { writeln!(md, "| {n} | {b:+.3} |")?; }
Ok(rows)
+2 -1
View File
@@ -100,7 +100,8 @@ pub fn run(db: &Db, args: &[String]) -> Result<()> {
if let Some(r) = arg(args, "--rule") { let _ = adds::ADD_RULE.set(r); }
let tau = common::thresholds(&p, 26);
let mut md = String::new();
let rows = adds::backtest_events(&p, &tau, 26, &mut md)?;
let win: usize = arg(args, "--w").and_then(|w| w.parse().ok()).unwrap_or(2);
let rows = adds::backtest_events(&p, &tau, 26, win, &mut md)?;
rank::create_tables(db)?;
let at = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string();
for (m, h, metric, v) in &rows {
+2 -1
View File
@@ -116,7 +116,8 @@ pub fn run(db: &Db, p: &Panel, max_h: usize, show_h: usize) -> Result<Vec<Line>>
let rot = Rot::new(p, &tau);
// The release-event model ("if they release a new song, will DR add it?"), the same for every
// window start: the artist's state now, the new song's NMF placement blended from history.
let ev = AddModel::fit_events_site(p, &rot, p.w() - 1);
// The site's default: added within 2 weeks of the release.
let ev = AddModel::fit_events_pooled(p, &rot, p.w() - 1, 2);
let t0 = p.w() - 1;
// A new song released in week t0+h+1: time-since features moved forward to then; none if too
// soon after the artist's last release.
+8 -4
View File
@@ -26,7 +26,10 @@ use tiny_http::{Header, Method, Response, Server};
const PAGE: &str = include_str!("web/index.html");
const MAX_H: usize = 12;
/// Window lengths offered (weeks after the release within which DR must add the song).
const WINDOWS: [usize; 6] = [2, 4, 6, 8, 10, 12];
const WINDOWS: [usize; 7] = [1, 2, 4, 6, 8, 10, 12];
/// The default window: most adds that come quickly come within two weeks, and a short window
/// shows the difference between release weeks most clearly.
const DEFAULT_WINDOW: usize = 2;
/// Release weeks shown in the best-week calendar.
const CALENDAR: usize = 26;
@@ -174,7 +177,7 @@ fn handle(st: &State, method: &Method, url: &str) -> Result<Reply> {
"/" | "/index.html" => Ok((200, "text/html; charset=utf-8", Arc::new(PAGE.to_string()))),
"/api/meta" => meta(st),
"/api/rank" => rank(st, q.get("h").and_then(|h| h.parse().ok()).unwrap_or(0).min(CALENDAR - 1),
q.get("w").and_then(|w| w.parse().ok()).filter(|w| WINDOWS.contains(w)).unwrap_or(8)),
q.get("w").and_then(|w| w.parse().ok()).filter(|w| WINDOWS.contains(w)).unwrap_or(DEFAULT_WINDOW)),
"/api/bestweek" => bestweek(st, &q),
"/api/artist" => artist(st, q.get("id").map(String::as_str).unwrap_or("")),
"/api/whatif" => whatif(st, &q),
@@ -322,7 +325,7 @@ fn bestweek(st: &State, q: &HashMap<String, String>) -> Result<Reply> {
let Some(&a) = m.index.get(q.get("id").map(String::as_str).unwrap_or("")) else {
return Ok((404, "application/json", Arc::new(json!({"error": "artist not in the model"}).to_string())));
};
let win = q.get("w").and_then(|w| w.parse().ok()).filter(|w| WINDOWS.contains(w)).unwrap_or(8);
let win = q.get("w").and_then(|w| w.parse().ok()).filter(|w| WINDOWS.contains(w)).unwrap_or(DEFAULT_WINDOW);
let nmf = match q.get("nmf").map(String::as_str) { Some("yes") => Some(true), Some("no") => Some(false), _ => None };
let p = &m.panel;
let t0 = p.w() - 1;
@@ -449,7 +452,8 @@ fn whatif(st: &State, q: &HashMap<String, String>) -> Result<Reply> {
let run = |overrides: bool| -> Value {
let f: Option<&dyn Fn(&mut [f64], [usize; 4])> = if overrides { Some(&over) } else { None };
// A new release: the release-event model; no release: the window model (current songs only).
let (add_c, add_any) = if release { m.events.predict_at(p, &m.rot, a, t0, h, crate::model::adds::Scenario::new(true, nmf).at(h), f) }
let ev = m.events_w.get(&DEFAULT_WINDOW).unwrap_or(&m.events);
let (add_c, add_any) = if release { ev.predict_at(p, &m.rot, a, t0, h, crate::model::adds::Scenario::new(true, nmf).at(h), f) }
else { am.predict(p, &m.rot, a, t0, crate::model::adds::Scenario::new(false, false), f) };
let mut x = vec![0.0; P];
let mut mu = [0.0; C];
+15 -12
View File
@@ -125,7 +125,7 @@ svg text { fill: var(--muted); font-size: 11px; }
<div class="controls">
<div class="ctl">Release week <b id="hlabel">—</b><input type="range" id="h" min="0" max="25" value="0"></div>
<div class="ctl">Added within
<select id="w"><option value="2">2 weeks</option><option value="4">4 weeks</option><option value="6">6 weeks</option><option value="8" selected>8 weeks</option><option value="10">10 weeks</option><option value="12">12 weeks</option></select>
<select id="w"><option value="1">1 week</option><option value="2" selected>2 weeks</option><option value="4">4 weeks</option><option value="6">6 weeks</option><option value="8">8 weeks</option><option value="10">10 weeks</option><option value="12">12 weeks</option></select>
</div>
<div class="ctl">Station
<select id="ch">
@@ -170,7 +170,7 @@ svg text { fill: var(--muted); font-size: 11px; }
<div class="ctl">From <input type="date" id="bw-from"></div>
<div class="ctl">To <input type="date" id="bw-to"></div>
<div class="ctl">Added within
<select id="bw-w"><option value="2">2 weeks</option><option value="4">4 weeks</option><option value="6">6 weeks</option><option value="8" selected>8 weeks</option><option value="10">10 weeks</option><option value="12">12 weeks</option></select>
<select id="bw-w"><option value="1">1 week</option><option value="2" selected>2 weeks</option><option value="4">4 weeks</option><option value="6">6 weeks</option><option value="8">8 weeks</option><option value="10">10 weeks</option><option value="12">12 weeks</option></select>
</div>
<div class="ctl">New Music Friday Denmark
<select id="bw-nmf"><option value="auto">As often as their releases were placed</option><option value="yes">Placed</option><option value="no">Not placed</option></select>
@@ -420,7 +420,7 @@ function whatIfUI(a, h) {
const d = (x, y, f) => { const v = y - x; if (Math.abs(v) < 0.005 * (f === 'p' ? 1 : 100)) return ''; return `<span class="d ${v > 0 ? 'up' : 'down'}">${v > 0 ? '+' : ''}${f === 'p' ? Math.round(v * 100) + ' pts' : Math.round(v)}</span>`; };
const rows = [['Any DR station', w.base.add_any, w.scenario.add_any]].concat(GROUPS.map(([g, l]) => [l, chanP(w.base, g), chanP(w.scenario, g)]));
const onNow = Object.keys(w.current.in_rotation || {});
$('wi-out').innerHTML = `<p class="note">${w.release ? (w.too_soon ? 'Too soon after their last release for a new-song estimate this week; the numbers below are the model\'s extrapolation.' : `A new song released the week of ${fmtDate(w.window_start)}: chance DR adds it within 8 weeks.`) : `No new release: chance of a new add from current songs in the 8 weeks from ${fmtDate(w.window_start)}.`} In rotation now on ${onNow.length ? onNow.length + ' station(s)' : 'no station'}. Now: ${w.current.t40 ? 'Track Top-40 #' + w.current.t40 : 'not on the Track Top-40'}, ${w.current.air ? 'Airplay Top-20 #' + w.current.air : 'not on the Airplay Top-20'}, ${w.current.entry == null ? 'no chart entry on record' : 'latest new chart entry ' + w.current.entry + ' weeks ago'}.</p>
$('wi-out').innerHTML = `<p class="note">${w.release ? (w.too_soon ? 'Too soon after their last release for a new-song estimate this week; the numbers below are the model\'s extrapolation.' : `A new song released the week of ${fmtDate(w.window_start)}: chance DR adds it within 2 weeks.`) : `No new release: chance of a new add from current songs in the 8 weeks from ${fmtDate(w.window_start)}.`} In rotation now on ${onNow.length ? onNow.length + ' station(s)' : 'no station'}. Now: ${w.current.t40 ? 'Track Top-40 #' + w.current.t40 : 'not on the Track Top-40'}, ${w.current.air ? 'Airplay Top-20 #' + w.current.air : 'not on the Airplay Top-20'}, ${w.current.entry == null ? 'no chart entry on record' : 'latest new chart entry ' + w.current.entry + ' weeks ago'}.</p>
<div class="kpis">${rows.map(([l, b, s]) => `<div class="kpi"><div>${l}</div><b>${pct(s)}</b>${d(b, s, 'p')}<div>was ${pct(b)}</div></div>`).join('')}
<div class="kpi"><div>Plays, first week</div><b>${Math.round(w.scenario.mean)}</b>${d(w.base.mean, w.scenario.mean, 'n')}<div>was ${Math.round(w.base.mean)}</div></div></div>`;
};
@@ -430,17 +430,20 @@ function whatIfUI(a, h) {
// ---------- model tab ----------
function renderModel() {
const sc = meta.backtest || [];
const get = (m, k) => sc.find(x => x.model === m && x.metric === k)?.value;
const ms = [['rate by in-rotation-now', 'Rate by "in rotation now"'], ['rate by last-12-weeks level', 'Rate by last-12-weeks airplay'],
['event model (pre-release: NMF from history)', 'Model, before release (as on this site)'], ['event model (post-release: NMF known)', 'Model, NMF placement known']];
const sc = (meta.backtest || []).filter(x => x.metric.startsWith('event_'));
const win = sc.length ? Math.max(...sc.map(x => x.h)) : 2;
const get = (m, k) => sc.find(x => x.model === m && x.metric === k && x.h === win)?.value;
const ms = [['rate by in-rotation-now', 'Rate by "in rotation now"'], ['rate by last-12-weeks level', 'Rate by last-12-weeks airplay'], ['release timing only', 'Release timing only'],
['model + pooled isotonic (pre-release)', 'Model before release, isotonic (any station on this site)'], ['model + pooled Platt (pre-release)', 'Model before release, Platt (per station on this site)'],
['model, NMF known (post-release)', 'Model, NMF placement known']];
const f = (v, d) => v == null || isNaN(v) ? '—' : v.toFixed(d);
const sk = get('event model', 'event_skill_any_vs_level');
$('cards').innerHTML = sc.some(x => x.metric.startsWith('event_')) ? `<div class="card"><h3>If an artist releases a new song, does DR add it within 8 weeks?</h3>
<table class="wtable"><thead><tr><th class="l"></th><th>Any station<br>log-loss</th><th>Any station<br>PR-AUC</th><th>Per station<br>PR-AUC</th></tr></thead><tbody>
${ms.map(([m, l]) => `<tr><td class="l">${l}</td><td>${f(get(m, 'event_logloss_any'), 3)}</td><td>${f(get(m, 'event_prauc_any'), 3)}</td><td>${f(get(m, 'event_prauc_ch'), 3)}</td></tr>`).join('')}
const ska = get('model + pooled isotonic (pre-release)', 'event_skill_any_vs_level'), skc = get('model + pooled Platt (pre-release)', 'event_skill_ch_vs_level');
const sign = v => `<b class="${v > 0 ? 'up' : 'down'}">${v > 0 ? '+' : ''}${f(v, 3)}</b>`;
$('cards').innerHTML = sc.length ? `<div class="card"><h3>If an artist releases a new song, does DR add it within ${win} week${win > 1 ? 's' : ''}?</h3>
<table class="wtable"><thead><tr><th class="l"></th><th>Any station<br>log-loss</th><th>Any station<br>PR-AUC</th><th>Per station<br>log-loss</th><th>Per station<br>PR-AUC</th></tr></thead><tbody>
${ms.map(([m, l]) => `<tr><td class="l">${l}</td><td>${f(get(m, 'event_logloss_any'), 3)}</td><td>${f(get(m, 'event_prauc_any'), 3)}</td><td>${f(get(m, 'event_logloss_ch'), 4)}</td><td>${f(get(m, 'event_prauc_ch'), 3)}</td></tr>`).join('')}
</tbody></table>
<p>Walk-forward over the past year: each week's releases scored by a model fitted only on releases whose outcome was already known. Skill of the before-release model vs last-12-weeks airplay: <b class="${sk > 0 ? 'up' : 'down'}">${sk > 0 ? '+' : ''}${f(sk, 3)}</b> (any station). Lower log-loss and higher PR-AUC are better; a leak test confirms no feature uses data from after the forecast week.</p></div>`
<p>Walk-forward over the past year: each week's releases scored by a model fitted only on releases whose outcome was already known, artists included only from when they were knowable. Skill vs last-12-weeks airplay: ${sign(ska)} any station, ${sign(skc)} per station. Lower log-loss and higher PR-AUC are better; a leak test confirms no feature uses data from after the forecast week.</p></div>`
: '<p class="note">No backtest stored yet (`airplay model events`).</p>';
}