Release-event model: 'if the artist releases a new song, will DR add it' trained on clean releases (no other release in the 6 weeks before), with its own walk-forward test; ranking defaults to it and leaves out artists who just released; pool note updated
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+117
-5
@@ -72,6 +72,25 @@ fn window(p: &Panel, t: usize, h: usize) -> (chrono::NaiveDate, chrono::NaiveDat
|
||||
(w0, w0 + chrono::Duration::weeks(W as i64))
|
||||
}
|
||||
|
||||
/// Did the artist release something (Deezer discography) in DR week w (Monday to Sunday)?
|
||||
pub fn released_in_week(p: &Panel, a: usize, w: usize) -> bool {
|
||||
let Some(dates) = p.releases[a].as_ref() else { return false };
|
||||
let start = p.weeks[0] + chrono::Duration::weeks(w as i64);
|
||||
dates.iter().any(|d| *d >= start && *d < start + chrono::Duration::days(7))
|
||||
}
|
||||
|
||||
/// A clean release event in week w: a release that week and none in the 6 weeks before it, so an
|
||||
/// add that follows can't be the previous song's (DR's counts are per artist, not per song).
|
||||
pub const CLEAN_WEEKS: usize = 6;
|
||||
pub fn clean_release(p: &Panel, a: usize, w: usize) -> bool {
|
||||
released_in_week(p, a, w) && (w.saturating_sub(CLEAN_WEEKS)..w).all(|k| !released_in_week(p, a, k))
|
||||
}
|
||||
|
||||
/// Did the artist release something in the CLEAN_WEEKS weeks up to and including week t?
|
||||
pub fn recent_release(p: &Panel, a: usize, t: usize) -> bool {
|
||||
((t + 1).saturating_sub(CLEAN_WEEKS)..=t).any(|k| released_in_week(p, a, k))
|
||||
}
|
||||
|
||||
/// P(an artist's release is placed on NMF Denmark), from the past two years (shrunk).
|
||||
pub fn p_nmf(p: &Panel, a: usize, t: usize) -> f64 {
|
||||
let end_t = p.weeks[t] + chrono::Duration::days(6);
|
||||
@@ -280,17 +299,26 @@ impl AddModel {
|
||||
/// The logits on every window closed by `last_week`, then a recalibration: an inner model fitted
|
||||
/// on windows closed before the last CAL_ORIGINS origins predicts those origins, and a Platt fit
|
||||
/// maps its probabilities to what happened. (The raw fits were overconfident above ~0.4.)
|
||||
pub fn fit(p: &Panel, r: &Rot, h: usize, last_week: usize) -> AddModel {
|
||||
let mut m = Self::fit_raw(p, r, h, last_week);
|
||||
pub fn fit(p: &Panel, r: &Rot, h: usize, last_week: usize) -> AddModel { Self::fit_with(p, r, h, last_week, false) }
|
||||
|
||||
/// The release-event model: "if the artist releases a new song in week t+1, will DR add it
|
||||
/// within 8 weeks?" Trained only on artist-weeks where the artist did release in week t+1, with
|
||||
/// the artist's state the week before (earlier releases and NMF placements count as history;
|
||||
/// the new song's own NMF placement is the scenario variable).
|
||||
pub fn fit_events(p: &Panel, r: &Rot, last_week: usize) -> AddModel { Self::fit_with(p, r, 0, last_week, true) }
|
||||
|
||||
fn fit_with(p: &Panel, r: &Rot, h: usize, last_week: usize, events: bool) -> AddModel {
|
||||
let mut m = Self::fit_raw(p, r, h, last_week, events);
|
||||
let hi = last_week.saturating_sub(h + W);
|
||||
if hi + 1 >= CAL_ORIGINS + LAGS {
|
||||
let c0 = hi + 1 - CAL_ORIGINS;
|
||||
// the inner model may only use windows that closed before the first calibration origin
|
||||
if c0 >= LAGS - 1 + h + W + 4 {
|
||||
let inner = Self::fit_raw(p, r, h, c0);
|
||||
let inner = Self::fit_raw(p, r, h, c0, events);
|
||||
let (mut dc, mut da) = (Design::new(2), Design::new(2));
|
||||
for t in c0..=hi {
|
||||
for a in 0..p.a() {
|
||||
if events && !clean_release(p, a, t + 1) { continue; }
|
||||
let (pc, pa) = inner.predict(p, r, a, t, Scenario::ACTUAL, None);
|
||||
for c in 0..C {
|
||||
if pc[c] < 1e-4 && !r.add_in_window(a, c, t, h) { continue; } // certain zeros carry no information
|
||||
@@ -308,15 +336,16 @@ impl AddModel {
|
||||
m
|
||||
}
|
||||
|
||||
fn fit_raw(p: &Panel, r: &Rot, h: usize, last_week: usize) -> AddModel {
|
||||
fn fit_raw(p: &Panel, r: &Rot, h: usize, last_week: usize, events: bool) -> AddModel {
|
||||
let (mut dc, mut da) = (Design::new(PC), Design::new(PA));
|
||||
let (mut x, mut xa) = (vec![0.0; PC], vec![0.0; PA]);
|
||||
for t in origins(h, last_week) {
|
||||
for a in 0..p.a() {
|
||||
if events && !clean_release(p, a, t + 1) { continue; }
|
||||
for c in 0..C {
|
||||
// Channels where the artist hasn't been heard for 12 weeks: keep a quarter, weighted 4.
|
||||
let quiet = (0..LAGS).all(|i| p.y(a, c, t - i) == 0.0);
|
||||
let wgt = if quiet { if mix((a as u64) << 32 ^ (c as u64) << 20 ^ t as u64 ^ 0xadd) % 4 != 0 { continue; } 4.0 } else { 1.0 };
|
||||
let wgt = if quiet && !events { if mix((a as u64) << 32 ^ (c as u64) << 20 ^ t as u64 ^ 0xadd) % 4 != 0 { continue; } 4.0 } else { 1.0 };
|
||||
row_channel(p, r, a, c, t, h, Scenario::ACTUAL, &mut x);
|
||||
dc.push(&x, r.add_in_window(a, c, t, h) as i32 as f64, wgt);
|
||||
}
|
||||
@@ -487,3 +516,86 @@ pub fn backtest(p: &Panel, tau: &[u32; C], horizons: &[usize], start: usize, md:
|
||||
writeln!(md)?;
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// Walk-forward test of the release-event model: at each origin t0 the model sees only events
|
||||
/// whose 8 weeks had closed, and scores the releases of week t0+1. The new song's NMF placement
|
||||
/// isn't known in advance, so it's blended from the artist's history (as on the site).
|
||||
pub fn backtest_events(p: &Panel, tau: &[u32; C], start: usize, md: &mut String) -> Result<Vec<(String, usize, String, f64)>> {
|
||||
let r = Rot::new(p, tau);
|
||||
let mut rows = Vec::new();
|
||||
let names = ["rate by in-rotation-now", "rate by last-12-weeks level", "event model"];
|
||||
let mut wk_any: Vec<[f64; 3]> = Vec::new();
|
||||
let mut wk_ch: Vec<[f64; 3]> = Vec::new();
|
||||
let mut pairs_any: [Vec<(f64, bool)>; 3] = Default::default();
|
||||
let mut pairs_ch: [Vec<(f64, bool)>; 3] = Default::default();
|
||||
let (mut n_ev, mut n_hit) = (0usize, 0usize);
|
||||
let lvl = |m: f64| ((m.ln_1p() / 0.5) as usize).min(7);
|
||||
let level = |a: usize, t: usize| (0..LAGS).map(|i| p.group_total(a, t - i, None)).sum::<f64>() / LAGS as f64;
|
||||
for t0 in start..p.w().saturating_sub(W) {
|
||||
if t0 + 1 >= p.w() { break; }
|
||||
let m = AddModel::fit_events(p, &r, t0);
|
||||
// baselines from the training events
|
||||
let (mut by_rot, mut by_lvl) = ([[1.0f64, 2.0]; 2], [[1.0f64, 2.0]; 8]);
|
||||
let (mut cr, mut cl) = ([[1.0f64, 2.0]; 2], [[1.0f64, 2.0]; 8]);
|
||||
for t in origins(0, t0) {
|
||||
for a in 0..p.a() {
|
||||
if !clean_release(p, a, t + 1) { continue; }
|
||||
let y = r.add_any_in_window(a, t, 0) as i32 as f64;
|
||||
let on = r.any(a, t) as usize;
|
||||
by_rot[on][0] += y; by_rot[on][1] += 1.0;
|
||||
let b = lvl(level(a, t)); by_lvl[b][0] += y; by_lvl[b][1] += 1.0;
|
||||
for c in 0..C {
|
||||
let yc = r.add_in_window(a, c, t, 0) as i32 as f64;
|
||||
let onc = r.on(a, c, t) as usize;
|
||||
cr[onc][0] += yc; cr[onc][1] += 1.0;
|
||||
let bc = lvl((0..LAGS).map(|i| p.y(a, c, t - i)).sum::<f64>() / LAGS as f64);
|
||||
cl[bc][0] += yc; cl[bc][1] += 1.0;
|
||||
}
|
||||
}
|
||||
}
|
||||
let (mut la, mut lc) = ([0.0; 3], [0.0; 3]);
|
||||
let mut any_events = false;
|
||||
for a in 0..p.a() {
|
||||
if !clean_release(p, a, t0 + 1) { continue; }
|
||||
any_events = true;
|
||||
let q = p_nmf(p, a, t0);
|
||||
let (pcn, pan) = m.predict(p, &r, a, t0, Scenario::new(true, true), None);
|
||||
let (pcr, par) = m.predict(p, &r, a, t0, Scenario::new(true, false), None);
|
||||
let pa = q * pan + (1.0 - q) * par;
|
||||
let y = r.add_any_in_window(a, t0, 0);
|
||||
let on = r.any(a, t0) as usize;
|
||||
let preds = [by_rot[on][0] / by_rot[on][1], by_lvl[lvl(level(a, t0))][0] / by_lvl[lvl(level(a, t0))][1], pa];
|
||||
for k in 0..3 { la[k] += ll(preds[k], y); pairs_any[k].push((preds[k], y)); }
|
||||
n_ev += 1; n_hit += y as usize;
|
||||
for c in 0..C {
|
||||
let pc = q * pcn[c] + (1.0 - q) * pcr[c];
|
||||
let yc = r.add_in_window(a, c, t0, 0);
|
||||
let onc = r.on(a, c, t0) as usize;
|
||||
let bc = lvl((0..LAGS).map(|i| p.y(a, c, t0 - i)).sum::<f64>() / LAGS as f64);
|
||||
let pr = [cr[onc][0] / cr[onc][1], cl[bc][0] / cl[bc][1], pc];
|
||||
for k in 0..3 { lc[k] += ll(pr[k], yc); pairs_ch[k].push((pr[k], yc)); }
|
||||
}
|
||||
}
|
||||
if any_events { wk_any.push(la); wk_ch.push(lc); }
|
||||
eprintln!("event backtest: origin {} done", p.weeks[t0]);
|
||||
}
|
||||
writeln!(md, "# Backtest: if the artist releases a new song, will DR add it?\n")?;
|
||||
writeln!(md, "Events: an artist in the panel releases something (Deezer) in week t0+1, with no other release in the {CLEAN_WEEKS} weeks before (so an add can't be the previous song's); target: a DR add (entering rotation after {GAP}+ weeks out) within the {W} weeks from the release week. The model knows the artist's state the week before; the new song's NMF Denmark placement is blended from the artist's history. {n_ev} release events scored, {n_hit} ({:.1}%) got an add on some DR channel.\n",
|
||||
100.0 * n_hit as f64 / n_ev.max(1) as f64)?;
|
||||
writeln!(md, "| Model | Log-loss, any channel | PR-AUC, any channel | Log-loss, per channel | PR-AUC, per channel |\n|---|---|---|---|---|")?;
|
||||
let nch = (n_ev * C).max(1) as f64;
|
||||
for k in 0..3 {
|
||||
let (a, c) = (wk_any.iter().map(|w| w[k]).sum::<f64>() / n_ev.max(1) as f64, wk_ch.iter().map(|w| w[k]).sum::<f64>() / nch);
|
||||
let (pa, pc) = (average_precision(pairs_any[k].clone()), average_precision(pairs_ch[k].clone()));
|
||||
writeln!(md, "| {} | {a:.4} | {pa:.3} | {c:.4} | {pc:.3} |", names[k])?;
|
||||
for (metric, v) in [("event_logloss_any", a), ("event_prauc_any", pa), ("event_logloss_ch", c), ("event_prauc_ch", pc)] { rows.push((names[k].to_string(), 0, metric.to_string(), v)); }
|
||||
}
|
||||
let col = |v: &Vec<[f64; 3]>, k: usize| v.iter().map(|w| w[k]).collect::<Vec<f64>>();
|
||||
let (s1, s2) = (skill(&col(&wk_any, 2), &col(&wk_any, 0)), skill(&col(&wk_any, 2), &col(&wk_any, 1)));
|
||||
rows.push(("event model".into(), 0, "event_skill_any_vs_level".into(), s2.0));
|
||||
writeln!(md, "\nSkill of the event model, any channel: {:.3} ({:.3} to {:.3}) vs in-rotation-now, {:.3} ({:.3} to {:.3}) vs last-12-weeks level.\n", s1.0, s1.1, s1.2, s2.0, s2.1, s2.2)?;
|
||||
let m = AddModel::fit_events(p, &r, p.w() - 1);
|
||||
writeln!(md, "| Any-channel feature (all events) | Coefficient |\n|---|---|")?;
|
||||
for (n, b) in ANY_NAMES.iter().chain(REL_NAMES).chain(WAVE_NAMES).chain(LONG_NAMES).chain(NMF_NAMES).zip(&m.any) { writeln!(md, "| {n} | {b:+.3} |")?; }
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
//! airplay model nmf [--out docs/NMF.md] New Music Friday Denmark vs DR adds
|
||||
//! airplay model synth [--out docs/SYNTH.md] synthetic rotation lists (archive + thresholds)
|
||||
//! airplay model ifpi the 20-year IFPI test bed -> docs/IFPI.md
|
||||
//! airplay model events [--out docs/BACKTEST-EVENTS.md] backtest of the release-event model
|
||||
//! airplay model adds [--h 0,4] [--out docs/BACKTEST-ADDS.md] backtest of the new-add model
|
||||
//! airplay model rank [--h 4] [--max-h 12] [--top 50]
|
||||
|
||||
@@ -82,6 +83,19 @@ pub fn run(db: &Db, args: &[String]) -> Result<()> {
|
||||
if let Some(out) = arg(args, "--out") { std::fs::write(out, &md)?; }
|
||||
print!("{md}");
|
||||
}
|
||||
Some("events") => {
|
||||
let tau = common::thresholds(&p, 26);
|
||||
let mut md = String::new();
|
||||
let rows = adds::backtest_events(&p, &tau, 26, &mut md)?;
|
||||
rank::create_tables(db)?;
|
||||
let at = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
for (m, h, metric, v) in &rows {
|
||||
db.conn.execute("INSERT OR REPLACE INTO backtest_scores VALUES (?1, ?2, ?3, ?4, ?5)", rusqlite::params![at, m, *h as i64, metric, v])?;
|
||||
}
|
||||
let out = arg(args, "--out").unwrap_or_else(|| "docs/BACKTEST-EVENTS.md".into());
|
||||
std::fs::write(&out, &md)?;
|
||||
print!("{md}");
|
||||
}
|
||||
Some("adds") => {
|
||||
if let Some(w) = arg(args, "--without") { let _ = adds::WITHOUT.set(w.split(',').map(str::to_string).collect()); }
|
||||
let horizons: Vec<usize> = arg(args, "--h").unwrap_or_else(|| "0,4".into()).split(',').filter_map(|s| s.trim().parse().ok()).collect();
|
||||
|
||||
+19
-13
@@ -35,7 +35,8 @@ pub fn create_tables(db: &Db) -> Result<()> {
|
||||
target TEXT NOT NULL, -- 'rotation' (prob, mean plays) per channel, 'any' across channels;
|
||||
-- 'add' / 'add_any': P(new add in the 8 weeks from target_week), blended
|
||||
-- by the artist's chance of releasing ('p_release'); 'add_rel' /
|
||||
-- 'add_any_rel': the same if the artist releases in the window;
|
||||
-- 'add_any_rel' / 'add_rel': if the artist releases a NEW song (release-event model),
|
||||
-- 'add_any_nmf': ...placed on NMF Denmark; 'recent_release': released in the last 6 weeks;
|
||||
-- 'now': in rotation in the origin week (prob 1/0, mean = plays)
|
||||
key TEXT NOT NULL, -- '{dr artist id}:{channel}' or '{dr artist id}:any'
|
||||
horizon INTEGER NOT NULL, -- weeks after origin_week
|
||||
@@ -113,6 +114,16 @@ pub fn run(db: &Db, p: &Panel, max_h: usize, show_h: usize) -> Result<Vec<Line>>
|
||||
// New adds: windows of W weeks starting h = 0..=max_h weeks after the origin.
|
||||
let tau = thresholds(p, p.w());
|
||||
let rot = Rot::new(p, &tau);
|
||||
// The release-event model ("if they release a new song, will DR add it?"), the same for every
|
||||
// window start: the artist's state now, the new song's NMF placement blended from history.
|
||||
let ev = AddModel::fit_events(p, &rot, p.w() - 1);
|
||||
let t0 = p.w() - 1;
|
||||
let event: Vec<([f64; C], f64, f64, f64, bool)> = (0..p.a()).map(|a| {
|
||||
let q = super::adds::p_nmf(p, a, t0);
|
||||
let (pcn, pan) = ev.predict(p, &rot, a, t0, Scenario::new(true, true), None);
|
||||
let (pcr, par) = ev.predict(p, &rot, a, t0, Scenario::new(true, false), None);
|
||||
(std::array::from_fn(|c| q * pcn[c] + (1.0 - q) * pcr[c]), q * pan + (1.0 - q) * par, pan, q, super::adds::recent_release(p, a, t0))
|
||||
}).collect();
|
||||
for h in 0..=max_h {
|
||||
let m = AddModel::fit(p, &rot, h, p.w() - 1);
|
||||
let start = (origin + Duration::weeks(h as i64 + 1)).to_string();
|
||||
@@ -121,29 +132,24 @@ pub fn run(db: &Db, p: &Panel, max_h: usize, show_h: usize) -> Result<Vec<Line>>
|
||||
let mut ins = tx.prepare("INSERT OR REPLACE INTO forecasts VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)")?;
|
||||
for a in 0..p.a() {
|
||||
let id = &p.artists[a].id;
|
||||
// If they release in the window, if they don't, and blended by their release rate.
|
||||
let t0 = p.w() - 1;
|
||||
// If they release: placed on NMF Denmark or not, weighted by how often the artist's
|
||||
// releases were placed; if they don't release: no NMF placement.
|
||||
let q = super::adds::p_nmf(p, a, t0);
|
||||
let (pcn, pan) = m.predict(p, &rot, a, t0, Scenario::new(true, true), None);
|
||||
let (pcr, par) = m.predict(p, &rot, a, t0, Scenario::new(true, false), None);
|
||||
// Any add in the window (current songs included): no new release, or one at the usual rate.
|
||||
let (pc1, pa1) = m.predict(p, &rot, a, t0, Scenario::new(true, false), None);
|
||||
let (pc0, pa0) = m.predict(p, &rot, a, t0, Scenario::new(false, false), None);
|
||||
let pc1: [f64; C] = std::array::from_fn(|c| q * pcn[c] + (1.0 - q) * pcr[c]);
|
||||
let pa1 = q * pan + (1.0 - q) * par;
|
||||
let pr = super::adds::p_release(p, a, t0);
|
||||
let mix = |p1: f64, p0: f64| match pr { Some(r) => r * p1 + (1.0 - r) * p0, None => p0 };
|
||||
let (ec, ea, ean, q, recent) = event[a];
|
||||
let none = Option::<f64>::None;
|
||||
let mut put = |target: &str, ch: &str, prob: f64| ins.execute(params![made, VERSION, target, format!("{id}:{ch}"), h as i64,
|
||||
origin.to_string(), start, prob, none, none, none]);
|
||||
put("add_any", "any", mix(pa1, pa0))?;
|
||||
put("add_any_rel", "any", pa1)?;
|
||||
put("add_any_nmf", "any", pan)?;
|
||||
put("add_any_rel", "any", ea)?;
|
||||
put("add_any_nmf", "any", ean)?;
|
||||
put("p_nmf", "any", q)?;
|
||||
if recent { put("recent_release", "any", 1.0)?; }
|
||||
if let Some(r) = pr { put("p_release", "any", r)?; }
|
||||
for c in 0..C {
|
||||
put("add", CHANNELS[c], mix(pc1[c], pc0[c]))?;
|
||||
put("add_rel", CHANNELS[c], pc1[c])?;
|
||||
put("add_rel", CHANNELS[c], ec[c])?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user