Add model: quiet former favourite flag (DR archive 2019-24), subgroup scores in the event backtest

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-17 13:02:08 +00:00
co-authored by Claude Opus 5.5
parent 44e00b9793
commit 5c02e8a710
4 changed files with 122 additions and 73 deletions
+30 -6
View File
@@ -148,17 +148,27 @@ pub fn nmf_feats(p: &Panel, a: usize, t: usize, h: usize, sc: Scenario) -> [f64;
}
/// Long-history covariates: DR's 2012–2021 archive and DR's real rotation lists before week t.
pub const LONG: usize = 4;
pub const LONG: usize = 5;
pub const LONG_NAMES: &[&str] = &["log archive plays per 100 sampled hours (2012-21)", "log DR lists with the artist before t (2023-)",
"DR list recency 26w", "DR lists known before t at all"];
"DR list recency 26w", "DR lists known before t at all",
"quiet former favourite (4+ weeks on DR 2019-24, out of rotation 26w)"];
pub fn long_feats(p: &Panel, a: usize, t: usize, shift: usize) -> [f64; LONG] {
/// Not in rotation anywhere in the 26 weeks to t (as far as the panel goes back).
fn quiet(r: &Rot, a: usize, t: usize) -> bool { (t.saturating_sub(25)..=t).all(|w| !r.any(a, w)) }
pub fn long_feats(p: &Panel, r: &Rot, a: usize, t: usize, _h: usize, sc: Scenario) -> [f64; LONG] {
if off("long") { return [0.0; LONG]; }
let shift = sc.shift;
// The comeback case: a former favourite (archive pages 2019-24, before the panel) out of
// rotation for half a year. One bounded flag: a continuous favourite score next to its
// interaction with "quiet" fitted two large opposing coefficients on one panel year.
let fav = !off("fav") && p.fav_weeks[a] >= 4.0 && quiet(r, a, t);
let end_t = p.weeks[t] + chrono::Duration::days(6);
// A list published on a Friday is known from then.
let before: Vec<&chrono::NaiveDate> = p.listed_on[a].iter().filter(|d| **d <= end_t).collect();
let since = before.last().map(|d| (end_t - **d).num_days() as f64 / 7.0 + shift as f64);
[p.archive_rate[a].ln_1p(), (before.len() as f64).ln_1p(), since.map(|s| (-s / 26.0).exp()).unwrap_or(0.0), (!before.is_empty()) as i32 as f64]
[p.archive_rate[a].ln_1p(), (before.len() as f64).ln_1p(), since.map(|s| (-s / 26.0).exp()).unwrap_or(0.0), (!before.is_empty()) as i32 as f64,
fav as i32 as f64]
}
/// Season: the mean log season index over the window's weeks (new Airplay Top-20 songs by week of
@@ -345,7 +355,7 @@ pub fn row_channel(p: &Panel, r: &Rot, a: usize, c: usize, t: usize, h: usize, s
f(!on && ratio >= 0.5), ratio.min(3.0) / 3.0, since]);
out[P + 10..P + 10 + REL].copy_from_slice(&rel_feats(p, a, t, h, sc, on));
out[P + 10 + REL..P + 10 + REL + WAVE].copy_from_slice(&wave_feats(p, r, a, t, sc.shift));
out[P + 10 + REL + WAVE..P + 10 + REL + WAVE + LONG].copy_from_slice(&long_feats(p, a, t, sc.shift));
out[P + 10 + REL + WAVE..P + 10 + REL + WAVE + LONG].copy_from_slice(&long_feats(p, r, a, t, h, sc));
out[P + 10 + REL + WAVE + LONG..PC - SEASON - AFF].copy_from_slice(&nmf_feats(p, a, t, h, sc));
out[PC - SEASON - AFF..PC - AFF].copy_from_slice(&season_feat(p, t, h, sc.win));
out[PC - AFF..PC].copy_from_slice(&aff_feats(p, a, c, t));
@@ -374,7 +384,7 @@ pub fn row_any(p: &Panel, r: &Rot, a: usize, t: usize, h: usize, sc: Scenario, o
out[..21].copy_from_slice(&v);
out[21..21 + REL].copy_from_slice(&rel_feats(p, a, t, h, sc, r.any(a, t)));
out[21 + REL..21 + REL + WAVE].copy_from_slice(&wave_feats(p, r, a, t, sc.shift));
out[21 + REL + WAVE..21 + REL + WAVE + LONG].copy_from_slice(&long_feats(p, a, t, sc.shift));
out[21 + REL + WAVE..21 + REL + WAVE + LONG].copy_from_slice(&long_feats(p, r, a, t, h, sc));
out[21 + REL + WAVE + LONG..PA - SEASON].copy_from_slice(&nmf_feats(p, a, t, h, sc));
out[PA - SEASON..PA].copy_from_slice(&season_feat(p, t, h, sc.win));
}
@@ -779,6 +789,9 @@ pub fn backtest_events(p: &Panel, tau: &[u32; C], start: usize, win: usize, md:
let (mut pool_c, mut pool_a) = (Pool(Vec::new()), Pool(Vec::new()));
let lvl = |m: f64| ((m.ln_1p() / 0.5) as usize).min(7);
let level = |a: usize, t: usize| (0..LAGS).map(|i| p.group_total(a, t - i, None)).sum::<f64>() / LAGS as f64;
// Release events by quiet artists (out of rotation 26 weeks): former favourites (4+ weeks on DR 2019-24) or not.
let mut favs: Vec<(f64, f64, bool)> = Vec::new();
let mut quiet_other: Vec<(f64, f64, bool)> = Vec::new();
for t0 in start..p.w().saturating_sub(win + 1) {
let m = AddModel::fit_events_win(p, &r, t0, win);
// Baselines from the training events. Release timing only: by weeks since the previous release.
@@ -823,6 +836,7 @@ pub fn backtest_events(p: &Panel, tau: &[u32; C], start: usize, win: usize, md:
s_a.add(&[by_rot[on][0] / by_rot[on][1], by_lvl[lvl(level(a, t0))][0] / by_lvl[lvl(level(a, t0))][1], tma[0] / tma[1],
q * pan + (1.0 - q) * par, raw, pl_a.apply(raw), iso_a.apply(raw), pax], y);
pool_a.0.push((t0, raw, y));
if quiet(&r, a, t0) { if p.fav_weeks[a] >= 4.0 { favs.push((raw, iso_a.apply(raw), y)); } else { quiet_other.push((raw, iso_a.apply(raw), y)); } }
for c in 0..C {
let yc = r.add_in(a, c, t0, 0, win);
let onc = r.on(a, c, t0) as usize;
@@ -840,6 +854,16 @@ pub fn backtest_events(p: &Panel, tau: &[u32; C], start: usize, win: usize, md:
writeln!(md, "Events: an artist knowable at the time releases something (Deezer) in week t0+1, with no other release in the {CLEAN_WEEKS} weeks before (so an add can't be the previous song's); target: a DR add (entering rotation after {GAP}+ weeks out, or plays jumping well above the artist's own 8-week baseline and staying up) within {win} week(s) from the release week. The model knows the artist's state the week before; pre-release rows blend the new song's NMF Denmark placement from the artist's history. \"Release timing only\" = the add rate by weeks since the artist's previous release. {} release events scored, {} ({:.1}%) got an add on some DR channel.\n",
s_a.n, s_a.hits, 100.0 * s_a.hits / s_a.n.max(1.0))?;
table(md, &s_c, &s_a, &mut rows, win, "event_")?;
writeln!(md, "\nQuiet artists (out of rotation everywhere for 26 weeks) releasing, any channel:\n\n| Group | Events | Adds | Rate | Mean predicted (raw) | Mean predicted (isotonic) | Log-loss (isotonic) | PR-AUC (raw) |\n|---|---|---|---|---|---|---|---|")?;
for (name, v) in [("former favourites (4+ weeks on DR 2019-24)", &favs), ("other quiet artists", &quiet_other)] {
let n = v.len().max(1) as f64;
let hits = v.iter().filter(|x| x.2).count();
writeln!(md, "| {name} | {} | {hits} | {:.3} | {:.3} | {:.3} | {:.4} | {:.3} |", v.len(), hits as f64 / n,
v.iter().map(|x| x.0).sum::<f64>() / n, v.iter().map(|x| x.1).sum::<f64>() / n,
v.iter().map(|x| ll(x.1.clamp(1e-6, 1.0 - 1e-6), x.2)).sum::<f64>() / n,
average_precision(v.iter().map(|x| (x.0, x.2)).collect()))?;
}
writeln!(md)?;
let m = AddModel::fit_events_win(p, &r, p.w() - 1, win);
writeln!(md, "| Any-channel feature (all events) | Coefficient |\n|---|---|")?;
for (n, b) in ANY_NAMES.iter().chain(REL_NAMES).chain(WAVE_NAMES).chain(LONG_NAMES).chain(NMF_NAMES).chain(SEASON_NAMES).zip(&m.any) { writeln!(md, "| {n} | {b:+.3} |")?; }
+1
View File
@@ -115,6 +115,7 @@ pub fn run(db: &Db, args: &[String]) -> Result<()> {
}
Some("events") => {
if let Some(r) = arg(args, "--rule") { let _ = adds::ADD_RULE.set(r); }
if let Some(w) = arg(args, "--without") { let _ = adds::WITHOUT.set(w.split(',').map(str::to_string).collect()); }
let tau = common::thresholds(&p, 26);
let mut md = String::new();
let win: usize = arg(args, "--w").and_then(|w| w.parse().ok()).unwrap_or(2);
+16 -1
View File
@@ -92,6 +92,8 @@ pub struct Panel {
/// The first week (index) at which the artist was knowable: their first release, DR play,
/// chart entry, DR list, archived play or NMF placement (0 = before the panel; w() = never).
pub first_known: Vec<usize>,
/// Weeks with 20+ DR plays (all channels) on the archived artist pages, 2019-24 (before the panel).
pub fav_weeks: Vec<f64>,
/// Season index per ISO week (1..=53): new songs reaching the Airplay Top-20 that week of the
/// year, relative to an average week, 2002–2024 (before the DR panel), smoothed over 5 weeks.
pub season: [f64; 54],
@@ -187,12 +189,25 @@ impl Panel {
let nmf_dk = load_nmf_dk(db, &id_of, &artists)?;
let season = load_season(db)?;
let (alltime, genre_fit) = load_affinity(db, &id_of, artists.len())?;
let mut p = Panel { weeks, artists, y, chart, list_tau, releases, archive_rate, listed_on, nmf_dk, season, alltime, genre_fit, first_known: Vec::new() };
let mut p = Panel { weeks, artists, y, chart, list_tau, releases, archive_rate, listed_on, nmf_dk, season, alltime, genre_fit, first_known: Vec::new(), fav_weeks: Vec::new() };
p.fav_weeks = load_fav_weeks(db, &id_of, p.a(), p.weeks[0])?;
p.first_known = first_known(db, &p, &id_of)?;
Ok(p)
}
}
fn load_fav_weeks(db: &Db, id_of: &HashMap<String, usize>, n: usize, start: NaiveDate) -> Result<Vec<f64>> {
let mut out = vec![0.0; n];
let has: i64 = db.conn.query_row("SELECT count(*) FROM sqlite_master WHERE name = 'dr_artist_history'", [], |r| r.get(0))?;
if has == 0 { return Ok(out); }
for r in db.conn.prepare("SELECT artist_id, count(*) FROM dr_artist_history WHERE plays >= 20 AND week_start < ?1 GROUP BY artist_id")?
.query_map([start.to_string()], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?)))? {
let (id, k) = r?;
if let Some(&a) = id_of.get(&id) { out[a] = k as f64; }
}
Ok(out)
}
/// Lower-case letters and digits (æ, ø, å kept), for matching names.
pub fn key(s: &str) -> String {
s.to_lowercase().chars().filter(|c| c.is_alphanumeric()).collect()