New-add model (add = entering rotation after 4+ weeks out; per channel and any DR channel, 8-week window), site ranks by it; DR rotation-list PDF parser (glyph positions, channel by play profile) and Wayback Machine backfill: 29 weekly lists 2023-2026

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-15 02:50:13 +00:00
co-authored by Claude Opus 5.5
parent 387502ebfa
commit c698925814
9 changed files with 1122 additions and 89 deletions
+308
View File
@@ -0,0 +1,308 @@
//! New adds: the chance that an artist gets a (new) song into rotation, rather than the chance
//! of being played at all.
//!
//! The DR artist pages count plays per artist, not per song, so a song-level add isn't visible.
//! The proxy used here:
//!
//! add on channel c in week w the artist is in rotation on c in w (plays >= tau_c) after at
//! least GAP weeks out of rotation on c
//! add on any DR channel the artist enters rotation on some channel after at least GAP
//! weeks out of rotation on every channel (closer to "a new song")
//!
//! The target is an add within the window of W weeks that starts h weeks after the origin
//! (weeks t+h+1 .. t+h+W): "if a window opened h weeks from now, would the artist get an add in it?"
//! An artist with a song in rotation now has to drop out first, which is how it usually works:
//! in the week of song-level plays we have, only 3-12% of the artists in rotation on a station
//! had two songs in rotation at once. Re-entries of an old song also count as adds; the parsed
//! rotation lists (song level) will separate them.
use super::common::{mix, Rng};
use super::features::{self, LAGS, P};
use super::glm::{self, Design};
use super::panel::{Panel, C};
use anyhow::Result;
use std::fmt::Write as _;
pub const GAP: usize = 4;
pub const W: usize = 8;
/// Per-channel rows: the panel features plus these.
pub const EXTRA: &[&str] = &["in rotation on channel now", "in rotation anywhere now", "channels in rotation now (/14)",
"run on channel = 1 week", "run = 2 weeks", "run = 3-5 weeks", "run >= 6 weeks",
"near threshold (>= half of tau, not in rotation)", "plays / tau (capped 3, /3)",
"weeks since in rotation on channel (/12)"];
pub const PC: usize = P + 10;
/// Artist-level ("any channel") rows.
pub const ANY_NAMES: &[&str] = &[
"intercept", "log DR plays t", "log DR plays 4w", "log DR plays 12w", "log max DR plays 12w",
"in rotation anywhere now", "channels in rotation now (/14)", "current run anywhere (/12)", "weeks since in rotation anywhere (/12)",
"Track Top-40 score", "Airplay Top-20 score", "chart entry recency 6w", "chart entry recency 26w",
"log Track Top-40 hits", "log Airplay Top-20 hits", "Danish", "group", "female", "career years (/50)", "debut unknown",
"Christmas in window",
];
pub const PA: usize = 21;
/// Positions of the chart covariates (for the what-if): (t40 score, air score, recency 6w, recency 26w).
pub const CHART_IDX_CHANNEL: [usize; 4] = [10, 11, 12, 13];
pub const CHART_IDX_ANY: [usize; 4] = [9, 10, 11, 12];
pub struct Rot {
nw: usize,
pub tau: [u32; C],
rot: Vec<bool>, // [a][c][w]
any: Vec<bool>, // [a][w]
}
impl Rot {
pub fn new(p: &Panel, tau: &[u32; C]) -> Rot {
let nw = p.w();
let mut rot = vec![false; p.a() * C * nw];
let mut any = vec![false; p.a() * nw];
for a in 0..p.a() { for c in 0..C { for w in 0..nw {
let r = p.y(a, c, w) >= tau[c] as f64;
rot[(a * C + c) * nw + w] = r;
if r { any[a * nw + w] = true; }
} } }
Rot { nw, tau: *tau, rot, any }
}
pub fn on(&self, a: usize, c: usize, w: usize) -> bool { self.rot[(a * C + c) * self.nw + w] }
pub fn any(&self, a: usize, w: usize) -> bool { self.any[a * self.nw + w] }
pub fn add(&self, a: usize, c: usize, w: usize) -> bool { w >= GAP && self.on(a, c, w) && (1..=GAP).all(|k| !self.on(a, c, w - k)) }
pub fn add_any(&self, a: usize, w: usize) -> bool { w >= GAP && self.any(a, w) && (1..=GAP).all(|k| !self.any(a, w - k)) }
/// An add in weeks t+h+1 ..= t+h+W (all inside the panel).
pub fn add_in_window(&self, a: usize, c: usize, t: usize, h: usize) -> bool { (t + h + 1..=t + h + W).any(|w| self.add(a, c, w)) }
pub fn add_any_in_window(&self, a: usize, t: usize, h: usize) -> bool { (t + h + 1..=t + h + W).any(|w| self.add_any(a, w)) }
}
fn tau_of(_p: &Panel, r: &Rot, _a: usize, c: usize, _t: usize) -> f64 { r.tau[c] as f64 }
fn run_and_since(flag: impl Fn(usize) -> bool, t: usize) -> (f64, f64) {
let run = (0..LAGS).take_while(|&i| flag(t - i)).count() as f64 / LAGS as f64;
let since = (0..LAGS).find(|&i| flag(t - i)).unwrap_or(LAGS) as f64 / LAGS as f64;
(run, since)
}
fn christmas_in_window(p: &Panel, t: usize, h: usize) -> bool { (t + h + 1..=t + h + W).any(|w| features::christmas(p, w)) }
pub fn row_channel(p: &Panel, r: &Rot, a: usize, c: usize, t: usize, h: usize, out: &mut [f64]) {
features::row(p, a, c, t, t + h + 1, &mut out[..P]);
out[21] = christmas_in_window(p, t, h) as i32 as f64;
let n_on = (0..C).filter(|&k| r.on(a, k, t)).count() as f64;
let (run, since) = run_and_since(|w| r.on(a, c, w), t);
let weeks = (run * LAGS as f64).round() as usize;
let on = r.on(a, c, t);
let ratio = p.y(a, c, t) / tau_of(p, r, a, c, t);
let f = |b: bool| b as i32 as f64;
out[P..PC].copy_from_slice(&[f(on), f(r.any(a, t)), n_on / C as f64,
f(weeks == 1), f(weeks == 2), f((3..=5).contains(&weeks)), f(weeks >= 6),
f(!on && ratio >= 0.5), ratio.min(3.0) / 3.0, since]);
}
pub fn row_any(p: &Panel, r: &Rot, a: usize, t: usize, h: usize, out: &mut [f64]) {
let tot = |w: usize| (0..C).map(|c| p.y(a, c, w)).sum::<f64>();
let mean = |n: usize| (0..n).map(|i| tot(t - i)).sum::<f64>() / n as f64;
let max12 = (0..LAGS).map(|i| tot(t - i)).fold(0.0, f64::max);
let n_on = (0..C).filter(|&k| r.on(a, k, t)).count() as f64;
let (run, since) = run_and_since(|w| r.any(a, w), t);
let ch = p.chart(a, t);
let art = &p.artists[a];
let recency = |s: f64| if ch.weeks_since_entry == u16::MAX { 0.0 } else { (-(ch.weeks_since_entry as f64) / s).exp() };
let year = chrono::Datelike::year(&p.weeks[t]);
let v = [
1.0, tot(t).ln_1p(), mean(4).ln_1p(), mean(12).ln_1p(), max12.ln_1p(),
r.any(a, t) as i32 as f64, n_on / C as f64, run, since,
if ch.t40_rank > 0 { (41.0 - ch.t40_rank as f64) / 40.0 } else { 0.0 },
if ch.air_rank > 0 { (21.0 - ch.air_rank as f64) / 20.0 } else { 0.0 },
recency(6.0), recency(26.0), (ch.t40_hits as f64).ln_1p(), (ch.air_hits as f64).ln_1p(),
art.danish as i32 as f64, art.group as i32 as f64, art.female as i32 as f64,
art.debut.map(|d| ((year - d).clamp(0, 50)) as f64 / 50.0).unwrap_or(0.0), art.debut.is_none() as i32 as f64,
christmas_in_window(p, t, h) as i32 as f64,
];
out.copy_from_slice(&v);
}
/// Logit models for one h, trained on origins whose whole window is observed by `last_week`.
pub struct AddModel {
pub h: usize,
pub channel: Vec<f64>,
pub any: Vec<f64>,
}
fn origins(h: usize, last_week: usize) -> std::ops::RangeInclusive<usize> {
let hi = last_week.saturating_sub(h + W);
LAGS - 1..=hi
}
impl AddModel {
pub fn fit(p: &Panel, r: &Rot, h: usize, last_week: usize) -> AddModel {
let (mut dc, mut da) = (Design::new(PC), Design::new(PA));
let (mut x, mut xa) = (vec![0.0; PC], vec![0.0; PA]);
for t in origins(h, last_week) {
for a in 0..p.a() {
for c in 0..C {
// Channels where the artist hasn't been heard for 12 weeks: keep a quarter, weighted 4.
let quiet = (0..LAGS).all(|i| p.y(a, c, t - i) == 0.0);
let wgt = if quiet { if mix((a as u64) << 32 ^ (c as u64) << 20 ^ t as u64 ^ 0xadd) % 4 != 0 { continue; } 4.0 } else { 1.0 };
row_channel(p, r, a, c, t, h, &mut x);
dc.push(&x, r.add_in_window(a, c, t, h) as i32 as f64, wgt);
}
row_any(p, r, a, t, h, &mut xa);
da.push(&xa, r.add_any_in_window(a, t, h) as i32 as f64, 1.0);
}
}
AddModel { h, channel: glm::logit(&dc, 1.0), any: glm::logit(&da, 1.0) }
}
/// (P(add) per channel, P(add on any channel)) from origin t.
pub fn predict(&self, p: &Panel, r: &Rot, a: usize, t: usize, chart: Option<&dyn Fn(&mut [f64], [usize; 4])>) -> ([f64; C], f64) {
let mut x = vec![0.0; PC];
let mut pc = [0.0; C];
for c in 0..C {
row_channel(p, r, a, c, t, self.h, &mut x);
if let Some(f) = chart { f(&mut x, CHART_IDX_CHANNEL); }
pc[c] = glm::sigmoid(glm::dot(&self.channel, &x)).clamp(1e-6, 1.0 - 1e-6);
}
let mut xa = vec![0.0; PA];
row_any(p, r, a, t, self.h, &mut xa);
if let Some(f) = chart { f(&mut xa, CHART_IDX_ANY); }
(pc, glm::sigmoid(glm::dot(&self.any, &xa)).clamp(1e-6, 1.0 - 1e-6))
}
}
/// Sets the chart covariates at `idx` (t40 score, air score, recency 6w, recency 26w).
pub fn set_chart(x: &mut [f64], idx: [usize; 4], entry: Option<Option<f64>>, t40: Option<f64>, air: Option<f64>) {
if let Some(e) = entry {
let (r6, r26) = match e { Some(w) => ((-w / 6.0).exp(), (-w / 26.0).exp()), None => (0.0, 0.0) };
x[idx[2]] = r6; x[idx[3]] = r26;
}
if let Some(r) = t40 { x[idx[0]] = if (1.0..=40.0).contains(&r) { (41.0 - r) / 40.0 } else { 0.0 }; }
if let Some(r) = air { x[idx[1]] = if (1.0..=20.0).contains(&r) { (21.0 - r) / 20.0 } else { 0.0 }; }
}
// ---------------------------------------------------------------- backtest
fn ll(p: f64, y: bool) -> f64 { -(if y { p } else { 1.0 - p }).clamp(1e-9, 1.0).ln() }
fn average_precision(mut v: Vec<(f64, bool)>) -> f64 {
v.sort_by(|a, b| b.0.total_cmp(&a.0));
let (mut hits, mut sum) = (0.0, 0.0);
for (i, (_, y)) in v.iter().enumerate() { if *y { hits += 1.0; sum += hits / (i as f64 + 1.0); } }
if hits > 0.0 { sum / hits } else { f64::NAN }
}
fn skill(model: &[f64], base: &[f64]) -> (f64, f64, f64) {
let point = 1.0 - model.iter().sum::<f64>() / base.iter().sum::<f64>();
let n = model.len();
let block = 3.min(n);
let mut rng = Rng(7);
let mut d: Vec<f64> = (0..2000).map(|_| {
let (mut lm, mut lb, mut k) = (0.0, 0.0, 0);
while k < n { let s = (rng.u() * (n - block + 1) as f64) as usize; for i in s..s + block { if k < n { lm += model[i]; lb += base[i]; k += 1; } } }
1.0 - lm / lb
}).collect();
d.sort_by(f64::total_cmp);
(point, d[100], d[1899])
}
/// Walk-forward: at each origin t0 the models see only windows that had closed by t0.
pub fn backtest(p: &Panel, tau: &[u32; C], horizons: &[usize], start: usize, md: &mut String) -> Result<Vec<(String, usize, String, f64)>> {
let r = Rot::new(p, tau);
let mut rows = Vec::new();
writeln!(md, "# Backtest: new adds\n")?;
writeln!(md, "An add = the artist enters rotation (plays ≥ τ) after at least {GAP} weeks out of rotation: per channel, and on any DR channel after {GAP} weeks out of rotation everywhere. Target: an add within the {W} weeks that start h weeks after the forecast origin. Origins from week {}; each fit uses only windows closed by its origin.\n", start + 1)?;
for &h in horizons {
let last_origin = p.w() - 1 - h - W;
if last_origin < start { continue; }
// [model][week] log-loss sums; models: base(rot now), base(popularity bins), logit
let names = ["rate by in-rotation-now", "rate by last-12-weeks level", "logit"];
let mut wk_c: Vec<[f64; 3]> = Vec::new();
let mut wk_a: Vec<[f64; 3]> = Vec::new();
let mut pairs_c: [Vec<(f64, bool)>; 3] = Default::default();
let mut pairs_a: [Vec<(f64, bool)>; 3] = Default::default();
let mut calib = [[0.0f64; 3]; 10];
let (mut nc, mut na, mut hits_c, mut hits_a) = (0.0, 0.0, 0.0, 0.0);
for t0 in start..=last_origin {
let m = AddModel::fit(p, &r, h, t0);
// Baselines from the same closed windows.
let mut by_rot = [[1.0f64, 2.0]; 2];
let mut by_lvl = [[1.0f64, 2.0]; 8];
let mut any_rot = [[1.0f64, 2.0]; 2];
let mut any_lvl = [[1.0f64, 2.0]; 8];
let lvl = |m12: f64| ((m12.ln_1p() / 0.5) as usize).min(7);
for t in origins(h, t0) {
for a in 0..p.a() {
let mut tot = 0.0;
for c in 0..C {
let y = r.add_in_window(a, c, t, h) as i32 as f64;
let on = r.on(a, c, t) as usize;
by_rot[on][0] += y; by_rot[on][1] += 1.0;
let m12 = (0..LAGS).map(|i| p.y(a, c, t - i)).sum::<f64>() / LAGS as f64;
let b = lvl(m12); by_lvl[b][0] += y; by_lvl[b][1] += 1.0;
tot += m12;
}
let y = r.add_any_in_window(a, t, h) as i32 as f64;
let on = r.any(a, t) as usize;
any_rot[on][0] += y; any_rot[on][1] += 1.0;
let b = lvl(tot); any_lvl[b][0] += y; any_lvl[b][1] += 1.0;
}
}
let (mut lc, mut la) = ([0.0; 3], [0.0; 3]);
for a in 0..p.a() {
let (pc, pa) = m.predict(p, &r, a, t0, None);
let mut tot = 0.0;
for c in 0..C {
let y = r.add_in_window(a, c, t0, h);
let on = r.on(a, c, t0) as usize;
let m12 = (0..LAGS).map(|i| p.y(a, c, t0 - i)).sum::<f64>() / LAGS as f64;
tot += m12;
let preds = [by_rot[on][0] / by_rot[on][1], by_lvl[lvl(m12)][0] / by_lvl[lvl(m12)][1], pc[c]];
for k in 0..3 { lc[k] += ll(preds[k], y); pairs_c[k].push((preds[k], y)); }
let d = ((pc[c] * 10.0) as usize).min(9);
calib[d][0] += pc[c]; calib[d][1] += y as i32 as f64; calib[d][2] += 1.0;
nc += 1.0; hits_c += y as i32 as f64;
}
let y = r.add_any_in_window(a, t0, h);
let on = r.any(a, t0) as usize;
let preds = [any_rot[on][0] / any_rot[on][1], any_lvl[lvl(tot)][0] / any_lvl[lvl(tot)][1], pa];
for k in 0..3 { la[k] += ll(preds[k], y); pairs_a[k].push((preds[k], y)); }
na += 1.0; hits_a += y as i32 as f64;
}
wk_c.push(lc); wk_a.push(la);
eprintln!("adds backtest h={h}: origin {} done", p.weeks[t0]);
}
let nwk = wk_c.len();
writeln!(md, "## Window starting h = {h} weeks ahead ({nwk} forecast origins)\n")?;
writeln!(md, "Base rates: per channel {:.2}% of artist-channels get an add in the window; on any channel {:.1}% of artists.\n", 100.0 * hits_c / nc, 100.0 * hits_a / na)?;
writeln!(md, "| Model | Log-loss, per channel | PR-AUC, per channel | Log-loss, any channel | PR-AUC, any channel |\n|---|---|---|---|---|")?;
for k in 0..3 {
let (sc, sa) = (wk_c.iter().map(|w| w[k]).sum::<f64>() / nc, wk_a.iter().map(|w| w[k]).sum::<f64>() / na);
let (ac, aa) = (average_precision(pairs_c[k].clone()), average_precision(pairs_a[k].clone()));
for (metric, v) in [("add_logloss", sc), ("add_prauc", ac), ("addany_logloss", sa), ("addany_prauc", aa)] {
rows.push((names[k].to_string(), h, metric.to_string(), v));
}
writeln!(md, "| {} | {sc:.4} | {ac:.3} | {sa:.4} | {aa:.3} |", names[k])?;
}
let col = |v: &Vec<[f64; 3]>, k: usize| v.iter().map(|w| w[k]).collect::<Vec<f64>>();
let (s1, s2) = (skill(&col(&wk_c, 2), &col(&wk_c, 0)), skill(&col(&wk_c, 2), &col(&wk_c, 1)));
let (s3, s4) = (skill(&col(&wk_a, 2), &col(&wk_a, 0)), skill(&col(&wk_a, 2), &col(&wk_a, 1)));
rows.push(("logit".into(), h, "add_skill_vs_level".into(), s2.0));
rows.push(("logit".into(), h, "addany_skill_vs_level".into(), s4.0));
writeln!(md, "\nSkill of the logit (90% block-bootstrap interval over origins):\n")?;
writeln!(md, "| | vs in-rotation-now | vs last-12-weeks level |\n|---|---|---|")?;
writeln!(md, "| per channel | {:.3} ({:.3} to {:.3}) | {:.3} ({:.3} to {:.3}) |", s1.0, s1.1, s1.2, s2.0, s2.1, s2.2)?;
writeln!(md, "| any channel | {:.3} ({:.3} to {:.3}) | {:.3} ({:.3} to {:.3}) |\n", s3.0, s3.1, s3.2, s4.0, s4.1, s4.2)?;
writeln!(md, "Calibration, per channel:\n\n| Predicted | Rows | Mean predicted | Observed |\n|---|---|---|---|")?;
for (d, c) in calib.iter().enumerate() {
if c[2] > 0.0 { writeln!(md, "| {:.1}–{:.1} | {} | {:.3} | {:.3} |", d as f64 / 10.0, (d + 1) as f64 / 10.0, c[2], c[0] / c[2], c[1] / c[2])?; }
}
writeln!(md)?;
if nwk < 8 { writeln!(md, "Fewer than 8 forecast origins: no verdict at this horizon (doc rule).\n")?; }
}
let m = AddModel::fit(p, &r, 0, p.w() - 1);
writeln!(md, "## Coefficients, window starting now (h = 0), all closed windows\n\n| Any-channel feature | Coefficient |\n|---|---|")?;
for (n, b) in ANY_NAMES.iter().zip(&m.any) { writeln!(md, "| {n} | {b:+.3} |")?; }
writeln!(md, "\n| Per-channel feature | Coefficient |\n|---|---|")?;
for (n, b) in features::NAMES.iter().chain(EXTRA).zip(&m.channel) { writeln!(md, "| {n} | {b:+.3} |")?; }
writeln!(md)?;
Ok(rows)
}