Long model: sustained-add target (blips were 58% of adds), sanity-check notes

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-17 10:39:49 +00:00
co-authored by Claude Opus 5.5
parent 79b2cdc425
commit 9cd227755b
3 changed files with 116 additions and 88 deletions
+28 -21
View File
@@ -37,6 +37,8 @@ pub struct Long {
releases: Vec<Option<Vec<NaiveDate>>>,
nmf: Vec<Vec<(NaiveDate, Option<i64>)>>,
season: [f64; 54],
/// Weeks after the add that the label looks at (see `label`).
pub stay: usize,
}
impl Long {
@@ -104,18 +106,21 @@ impl Long {
let chart = chart_states(db, &weeks, &names, na)?;
let releases = load_releases(db, &id_of, na)?;
let nmf = load_nmf_dk(db, &id_of, &artists)?;
Ok(Long { weeks, artists, y, obs, cur0, known, chart, releases, nmf, season: p.season })
Ok(Long { weeks, artists, y, obs, cur0, known, chart, releases, nmf, season: p.season, stay: 3 })
}
/// Does origin (a, t) qualify, and if so was there an add in the window?
fn label(&self, a: usize, t: usize, win: usize, tau: f64) -> Option<bool> {
if t + 1 < OFF || t + win >= self.w() { return None; }
let end = t + win + self.stay;
if t + 1 < OFF || end >= self.w() { return None; }
if t >= self.cur0 && self.known[a] > t { return None; }
// Don't let a window straddle the gap between the eras.
if t < self.cur0 && t + win >= self.cur0 { return None; }
if !(t + 1 - OFF..=t + win).all(|w| self.obs(a, w)) { return None; }
if t < self.cur0 && end >= self.cur0 { return None; }
if !(t + 1 - OFF..=end).all(|w| self.obs(a, w)) { return None; }
if (t + 1 - OFF..=t).any(|w| self.y(a, w) >= tau) { return None; }
Some((t + 1..=t + win).any(|w| self.y(a, w) >= tau))
// stay = 0: any week at τ (a blip counts). stay = k: and τ again in at least 2 of the k weeks after it.
let on = |w: usize| self.y(a, w) >= tau;
Some((t + 1..=t + win).any(|w| on(w) && (self.stay == 0 || (w + 1..=w + self.stay).filter(|&v| on(v)).count() >= 2.min(self.stay))))
}
}
@@ -267,20 +272,22 @@ fn boot(te: &Test, m: usize, base: usize) -> (f64, f64, f64) {
(d, v[50], v[949])
}
pub fn run(db: &Db, p: &Panel, tau: f64, win: usize) -> Result<String> {
let l = Long::load(db, p)?;
pub fn run(db: &Db, p: &Panel, tau: f64, win: usize, stay: usize) -> Result<String> {
let mut l = Long::load(db, p)?;
l.stay = stay;
let all = rows(&l, win, tau);
let mut md = String::new();
let n_old = all.iter().filter(|r| r.0 < l.cur0).count();
let a_old = all.iter().filter(|r| r.0 < l.cur0 && r.2).count();
let _ = writeln!(md, "# The long any-DR model (2019–)\n");
let _ = writeln!(md, "Total DR plays per artist and week (all channels, P4 regions included). An *add* is τ = {tau} or more plays in a week of the next {win} week(s) after {OFF} weeks under τ. \
let _ = writeln!(md, "Total DR plays per artist and week (all channels, P4 regions included). An *add* is τ = {tau} or more plays in a week of the next {win} week(s) after {OFF} weeks under τ{}. \
Archive era {} to {} ({} artists observed, {} qualifying rows, {} adds); current panel {} to {} ({} rows, {} adds).\n",
l.weeks[0], l.weeks[l.cur0 - 1], (0..l.artists.len()).filter(|&a| (0..l.cur0).any(|w| l.obs(a, w))).count(), n_old, a_old,
if stay > 0 { format!(", and τ again in at least 2 of the {stay} weeks after it (it stays on)") } else { " (a one-week blip counts)".into() },
l.weeks[0], l.weeks[(0..l.cur0).rev().find(|&w| (0..l.artists.len()).any(|a| l.obs(a, w))).unwrap_or(0)], (0..l.artists.len()).filter(|&a| (0..l.cur0).any(|w| l.obs(a, w))).count(), n_old, a_old,
l.weeks[l.cur0], l.weeks[l.w() - 1], all.len() - n_old, all.iter().filter(|r| r.0 >= l.cur0 && r.2).count());
let _ = writeln!(md, "Models: **1-year** trains on the 52 weeks of origins before the test period (what the site can do from the current panel alone); \
**all years** on every earlier origin; **+ older history** adds four features from the artist's weeks before the last 52 (for today's predictions: the 2019–24 archive); **all years + era** adds an archive-era dummy, switched off when predicting (the archive over-represents popular artists, so its base rate is higher). \
Training rows end `win` weeks before the test origins, so no label is seen early. Δ < 0 means better than 1-year.\n");
Training rows' labels (window plus the weeks checked after it) end before the first test origin, so no label is seen early. Δ < 0 means better than 1-year.\n");
let models = ["1-year", "all years", "all years + era", "1-year + older history", "all years + era + older history"];
let mut tests = Vec::new();
@@ -292,11 +299,11 @@ pub fn run(db: &Db, p: &Panel, tau: f64, win: usize) -> Result<String> {
let test: Vec<&(usize, usize, bool)> = all.iter().filter(|r| l.weeks[r.0] >= s && l.weeks[r.0] < e && r.0 < l.cur0).collect();
if test.iter().filter(|r| r.2).count() < 20 { continue; }
let fits = [
Fit::new(&l, &all, |t| t + win < t0 && t + HIST >= t0, win, tau, Opt { era: false, hist: false }),
Fit::new(&l, &all, |t| t + win < t0, win, tau, Opt { era: false, hist: false }),
Fit::new(&l, &all, |t| t + win < t0, win, tau, Opt { era: true, hist: false }),
Fit::new(&l, &all, |t| t + win < t0 && t + HIST >= t0, win, tau, Opt { era: false, hist: true }),
Fit::new(&l, &all, |t| t + win < t0, win, tau, Opt { era: true, hist: true }),
Fit::new(&l, &all, |t| t + win + l.stay < t0 && t + HIST >= t0, win, tau, Opt { era: false, hist: false }),
Fit::new(&l, &all, |t| t + win + l.stay < t0, win, tau, Opt { era: false, hist: false }),
Fit::new(&l, &all, |t| t + win + l.stay < t0, win, tau, Opt { era: true, hist: false }),
Fit::new(&l, &all, |t| t + win + l.stay < t0 && t + HIST >= t0, win, tau, Opt { era: false, hist: true }),
Fit::new(&l, &all, |t| t + win + l.stay < t0, win, tau, Opt { era: true, hist: true }),
];
tests.push(Test { name: year.to_string(), t: test.iter().map(|r| r.0).collect(), y: test.iter().map(|r| r.2).collect(),
p: fits.iter().map(|f| test.iter().map(|r| f.p(&l, r.1, r.0, win, tau)).collect()).collect() });
@@ -308,11 +315,11 @@ pub fn run(db: &Db, p: &Panel, tau: f64, win: usize) -> Result<String> {
while t0 + win < l.w() {
let t1 = (t0 + 4).min(l.w());
let fits = [
Fit::new(&l, &all, |t| t >= l.cur0 && t + win < t0, win, tau, Opt { era: false, hist: false }),
Fit::new(&l, &all, |t| t + win < t0, win, tau, Opt { era: false, hist: false }),
Fit::new(&l, &all, |t| t + win < t0, win, tau, Opt { era: true, hist: false }),
Fit::new(&l, &all, |t| t >= l.cur0 && t + win < t0, win, tau, Opt { era: false, hist: true }),
Fit::new(&l, &all, |t| t + win < t0, win, tau, Opt { era: true, hist: true }),
Fit::new(&l, &all, |t| t >= l.cur0 && t + win + l.stay < t0, win, tau, Opt { era: false, hist: false }),
Fit::new(&l, &all, |t| t + win + l.stay < t0, win, tau, Opt { era: false, hist: false }),
Fit::new(&l, &all, |t| t + win + l.stay < t0, win, tau, Opt { era: true, hist: false }),
Fit::new(&l, &all, |t| t >= l.cur0 && t + win + l.stay < t0, win, tau, Opt { era: false, hist: true }),
Fit::new(&l, &all, |t| t + win + l.stay < t0, win, tau, Opt { era: true, hist: true }),
];
for r in all.iter().filter(|r| r.0 >= t0 && r.0 < t1) {
cur.t.push(r.0); cur.y.push(r.2);
@@ -325,7 +332,7 @@ pub fn run(db: &Db, p: &Panel, tau: f64, win: usize) -> Result<String> {
let _ = writeln!(md, "## Out of sample\n");
report(&mut md, &models, &tests);
let f = Fit::new(&l, &all, |t| t + win < l.w(), win, tau, Opt { era: true, hist: true });
let f = Fit::new(&l, &all, |t| t + win + l.stay < l.w(), win, tau, Opt { era: true, hist: true });
let _ = writeln!(md, "\n## Coefficients (all years + era + older history, every row)\n\n| feature | β |\n|---|---|");
for (n, b) in NAMES.iter().zip(&f.b) { let _ = writeln!(md, "| {n} | {b:+.3} |"); }
Ok(md)