diff --git a/docs/LONG.md b/docs/LONG.md new file mode 100644 index 0000000..c4e1b95 --- /dev/null +++ b/docs/LONG.md @@ -0,0 +1,80 @@ +# The long any-DR model (2019–) + +Total DR plays per artist and week (all channels, P4 regions included). An *add* is τ = 20 or more plays in a week of the next 2 week(s) after 8 weeks under τ. Archive era 2019-04-08 to 2025-11-10 (15538 artists observed, 1040482 qualifying rows, 4681 adds); current panel 2025-11-17 to 2026-11-02 (315968 rows, 1229 adds). + +Models: **1-year** trains on the 52 weeks of origins before the test period (what the site can do from the current panel alone); **all years** on every earlier origin; **+ older history** adds four features from the artist's weeks before the last 52 (for today's predictions: the 2019–24 archive); **all years + era** adds an archive-era dummy, switched off when predicting (the archive over-represents popular artists, so its base rate is higher). Training rows end `win` weeks before the test origins, so no label is seen early. Δ < 0 means better than 1-year. + +## Findings (2026-11-17) + +- **Training on more years doesn't improve the model.** 2019–2024 vs. the last 52 weeks makes ±0.0003 difference in log loss in every test year and for the current panel, with the same average precision. τ = 50 gives the same picture. A model with ~30 coefficients is already pinned down by one year (300k rows, ~1,200 adds). +- **An artist's older history doesn't help today's predictions either.** The four "older history" features (archive 2019–24 for today's origins) gain at most 0.0001. +- **Pooling with the archive hurts calibration today** (mean p 0.0051 vs a rate of 0.0037), even with an era dummy. The archive over-represents artists whose pages got crawled. +- **So the one-year limit isn't what holds the model back.** Signal is: there's little to learn about *when* an artist crosses into heavy rotation beyond their current trajectory, releases, NMF and charts. Better data on upcoming releases and promo (singles sent to radio) is more promising than more history. + +## Out of sample + +| test period | rows | adds | model | log loss | Brier skill | avg precision | mean p / rate | Δ log loss vs 1-year (90% CI) | +|---|---|---|---|---|---|---|---|---| +| 2021 | 342367 | 1605 | 1-year | 0.0219 | +0.062 | 0.110 | 0.0038 / 0.0047 | — | +| 2021 | 342367 | 1605 | all years | 0.0219 | +0.061 | 0.108 | 0.0038 / 0.0047 | +0.0001 (+0.0000 to +0.0001) | +| 2021 | 342367 | 1605 | all years + era | 0.0219 | +0.061 | 0.108 | 0.0038 / 0.0047 | +0.0001 (+0.0000 to +0.0001) | +| 2021 | 342367 | 1605 | 1-year + older history | 0.0221 | +0.059 | 0.106 | 0.0034 / 0.0047 | +0.0002 (+0.0001 to +0.0004) | +| 2021 | 342367 | 1605 | all years + era + older history | 0.0222 | +0.058 | 0.105 | 0.0034 / 0.0047 | +0.0003 (+0.0002 to +0.0004) | +| 2022 | 146672 | 1098 | 1-year | 0.0329 | +0.042 | 0.109 | 0.0088 / 0.0075 | — | +| 2022 | 146672 | 1098 | all years | 0.0330 | +0.049 | 0.108 | 0.0077 / 0.0075 | +0.0001 (-0.0002 to +0.0004) | +| 2022 | 146672 | 1098 | all years + era | 0.0330 | +0.049 | 0.108 | 0.0077 / 0.0075 | +0.0001 (-0.0002 to +0.0004) | +| 2022 | 146672 | 1098 | 1-year + older history | 0.0329 | +0.045 | 0.111 | 0.0087 / 0.0075 | -0.0001 (-0.0001 to -0.0000) | +| 2022 | 146672 | 1098 | all years + era + older history | 0.0329 | +0.050 | 0.109 | 0.0080 / 0.0075 | -0.0000 (-0.0003 to +0.0003) | +| 2023 | 134721 | 549 | 1-year | 0.0195 | +0.043 | 0.111 | 0.0062 / 0.0041 | — | +| 2023 | 134721 | 549 | all years | 0.0193 | +0.040 | 0.111 | 0.0061 / 0.0041 | -0.0002 (-0.0003 to +0.0001) | +| 2023 | 134721 | 549 | all years + era | 0.0193 | +0.040 | 0.111 | 0.0061 / 0.0041 | -0.0002 (-0.0003 to +0.0001) | +| 2023 | 134721 | 549 | 1-year + older history | 0.0197 | +0.033 | 0.099 | 0.0064 / 0.0041 | +0.0002 (+0.0001 to +0.0003) | +| 2023 | 134721 | 549 | all years + era + older history | 0.0194 | +0.036 | 0.101 | 0.0061 / 0.0041 | -0.0001 (-0.0002 to +0.0002) | +| 2024 | 20378 | 107 | 1-year | 0.0220 | +0.097 | 0.179 | 0.0055 / 0.0053 | — | +| 2024 | 20378 | 107 | all years | 0.0223 | +0.095 | 0.179 | 0.0067 / 0.0053 | +0.0003 (+0.0001 to +0.0009) | +| 2024 | 20378 | 107 | all years + era | 0.0223 | +0.095 | 0.179 | 0.0067 / 0.0053 | +0.0003 (+0.0001 to +0.0009) | +| 2024 | 20378 | 107 | 1-year + older history | 0.0218 | +0.101 | 0.186 | 0.0056 / 0.0053 | -0.0002 (-0.0004 to -0.0001) | +| 2024 | 20378 | 107 | all years + era + older history | 0.0225 | +0.088 | 0.164 | 0.0066 / 0.0053 | +0.0005 (+0.0003 to +0.0010) | +| current panel, origins from 2026-05-18 | 173697 | 641 | 1-year | 0.0163 | +0.069 | 0.128 | 0.0034 / 0.0037 | — | +| current panel, origins from 2026-05-18 | 173697 | 641 | all years | 0.0164 | +0.072 | 0.125 | 0.0037 / 0.0037 | +0.0001 (-0.0000 to +0.0002) | +| current panel, origins from 2026-05-18 | 173697 | 641 | all years + era | 0.0166 | +0.065 | 0.126 | 0.0051 / 0.0037 | +0.0003 (+0.0001 to +0.0005) | +| current panel, origins from 2026-05-18 | 173697 | 641 | 1-year + older history | 0.0162 | +0.069 | 0.129 | 0.0033 / 0.0037 | -0.0000 (-0.0001 to +0.0001) | +| current panel, origins from 2026-05-18 | 173697 | 641 | all years + era + older history | 0.0164 | +0.065 | 0.126 | 0.0048 / 0.0037 | +0.0002 (-0.0000 to +0.0003) | + +## Coefficients (all years + era + older history, every row) + +| feature | β | +|---|---| +| intercept | -7.944 | +| log plays week t | +0.720 | +| log mean plays last 4w | +0.329 | +| log mean plays last 12w | +1.063 | +| log max weekly plays last 52w | +0.583 | +| on DR 9-25 weeks ago | +0.434 | +| on DR 26-52 weeks ago | -0.122 | +| weeks observed of last 52 (share) | +0.364 | +| weeks with plays of last 12 (share) | -2.339 | +| trend: last 4w vs the 8 before | +0.511 | +| discography known | +0.256 | +| release in last 4w | +0.405 | +| release 5-12w ago | +0.187 | +| release in the window | +0.381 | +| log releases last 52w | -0.163 | +| on Track Top-40 now | +0.587 | +| on Airplay Top-20 now | +0.546 | +| log Top-40 hits so far | +0.035 | +| log Airplay hits so far | +0.094 | +| new chart entry in last 4w | +0.078 | +| NMF data covers t | -0.397 | +| NMF Denmark in last 4w | +0.920 | +| NMF Denmark in the window | +1.103 | +| log NMF placements last 52w | -0.274 | +| Danish | +0.065 | +| group | +0.099 | +| female | +0.284 | +| season index of the window | +0.019 | +| archive era | -0.204 | +| older history: observed before the last 52w | +0.226 | +| older history: log max weekly plays | -0.034 | +| older history: log adds | +0.276 | +| older history: share of weeks on DR | -1.115 | diff --git a/src/model/long.rs b/src/model/long.rs new file mode 100644 index 0000000..35e9302 --- /dev/null +++ b/src/model/long.rs @@ -0,0 +1,332 @@ +//! The long any-DR model: total DR plays per artist and week (all channels, P4 regions included) +//! from 2019 on. 2019–2024 come from archived artist pages (Wayback, `dr_artist_history`): each +//! capture covers the 52 weeks before it, so an artist-week is observed only inside a capture's +//! window. 2025– comes from the current panel (every pool artist, every week). +//! +//! Target, per origin week t: the artist is off DR (under τ plays in each of the 8 weeks to t) and +//! gets τ or more plays in a week of the window t+1..t+win. Rows need all those weeks observed. +//! +//! airplay model long [--tau 20] [--w 2] [--out docs/LONG.md] +//! +//! It answers: does training on 2019–2024 beat training on the last year only, (1) for each +//! archived year and (2) for the current year, which is what the site predicts? + +use super::common::mix; +use super::glm::{self, Design}; +use super::panel::{chart_states, key, load_nmf_dk, load_releases, Artist, Chart, Panel, C}; +use crate::db::Db; +use anyhow::Result; +use chrono::{Datelike, Duration, NaiveDate}; +use std::collections::HashMap; +use std::fmt::Write; + +const OFF: usize = 8; // weeks off DR before an add +const HIST: usize = 52; +const NMF_START: &str = "2022-01-07"; + +pub struct Long { + weeks: Vec, + artists: Vec, + y: Vec, + obs: Vec, + /// First week of the current panel (weeks before are the archive era). + cur0: usize, + /// Current era: the week the artist was knowable (the site's pool rule); old-only artists: never. + known: Vec, + chart: Vec, + releases: Vec>>, + nmf: Vec)>>, + season: [f64; 54], +} + +impl Long { + fn w(&self) -> usize { self.weeks.len() } + fn i(&self, a: usize, w: usize) -> usize { a * self.weeks.len() + w } + fn y(&self, a: usize, w: usize) -> f64 { self.y[self.i(a, w)] as f64 } + fn obs(&self, a: usize, w: usize) -> bool { self.obs[self.i(a, w)] } + + pub fn load(db: &Db, p: &Panel) -> Result { + let first: NaiveDate = db.conn.query_row("SELECT min(week_start) FROM dr_artist_history", [], |r| r.get::<_, String>(0))?.parse()?; + let mut weeks = Vec::new(); + let mut d = first; + while d < p.weeks[0] { weeks.push(d); d += Duration::weeks(1); } + let cur0 = weeks.len(); + weeks.extend(p.weeks.iter().copied()); + let week_of: HashMap = weeks.iter().enumerate().map(|(i, d)| (*d, i)).collect(); + let nw = weeks.len(); + + // Artists: the panel's, then the archive-only ones. + let mut artists = p.artists.clone(); + let mut id_of: HashMap = artists.iter().enumerate().map(|(i, a)| (a.id.clone(), i)).collect(); + for r in db.conn.prepare("SELECT artist_id, coalesce(name, ''), coalesce(country, ''), coalesce(gender, ''), coalesce(artist_type, '') FROM old_artist_meta")? + .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, String>(3)?, r.get::<_, String>(4)?)))? { + let (id, name, country, gender, kind) = r?; + if let Some(&a) = id_of.get(&id) { + // Panel artists keep their current page; fill what it lacks. + if !artists[a].danish && country == "Danmark" { artists[a].danish = true; } + continue; + } + id_of.insert(id.clone(), artists.len()); + artists.push(Artist { id, name, danish: country == "Danmark", group: kind.starts_with("orkester"), female: gender == "kvinde", debut: None }); + } + let na = artists.len(); + let mut y = vec![0f32; na * nw]; + let mut obs = vec![false; na * nw]; + + // Archive era: each fetched capture observes the 52 complete weeks before it. + for r in db.conn.prepare("SELECT artist_id, snapshot FROM old_artist_captures WHERE fetched_at IS NOT NULL AND weeks > 0")? + .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? { + let (id, snap) = r?; + let (Some(&a), Ok(day)) = (id_of.get(&id), NaiveDate::parse_from_str(&snap[..8], "%Y%m%d")) else { continue }; + let monday = day - Duration::days(day.weekday().num_days_from_monday() as i64); + for k in 1..=HIST as i64 { + if let Some(&w) = week_of.get(&(monday - Duration::weeks(k))) { if w < cur0 { obs[a * nw + w] = true; } } + } + } + for r in db.conn.prepare("SELECT artist_id, week_start, plays FROM dr_artist_history")? + .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, i64>(2)?)))? { + let (id, ws, n) = r?; + let (Some(&a), Ok(d)) = (id_of.get(&id), ws.parse::()) else { continue }; + if let Some(&w) = week_of.get(&d) { if w < cur0 && obs[a * nw + w] { y[a * nw + w] = n as f32; } } + } + // Current era: the panel, all channels summed. + let mut known = vec![usize::MAX; na]; + for a in 0..p.a() { + known[a] = cur0 + p.first_known[a]; + for w in 0..p.w() { + y[a * nw + cur0 + w] = (0..C).map(|c| p.y(a, c, w)).sum::() as f32; + obs[a * nw + cur0 + w] = true; + } + } + + let mut names: HashMap> = HashMap::new(); + for (i, a) in artists.iter().enumerate() { let k = key(&a.name); if !k.is_empty() { names.entry(k).or_default().push(i); } } + let chart = chart_states(db, &weeks, &names, na)?; + let releases = load_releases(db, &id_of, na)?; + let nmf = load_nmf_dk(db, &id_of, &artists)?; + Ok(Long { weeks, artists, y, obs, cur0, known, chart, releases, nmf, season: p.season }) + } + + /// Does origin (a, t) qualify, and if so was there an add in the window? + fn label(&self, a: usize, t: usize, win: usize, tau: f64) -> Option { + if t + 1 < OFF || t + win >= self.w() { return None; } + if t >= self.cur0 && self.known[a] > t { return None; } + // Don't let a window straddle the gap between the eras. + if t < self.cur0 && t + win >= self.cur0 { return None; } + if !(t + 1 - OFF..=t + win).all(|w| self.obs(a, w)) { return None; } + if (t + 1 - OFF..=t).any(|w| self.y(a, w) >= tau) { return None; } + Some((t + 1..=t + win).any(|w| self.y(a, w) >= tau)) + } +} + +pub const NAMES: &[&str] = &[ + "intercept", "log plays week t", "log mean plays last 4w", "log mean plays last 12w", "log max weekly plays last 52w", + "on DR 9-25 weeks ago", "on DR 26-52 weeks ago", "weeks observed of last 52 (share)", "weeks with plays of last 12 (share)", + "trend: last 4w vs the 8 before", "discography known", "release in last 4w", "release 5-12w ago", "release in the window", + "log releases last 52w", "on Track Top-40 now", "on Airplay Top-20 now", "log Top-40 hits so far", "log Airplay hits so far", + "new chart entry in last 4w", "NMF data covers t", "NMF Denmark in last 4w", "NMF Denmark in the window", "log NMF placements last 52w", + "Danish", "group", "female", "season index of the window", "archive era", + "older history: observed before the last 52w", "older history: log max weekly plays", "older history: log adds", "older history: share of weeks on DR", +]; +pub const P: usize = 33; + +#[derive(Clone, Copy)] +struct Opt { era: bool, hist: bool } + +fn feats(l: &Long, a: usize, t: usize, win: usize, tau: f64, o: Opt, x: &mut [f64; P]) { + let ob = |w: usize| l.obs(a, w); + let mean = |from: usize, to: usize| -> f64 { + let v: Vec = (from..=to).filter(|&w| ob(w)).map(|w| l.y(a, w)).collect(); + if v.is_empty() { 0.0 } else { v.iter().sum::() / v.len() as f64 } + }; + let lo = |k: usize| t.saturating_sub(k - 1); + let last_on = (lo(HIST)..=t).rev().find(|&w| ob(w) && l.y(a, w) >= tau).map(|w| t - w); + let observed = (lo(HIST)..=t).filter(|&w| ob(w)).count() as f64 / HIST as f64; + let nz = (lo(12)..=t).filter(|&w| ob(w) && l.y(a, w) > 0.0).count() as f64 / 12.0; + let max52 = (lo(HIST)..=t).filter(|&w| ob(w)).map(|w| l.y(a, w)).fold(0.0, f64::max); + let (m4, m12) = (mean(lo(4), t), mean(lo(12), t)); + let prev8 = if t >= 4 { mean(lo(12), t - 4) } else { 0.0 }; + + let end_t = l.weeks[t] + Duration::days(6); + let win_end = l.weeks[t + win] + Duration::days(6); + let rel = l.releases[a].as_ref(); + let cnt = |from: NaiveDate, to: NaiveDate| rel.map_or(0, |r| r.iter().filter(|d| **d > from && **d <= to).count()) as f64; + let ch = l.chart[l.i(a, t)]; + let nmf_in = |from: NaiveDate, to: NaiveDate| l.nmf[a].iter().filter(|(d, _)| *d > from && *d <= to).count() as f64; + let nmf_ok = l.weeks[t] >= NMF_START.parse::().unwrap(); + let season = (1..=win).map(|k| l.season[l.weeks[t + k].iso_week().week() as usize].ln()).sum::() / win as f64; + + *x = [ + 1.0, l.y(a, t).ln_1p(), m4.ln_1p(), m12.ln_1p(), max52.ln_1p(), + last_on.is_some_and(|k| k < 26) as i32 as f64, last_on.is_some_and(|k| k >= 26) as i32 as f64, observed, nz, + m4.ln_1p() - prev8.ln_1p(), rel.is_some() as i32 as f64, + cnt(end_t - Duration::weeks(4), end_t).min(1.0), cnt(end_t - Duration::weeks(12), end_t - Duration::weeks(4)).min(1.0), + cnt(end_t, win_end).min(1.0), cnt(end_t - Duration::weeks(52), end_t).ln_1p(), + (ch.t40_rank > 0) as i32 as f64, (ch.air_rank > 0) as i32 as f64, (ch.t40_hits as f64).ln_1p(), (ch.air_hits as f64).ln_1p(), + (ch.weeks_since_entry <= 4) as i32 as f64, + nmf_ok as i32 as f64, nmf_in(end_t - Duration::weeks(4), end_t).min(1.0), nmf_in(end_t, win_end).min(1.0), + nmf_in(end_t - Duration::weeks(52), end_t).ln_1p(), + l.artists[a].danish as i32 as f64, l.artists[a].group as i32 as f64, l.artists[a].female as i32 as f64, season, + (o.era && t < l.cur0) as i32 as f64, 0.0, 0.0, 0.0, 0.0, + ]; + // What the artist did before the last 52 weeks (the archive, for current-panel origins). + if o.hist && t >= HIST { + let old: Vec = (0..t + 1 - HIST).filter(|&w| ob(w)).collect(); + if !old.is_empty() { + let on = |w: usize| l.y(a, w) >= tau; + let adds = old.iter().filter(|&&w| on(w) && (w == 0 || !ob(w - 1) || !on(w - 1))).count() as f64; + x[29] = 1.0; + x[30] = old.iter().map(|&w| l.y(a, w)).fold(0.0, f64::max).ln_1p(); + x[31] = adds.ln_1p(); + x[32] = old.iter().filter(|&&w| on(w)).count() as f64 / old.len() as f64; + } + } +} + +/// Every qualifying row as (origin week, artist, label). +fn rows(l: &Long, win: usize, tau: f64) -> Vec<(usize, usize, bool)> { + let mut v = Vec::new(); + for t in 0..l.w() { for a in 0..l.artists.len() { if let Some(y) = l.label(a, t, win, tau) { v.push((t, a, y)); } } } + v +} + +struct Fit { b: Vec, o: Opt } +impl Fit { + fn new(l: &Long, rows: &[(usize, usize, bool)], keep: impl Fn(usize) -> bool, win: usize, tau: f64, o: Opt) -> Fit { + let mut d = Design::new(P); + let mut x = [0.0; P]; + for &(t, a, y) in rows.iter().filter(|r| keep(r.0)) { + feats(l, a, t, win, tau, o, &mut x); + d.push(&x, y as i32 as f64, 1.0); + } + Fit { b: glm::logit(&d, 1.0), o } + } + fn p(&self, l: &Long, a: usize, t: usize, win: usize, tau: f64) -> f64 { + let mut x = [0.0; P]; + feats(l, a, t, win, tau, Opt { era: false, ..self.o }, &mut x); // predictions: era dummy off (current-day base rate) + glm::sigmoid(glm::dot(&self.b, &x)).clamp(1e-6, 1.0 - 1e-6) + } +} + +fn ll(p: f64, y: bool) -> f64 { -(if y { p } else { 1.0 - p }).ln() } +fn ap(mut v: Vec<(f64, bool)>) -> f64 { + v.sort_by(|a, b| b.0.total_cmp(&a.0)); + let (mut hits, mut s) = (0.0, 0.0); + for (i, (_, y)) in v.iter().enumerate() { if *y { hits += 1.0; s += hits / (i + 1) as f64; } } + if hits > 0.0 { s / hits } else { 0.0 } +} + +/// Scores for one test period: per model, the predictions on the same rows. +struct Test { name: String, t: Vec, y: Vec, p: Vec> } + +fn report(md: &mut String, models: &[&str], tests: &[Test]) { + let _ = writeln!(md, "| test period | rows | adds | model | log loss | Brier skill | avg precision | mean p / rate | Δ log loss vs 1-year (90% CI) |"); + let _ = writeln!(md, "|---|---|---|---|---|---|---|---|---|"); + for te in tests { + let n = te.y.len() as f64; + let rate = te.y.iter().filter(|y| **y).count() as f64 / n; + let brier_ref = rate * (1.0 - rate); + for (m, name) in models.iter().enumerate() { + let p = &te.p[m]; + let l: f64 = p.iter().zip(&te.y).map(|(p, y)| ll(*p, *y)).sum::() / n; + let b: f64 = p.iter().zip(&te.y).map(|(p, y)| (p - *y as i32 as f64).powi(2)).sum::() / n; + let mp = p.iter().sum::() / n; + let delta = if m == 0 { "—".to_string() } else { + let (d, lo, hi) = boot(te, m, 0); + format!("{:+.4} ({:+.4} to {:+.4})", d, lo, hi) + }; + let _ = writeln!(md, "| {} | {} | {} | {} | {:.4} | {:+.3} | {:.3} | {:.4} / {:.4} | {} |", te.name, te.y.len(), te.y.iter().filter(|y| **y).count(), + name, l, 1.0 - b / brier_ref, ap(p.iter().copied().zip(te.y.iter().copied()).collect()), mp, rate, delta); + } + } +} + +/// Mean log-loss difference (model m minus model base) with a moving-block bootstrap over 8-week blocks. +fn boot(te: &Test, m: usize, base: usize) -> (f64, f64, f64) { + let mut by: std::collections::BTreeMap = std::collections::BTreeMap::new(); + for i in 0..te.y.len() { + let e = by.entry(te.t[i]).or_insert((0.0, 0.0)); + e.0 += ll(te.p[m][i], te.y[i]) - ll(te.p[base][i], te.y[i]); + e.1 += 1.0; + } + let weeks: Vec<(f64, f64)> = by.values().copied().collect(); + let d = weeks.iter().map(|w| w.0).sum::() / weeks.iter().map(|w| w.1).sum::(); + let nb = weeks.len(); + let blk = 8.min(nb); + let mut s = 0x5eedu64; + let mut v: Vec = (0..1000).map(|_| { + let (mut num, mut den) = (0.0, 0.0); + for _ in 0..nb.div_ceil(blk) { + s = mix(s); + let start = (s % (nb - blk + 1) as u64) as usize; + for w in &weeks[start..start + blk] { num += w.0; den += w.1; } + } + num / den + }).collect(); + v.sort_by(f64::total_cmp); + (d, v[50], v[949]) +} + +pub fn run(db: &Db, p: &Panel, tau: f64, win: usize) -> Result { + let l = Long::load(db, p)?; + let all = rows(&l, win, tau); + let mut md = String::new(); + let n_old = all.iter().filter(|r| r.0 < l.cur0).count(); + let a_old = all.iter().filter(|r| r.0 < l.cur0 && r.2).count(); + let _ = writeln!(md, "# The long any-DR model (2019–)\n"); + let _ = writeln!(md, "Total DR plays per artist and week (all channels, P4 regions included). An *add* is τ = {tau} or more plays in a week of the next {win} week(s) after {OFF} weeks under τ. \ + Archive era {} to {} ({} artists observed, {} qualifying rows, {} adds); current panel {} to {} ({} rows, {} adds).\n", + l.weeks[0], l.weeks[l.cur0 - 1], (0..l.artists.len()).filter(|&a| (0..l.cur0).any(|w| l.obs(a, w))).count(), n_old, a_old, + l.weeks[l.cur0], l.weeks[l.w() - 1], all.len() - n_old, all.iter().filter(|r| r.0 >= l.cur0 && r.2).count()); + let _ = writeln!(md, "Models: **1-year** trains on the 52 weeks of origins before the test period (what the site can do from the current panel alone); \ + **all years** on every earlier origin; **+ older history** adds four features from the artist's weeks before the last 52 (for today's predictions: the 2019–24 archive); **all years + era** adds an archive-era dummy, switched off when predicting (the archive over-represents popular artists, so its base rate is higher). \ + Training rows end `win` weeks before the test origins, so no label is seen early. Δ < 0 means better than 1-year.\n"); + + let models = ["1-year", "all years", "all years + era", "1-year + older history", "all years + era + older history"]; + let mut tests = Vec::new(); + // (1) each archived calendar year (2020–2024), refit once at the start of the year. + for year in 2020..=2024 { + let (s, e) = (NaiveDate::from_ymd_opt(year, 1, 1).unwrap(), NaiveDate::from_ymd_opt(year + 1, 1, 1).unwrap()); + let Some(t0) = l.weeks.iter().position(|d| *d >= s) else { continue }; + if t0 < HIST + win { continue; } + let test: Vec<&(usize, usize, bool)> = all.iter().filter(|r| l.weeks[r.0] >= s && l.weeks[r.0] < e && r.0 < l.cur0).collect(); + if test.iter().filter(|r| r.2).count() < 20 { continue; } + let fits = [ + Fit::new(&l, &all, |t| t + win < t0 && t + HIST >= t0, win, tau, Opt { era: false, hist: false }), + Fit::new(&l, &all, |t| t + win < t0, win, tau, Opt { era: false, hist: false }), + Fit::new(&l, &all, |t| t + win < t0, win, tau, Opt { era: true, hist: false }), + Fit::new(&l, &all, |t| t + win < t0 && t + HIST >= t0, win, tau, Opt { era: false, hist: true }), + Fit::new(&l, &all, |t| t + win < t0, win, tau, Opt { era: true, hist: true }), + ]; + tests.push(Test { name: year.to_string(), t: test.iter().map(|r| r.0).collect(), y: test.iter().map(|r| r.2).collect(), + p: fits.iter().map(|f| test.iter().map(|r| f.p(&l, r.1, r.0, win, tau)).collect()).collect() }); + eprintln!("long: {year} done"); + } + // (2) the current panel: second half, refit every 4 weeks. 1-year = current panel only. + let mut cur = Test { name: format!("current panel, origins from {}", l.weeks[l.cur0 + 26]), t: vec![], y: vec![], p: vec![vec![]; 5] }; + let mut t0 = l.cur0 + 26; + while t0 + win < l.w() { + let t1 = (t0 + 4).min(l.w()); + let fits = [ + Fit::new(&l, &all, |t| t >= l.cur0 && t + win < t0, win, tau, Opt { era: false, hist: false }), + Fit::new(&l, &all, |t| t + win < t0, win, tau, Opt { era: false, hist: false }), + Fit::new(&l, &all, |t| t + win < t0, win, tau, Opt { era: true, hist: false }), + Fit::new(&l, &all, |t| t >= l.cur0 && t + win < t0, win, tau, Opt { era: false, hist: true }), + Fit::new(&l, &all, |t| t + win < t0, win, tau, Opt { era: true, hist: true }), + ]; + for r in all.iter().filter(|r| r.0 >= t0 && r.0 < t1) { + cur.t.push(r.0); cur.y.push(r.2); + for (m, f) in fits.iter().enumerate() { cur.p[m].push(f.p(&l, r.1, r.0, win, tau)); } + } + eprintln!("long: current from {} done", l.weeks[t0]); + t0 = t1; + } + tests.push(cur); + let _ = writeln!(md, "## Out of sample\n"); + report(&mut md, &models, &tests); + + let f = Fit::new(&l, &all, |t| t + win < l.w(), win, tau, Opt { era: true, hist: true }); + let _ = writeln!(md, "\n## Coefficients (all years + era + older history, every row)\n\n| feature | β |\n|---|---|"); + for (n, b) in NAMES.iter().zip(&f.b) { let _ = writeln!(md, "| {n} | {b:+.3} |"); } + Ok(md) +} diff --git a/src/model/mod.rs b/src/model/mod.rs index bd9e2c6..8b04570 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -9,6 +9,7 @@ //! airplay model ifpi the 20-year IFPI test bed -> docs/IFPI.md //! airplay model events [--out docs/BACKTEST-EVENTS.md] backtest of the release-event model //! airplay model adds [--h 0,4] [--out docs/BACKTEST-ADDS.md] backtest of the new-add model +//! airplay model long [--tau 20] [--w 2] [--out docs/LONG.md] 2019– any-DR model, all years vs 1 year //! airplay model rank [--h 4] [--max-h 12] [--top 50] pub mod addcheck; @@ -22,6 +23,7 @@ pub mod glm; pub mod ifpi; pub mod kalman; pub mod leakcheck; +pub mod long; pub mod nmf; pub mod panel; pub mod rank; @@ -89,6 +91,13 @@ pub fn run(db: &Db, args: &[String]) -> Result<()> { if let Some(out) = arg(args, "--out") { std::fs::write(out, &md)?; } print!("{md}"); } + Some("long") => { + let tau: f64 = arg(args, "--tau").and_then(|s| s.parse().ok()).unwrap_or(20.0); + let win: usize = arg(args, "--w").and_then(|s| s.parse().ok()).unwrap_or(2); + let md = long::run(db, &p, tau, win)?; + if let Some(out) = arg(args, "--out") { std::fs::write(out, &md)?; } + print!("{md}"); + } Some("leakcheck") => { print!("{}", leakcheck::run(&p)?); } Some("addcheck") => { if let Some(r) = arg(args, "--rule") { let _ = adds::ADD_RULE.set(r); } diff --git a/src/model/panel.rs b/src/model/panel.rs index 3a30c3a..34b32bf 100644 --- a/src/model/panel.rs +++ b/src/model/panel.rs @@ -213,7 +213,7 @@ fn chart_artists(s: &str, names: &HashMap>) -> Vec { found } -fn chart_states(db: &Db, weeks: &[NaiveDate], names: &HashMap>, n_artists: usize) -> Result> { +pub(crate) fn chart_states(db: &Db, weeks: &[NaiveDate], names: &HashMap>, n_artists: usize) -> Result> { struct Entry { monday: NaiveDate, t40: bool, rank: u8, new: bool, song: String } let mut by_artist: Vec> = (0..n_artists).map(|_| Vec::new()).collect(); let mut mondays: [BTreeSet; 2] = [BTreeSet::new(), BTreeSet::new()]; @@ -273,7 +273,7 @@ fn chart_states(db: &Db, weeks: &[NaiveDate], names: &HashMap Ok(out) } -fn load_releases(db: &Db, id_of: &HashMap, n: usize) -> Result>>> { +pub(crate) fn load_releases(db: &Db, id_of: &HashMap, n: usize) -> Result>>> { let mut out: Vec>> = vec![None; n]; let exists: i64 = db.conn.query_row("SELECT count(*) FROM sqlite_master WHERE name = 'deezer_artists'", [], |r| r.get(0))?; if exists == 0 { return Ok(out); } @@ -321,7 +321,7 @@ fn load_long_history(db: &Db, id_of: &HashMap, names: &HashMap, artists: &[Artist]) -> Result)>>> { +pub(crate) fn load_nmf_dk(db: &Db, id_of: &HashMap, artists: &[Artist]) -> Result)>>> { let mut out: Vec)>> = vec![Vec::new(); artists.len()]; let has = |t: &str| -> Result { Ok(db.conn.query_row("SELECT count(*) FROM sqlite_master WHERE name = ?1", [t], |r| r.get::<_, i64>(0))? > 0) }; if !has("nmf_artist_tracks")? || !has("nmf_additions")? { return Ok(out); }