432 lines
25 KiB
Rust
432 lines
25 KiB
Rust
//! The long any-DR model: total DR plays per artist and week (all channels, P4 regions included)
|
||
//! from 2019 on. 2019–2024 come from archived artist pages (Wayback, `dr_artist_history`): each
|
||
//! capture covers the 52 weeks before it, so an artist-week is observed only inside a capture's
|
||
//! window. 2025– comes from the current panel (every pool artist, every week).
|
||
//!
|
||
//! Target, per origin week t: the artist is off DR (under τ plays in each of the 8 weeks to t) and
|
||
//! gets τ or more plays in a week of the window t+1..t+win. Rows need all those weeks observed.
|
||
//!
|
||
//! airplay model long [--tau 20] [--w 2] [--out docs/LONG.md]
|
||
//!
|
||
//! It answers: does training on 2019–2024 beat training on the last year only, (1) for each
|
||
//! archived year and (2) for the current year, which is what the site predicts?
|
||
|
||
use super::common::mix;
|
||
use super::glm::{self, Design};
|
||
use super::panel::{chart_states, key, load_nmf_dk, load_releases, Artist, Chart, Panel, C};
|
||
use crate::db::Db;
|
||
use anyhow::Result;
|
||
use chrono::{Datelike, Duration, NaiveDate};
|
||
use std::collections::HashMap;
|
||
use std::fmt::Write;
|
||
|
||
const OFF: usize = 8; // weeks off DR before an add
|
||
const HIST: usize = 52;
|
||
const NMF_START: &str = "2022-01-07";
|
||
|
||
pub struct Long {
|
||
weeks: Vec<NaiveDate>,
|
||
artists: Vec<Artist>,
|
||
y: Vec<f32>,
|
||
obs: Vec<bool>,
|
||
/// First week of the current panel (weeks before are the archive era).
|
||
cur0: usize,
|
||
/// Current era: the week the artist was knowable (the site's pool rule); old-only artists: never.
|
||
known: Vec<usize>,
|
||
chart: Vec<Chart>,
|
||
releases: Vec<Option<Vec<NaiveDate>>>,
|
||
nmf: Vec<Vec<(NaiveDate, Option<i64>)>>,
|
||
season: [f64; 54],
|
||
/// Weeks after the add that the label looks at (see `label`).
|
||
pub stay: usize,
|
||
}
|
||
|
||
impl Long {
|
||
fn w(&self) -> usize { self.weeks.len() }
|
||
fn i(&self, a: usize, w: usize) -> usize { a * self.weeks.len() + w }
|
||
fn y(&self, a: usize, w: usize) -> f64 { self.y[self.i(a, w)] as f64 }
|
||
fn obs(&self, a: usize, w: usize) -> bool { self.obs[self.i(a, w)] }
|
||
|
||
pub fn load(db: &Db, p: &Panel) -> Result<Long> {
|
||
let first: NaiveDate = db.conn.query_row("SELECT min(week_start) FROM dr_artist_history", [], |r| r.get::<_, String>(0))?.parse()?;
|
||
let mut weeks = Vec::new();
|
||
let mut d = first;
|
||
while d < p.weeks[0] { weeks.push(d); d += Duration::weeks(1); }
|
||
let cur0 = weeks.len();
|
||
weeks.extend(p.weeks.iter().copied());
|
||
let week_of: HashMap<NaiveDate, usize> = weeks.iter().enumerate().map(|(i, d)| (*d, i)).collect();
|
||
let nw = weeks.len();
|
||
|
||
// Artists: the panel's, then the archive-only ones.
|
||
let mut artists = p.artists.clone();
|
||
let mut id_of: HashMap<String, usize> = artists.iter().enumerate().map(|(i, a)| (a.id.clone(), i)).collect();
|
||
for r in db.conn.prepare("SELECT artist_id, coalesce(name, ''), coalesce(country, ''), coalesce(gender, ''), coalesce(artist_type, '') FROM old_artist_meta")?
|
||
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, String>(3)?, r.get::<_, String>(4)?)))? {
|
||
let (id, name, country, gender, kind) = r?;
|
||
if let Some(&a) = id_of.get(&id) {
|
||
// Panel artists keep their current page; fill what it lacks.
|
||
if !artists[a].danish && country == "Danmark" { artists[a].danish = true; }
|
||
continue;
|
||
}
|
||
id_of.insert(id.clone(), artists.len());
|
||
artists.push(Artist { id, name, danish: country == "Danmark", group: kind.starts_with("orkester"), female: gender == "kvinde", debut: None });
|
||
}
|
||
let na = artists.len();
|
||
let mut y = vec![0f32; na * nw];
|
||
let mut obs = vec![false; na * nw];
|
||
|
||
// Archive era: each fetched capture observes the 52 complete weeks before it.
|
||
for r in db.conn.prepare("SELECT artist_id, snapshot FROM old_artist_captures WHERE fetched_at IS NOT NULL AND weeks > 0")?
|
||
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? {
|
||
let (id, snap) = r?;
|
||
let (Some(&a), Ok(day)) = (id_of.get(&id), NaiveDate::parse_from_str(&snap[..8], "%Y%m%d")) else { continue };
|
||
let monday = day - Duration::days(day.weekday().num_days_from_monday() as i64);
|
||
for k in 1..=HIST as i64 {
|
||
if let Some(&w) = week_of.get(&(monday - Duration::weeks(k))) { if w < cur0 { obs[a * nw + w] = true; } }
|
||
}
|
||
}
|
||
for r in db.conn.prepare("SELECT artist_id, week_start, plays FROM dr_artist_history")?
|
||
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, i64>(2)?)))? {
|
||
let (id, ws, n) = r?;
|
||
let (Some(&a), Ok(d)) = (id_of.get(&id), ws.parse::<NaiveDate>()) else { continue };
|
||
if let Some(&w) = week_of.get(&d) { if w < cur0 && obs[a * nw + w] { y[a * nw + w] = n as f32; } }
|
||
}
|
||
// Current era: the panel, all channels summed.
|
||
let mut known = vec![usize::MAX; na];
|
||
for a in 0..p.a() {
|
||
known[a] = cur0 + p.first_known[a];
|
||
for w in 0..p.w() {
|
||
y[a * nw + cur0 + w] = (0..C).map(|c| p.y(a, c, w)).sum::<f64>() as f32;
|
||
obs[a * nw + cur0 + w] = true;
|
||
}
|
||
}
|
||
|
||
let mut names: HashMap<String, Vec<usize>> = HashMap::new();
|
||
for (i, a) in artists.iter().enumerate() { let k = key(&a.name); if !k.is_empty() { names.entry(k).or_default().push(i); } }
|
||
let chart = chart_states(db, &weeks, &names, na)?;
|
||
let releases = load_releases(db, &id_of, na)?;
|
||
let nmf = load_nmf_dk(db, &id_of, &artists)?;
|
||
Ok(Long { weeks, artists, y, obs, cur0, known, chart, releases, nmf, season: p.season, stay: 3 })
|
||
}
|
||
|
||
/// Does origin (a, t) qualify, and if so was there an add in the window?
|
||
fn label(&self, a: usize, t: usize, win: usize, tau: f64) -> Option<bool> {
|
||
let end = t + win + self.stay;
|
||
if t + 1 < OFF || end >= self.w() { return None; }
|
||
if t >= self.cur0 && self.known[a] > t { return None; }
|
||
// Don't let a window straddle the gap between the eras.
|
||
if t < self.cur0 && end >= self.cur0 { return None; }
|
||
if !(t + 1 - OFF..=end).all(|w| self.obs(a, w)) { return None; }
|
||
if (t + 1 - OFF..=t).any(|w| self.y(a, w) >= tau) { return None; }
|
||
// stay = 0: any week at τ (a blip counts). stay = k: and τ again in at least 2 of the k weeks after it.
|
||
let on = |w: usize| self.y(a, w) >= tau;
|
||
Some((t + 1..=t + win).any(|w| on(w) && (self.stay == 0 || (w + 1..=w + self.stay).filter(|&v| on(v)).count() >= 2.min(self.stay))))
|
||
}
|
||
}
|
||
|
||
pub const NAMES: &[&str] = &[
|
||
"intercept", "log plays week t", "log mean plays last 4w", "log mean plays last 12w", "log max weekly plays last 52w",
|
||
"on DR 9-25 weeks ago", "on DR 26-52 weeks ago", "weeks observed of last 52 (share)", "weeks with plays of last 12 (share)",
|
||
"trend: last 4w vs the 8 before", "discography known", "release in last 4w", "release 5-12w ago", "release in the window",
|
||
"log releases last 52w", "on Track Top-40 now", "on Airplay Top-20 now", "log Top-40 hits so far", "log Airplay hits so far",
|
||
"new chart entry in last 4w", "NMF data covers t", "NMF Denmark in last 4w", "NMF Denmark in the window", "log NMF placements last 52w",
|
||
"Danish", "group", "female", "season index of the window", "archive era",
|
||
"older history: observed before the last 52w", "older history: log max weekly plays", "older history: log adds", "older history: share of weeks on DR",
|
||
];
|
||
pub const P: usize = 33;
|
||
|
||
#[derive(Clone, Copy)]
|
||
struct Opt { era: bool, hist: bool }
|
||
|
||
fn feats(l: &Long, a: usize, t: usize, win: usize, tau: f64, o: Opt, x: &mut [f64; P]) {
|
||
let ob = |w: usize| l.obs(a, w);
|
||
let mean = |from: usize, to: usize| -> f64 {
|
||
let v: Vec<f64> = (from..=to).filter(|&w| ob(w)).map(|w| l.y(a, w)).collect();
|
||
if v.is_empty() { 0.0 } else { v.iter().sum::<f64>() / v.len() as f64 }
|
||
};
|
||
let lo = |k: usize| t.saturating_sub(k - 1);
|
||
let last_on = (lo(HIST)..=t).rev().find(|&w| ob(w) && l.y(a, w) >= tau).map(|w| t - w);
|
||
let observed = (lo(HIST)..=t).filter(|&w| ob(w)).count() as f64 / HIST as f64;
|
||
let nz = (lo(12)..=t).filter(|&w| ob(w) && l.y(a, w) > 0.0).count() as f64 / 12.0;
|
||
let max52 = (lo(HIST)..=t).filter(|&w| ob(w)).map(|w| l.y(a, w)).fold(0.0, f64::max);
|
||
let (m4, m12) = (mean(lo(4), t), mean(lo(12), t));
|
||
let prev8 = if t >= 4 { mean(lo(12), t - 4) } else { 0.0 };
|
||
|
||
let end_t = l.weeks[t] + Duration::days(6);
|
||
let win_end = l.weeks[t + win] + Duration::days(6);
|
||
let rel = l.releases[a].as_ref();
|
||
let cnt = |from: NaiveDate, to: NaiveDate| rel.map_or(0, |r| r.iter().filter(|d| **d > from && **d <= to).count()) as f64;
|
||
let ch = l.chart[l.i(a, t)];
|
||
let nmf_in = |from: NaiveDate, to: NaiveDate| l.nmf[a].iter().filter(|(d, _)| *d > from && *d <= to).count() as f64;
|
||
let nmf_ok = l.weeks[t] >= NMF_START.parse::<NaiveDate>().unwrap();
|
||
let season = (1..=win).map(|k| l.season[l.weeks[t + k].iso_week().week() as usize].ln()).sum::<f64>() / win as f64;
|
||
|
||
*x = [
|
||
1.0, l.y(a, t).ln_1p(), m4.ln_1p(), m12.ln_1p(), max52.ln_1p(),
|
||
last_on.is_some_and(|k| k < 26) as i32 as f64, last_on.is_some_and(|k| k >= 26) as i32 as f64, observed, nz,
|
||
m4.ln_1p() - prev8.ln_1p(), rel.is_some() as i32 as f64,
|
||
cnt(end_t - Duration::weeks(4), end_t).min(1.0), cnt(end_t - Duration::weeks(12), end_t - Duration::weeks(4)).min(1.0),
|
||
cnt(end_t, win_end).min(1.0), cnt(end_t - Duration::weeks(52), end_t).ln_1p(),
|
||
(ch.t40_rank > 0) as i32 as f64, (ch.air_rank > 0) as i32 as f64, (ch.t40_hits as f64).ln_1p(), (ch.air_hits as f64).ln_1p(),
|
||
(ch.weeks_since_entry <= 4) as i32 as f64,
|
||
nmf_ok as i32 as f64, nmf_in(end_t - Duration::weeks(4), end_t).min(1.0), nmf_in(end_t, win_end).min(1.0),
|
||
nmf_in(end_t - Duration::weeks(52), end_t).ln_1p(),
|
||
l.artists[a].danish as i32 as f64, l.artists[a].group as i32 as f64, l.artists[a].female as i32 as f64, season,
|
||
(o.era && t < l.cur0) as i32 as f64, 0.0, 0.0, 0.0, 0.0,
|
||
];
|
||
// What the artist did before the last 52 weeks (the archive, for current-panel origins).
|
||
if o.hist && t >= HIST {
|
||
let old: Vec<usize> = (0..t + 1 - HIST).filter(|&w| ob(w)).collect();
|
||
if !old.is_empty() {
|
||
let on = |w: usize| l.y(a, w) >= tau;
|
||
let adds = old.iter().filter(|&&w| on(w) && (w == 0 || !ob(w - 1) || !on(w - 1))).count() as f64;
|
||
x[29] = 1.0;
|
||
x[30] = old.iter().map(|&w| l.y(a, w)).fold(0.0, f64::max).ln_1p();
|
||
x[31] = adds.ln_1p();
|
||
x[32] = old.iter().filter(|&&w| on(w)).count() as f64 / old.len() as f64;
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Every qualifying row as (origin week, artist, label).
|
||
fn rows(l: &Long, win: usize, tau: f64) -> Vec<(usize, usize, bool)> {
|
||
let mut v = Vec::new();
|
||
for t in 0..l.w() { for a in 0..l.artists.len() { if let Some(y) = l.label(a, t, win, tau) { v.push((t, a, y)); } } }
|
||
v
|
||
}
|
||
|
||
struct Fit { b: Vec<f64>, o: Opt }
|
||
impl Fit {
|
||
fn new(l: &Long, rows: &[(usize, usize, bool)], keep: impl Fn(usize) -> bool, win: usize, tau: f64, o: Opt) -> Fit {
|
||
let mut d = Design::new(P);
|
||
let mut x = [0.0; P];
|
||
for &(t, a, y) in rows.iter().filter(|r| keep(r.0)) {
|
||
feats(l, a, t, win, tau, o, &mut x);
|
||
d.push(&x, y as i32 as f64, 1.0);
|
||
}
|
||
Fit { b: glm::logit(&d, 1.0), o }
|
||
}
|
||
fn p(&self, l: &Long, a: usize, t: usize, win: usize, tau: f64) -> f64 {
|
||
let mut x = [0.0; P];
|
||
feats(l, a, t, win, tau, Opt { era: false, ..self.o }, &mut x); // predictions: era dummy off (current-day base rate)
|
||
glm::sigmoid(glm::dot(&self.b, &x)).clamp(1e-6, 1.0 - 1e-6)
|
||
}
|
||
}
|
||
|
||
fn ll(p: f64, y: bool) -> f64 { -(if y { p } else { 1.0 - p }).ln() }
|
||
fn ap(mut v: Vec<(f64, bool)>) -> f64 {
|
||
v.sort_by(|a, b| b.0.total_cmp(&a.0));
|
||
let (mut hits, mut s) = (0.0, 0.0);
|
||
for (i, (_, y)) in v.iter().enumerate() { if *y { hits += 1.0; s += hits / (i + 1) as f64; } }
|
||
if hits > 0.0 { s / hits } else { 0.0 }
|
||
}
|
||
|
||
/// Scores for one test period: per model, the predictions on the same rows.
|
||
struct Test { name: String, t: Vec<usize>, y: Vec<bool>, p: Vec<Vec<f64>> }
|
||
|
||
fn report(md: &mut String, models: &[&str], tests: &[Test]) {
|
||
let _ = writeln!(md, "| test period | rows | adds | model | log loss | Brier skill | avg precision | mean p / rate | Δ log loss vs 1-year (90% CI) |");
|
||
let _ = writeln!(md, "|---|---|---|---|---|---|---|---|---|");
|
||
for te in tests {
|
||
let n = te.y.len() as f64;
|
||
let rate = te.y.iter().filter(|y| **y).count() as f64 / n;
|
||
let brier_ref = rate * (1.0 - rate);
|
||
for (m, name) in models.iter().enumerate() {
|
||
let p = &te.p[m];
|
||
let l: f64 = p.iter().zip(&te.y).map(|(p, y)| ll(*p, *y)).sum::<f64>() / n;
|
||
let b: f64 = p.iter().zip(&te.y).map(|(p, y)| (p - *y as i32 as f64).powi(2)).sum::<f64>() / n;
|
||
let mp = p.iter().sum::<f64>() / n;
|
||
let delta = if m == 0 { "—".to_string() } else {
|
||
let (d, lo, hi) = boot(te, m, 0);
|
||
format!("{:+.4} ({:+.4} to {:+.4})", d, lo, hi)
|
||
};
|
||
let _ = writeln!(md, "| {} | {} | {} | {} | {:.4} | {:+.3} | {:.3} | {:.4} / {:.4} | {} |", te.name, te.y.len(), te.y.iter().filter(|y| **y).count(),
|
||
name, l, 1.0 - b / brier_ref, ap(p.iter().copied().zip(te.y.iter().copied()).collect()), mp, rate, delta);
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Mean log-loss difference (model m minus model base) with a moving-block bootstrap over 8-week blocks.
|
||
fn boot(te: &Test, m: usize, base: usize) -> (f64, f64, f64) {
|
||
let mut by: std::collections::BTreeMap<usize, (f64, f64)> = std::collections::BTreeMap::new();
|
||
for i in 0..te.y.len() {
|
||
let e = by.entry(te.t[i]).or_insert((0.0, 0.0));
|
||
e.0 += ll(te.p[m][i], te.y[i]) - ll(te.p[base][i], te.y[i]);
|
||
e.1 += 1.0;
|
||
}
|
||
let weeks: Vec<(f64, f64)> = by.values().copied().collect();
|
||
let d = weeks.iter().map(|w| w.0).sum::<f64>() / weeks.iter().map(|w| w.1).sum::<f64>();
|
||
let nb = weeks.len();
|
||
let blk = 8.min(nb);
|
||
let mut s = 0x5eedu64;
|
||
let mut v: Vec<f64> = (0..1000).map(|_| {
|
||
let (mut num, mut den) = (0.0, 0.0);
|
||
for _ in 0..nb.div_ceil(blk) {
|
||
s = mix(s);
|
||
let start = (s % (nb - blk + 1) as u64) as usize;
|
||
for w in &weeks[start..start + blk] { num += w.0; den += w.1; }
|
||
}
|
||
num / den
|
||
}).collect();
|
||
v.sort_by(f64::total_cmp);
|
||
(d, v[50], v[949])
|
||
}
|
||
|
||
pub fn run(db: &Db, p: &Panel, tau: f64, win: usize, stay: usize) -> Result<String> {
|
||
let mut l = Long::load(db, p)?;
|
||
l.stay = stay;
|
||
let all = rows(&l, win, tau);
|
||
let mut md = String::new();
|
||
let n_old = all.iter().filter(|r| r.0 < l.cur0).count();
|
||
let a_old = all.iter().filter(|r| r.0 < l.cur0 && r.2).count();
|
||
let _ = writeln!(md, "# The long any-DR model (2019–)\n");
|
||
let _ = writeln!(md, "Total DR plays per artist and week (all channels, P4 regions included). An *add* is τ = {tau} or more plays in a week of the next {win} week(s) after {OFF} weeks under τ{}. \
|
||
Archive era {} to {} ({} artists observed, {} qualifying rows, {} adds); current panel {} to {} ({} rows, {} adds).\n",
|
||
if stay > 0 { format!(", and τ again in at least 2 of the {stay} weeks after it (it stays on)") } else { " (a one-week blip counts)".into() },
|
||
l.weeks[0], l.weeks[(0..l.cur0).rev().find(|&w| (0..l.artists.len()).any(|a| l.obs(a, w))).unwrap_or(0)], (0..l.artists.len()).filter(|&a| (0..l.cur0).any(|w| l.obs(a, w))).count(), n_old, a_old,
|
||
l.weeks[l.cur0], l.weeks[l.w() - 1], all.len() - n_old, all.iter().filter(|r| r.0 >= l.cur0 && r.2).count());
|
||
let _ = writeln!(md, "Models: **1-year** trains on the 52 weeks of origins before the test period (what the site can do from the current panel alone); \
|
||
**all years** on every earlier origin; **+ older history** adds four features from the artist's weeks before the last 52 (for today's predictions: the 2019–24 archive); **all years + era** adds an archive-era dummy, switched off when predicting (the archive over-represents popular artists, so its base rate is higher). \
|
||
Training rows' labels (window plus the weeks checked after it) end before the first test origin, so no label is seen early. Δ < 0 means better than 1-year.\n");
|
||
|
||
let models = ["1-year", "all years", "all years + era", "1-year + older history", "all years + era + older history"];
|
||
let mut tests = Vec::new();
|
||
// (1) each archived calendar year (2020–2024), refit once at the start of the year.
|
||
for year in 2020..=2024 {
|
||
let (s, e) = (NaiveDate::from_ymd_opt(year, 1, 1).unwrap(), NaiveDate::from_ymd_opt(year + 1, 1, 1).unwrap());
|
||
let Some(t0) = l.weeks.iter().position(|d| *d >= s) else { continue };
|
||
if t0 < HIST + win { continue; }
|
||
let test: Vec<&(usize, usize, bool)> = all.iter().filter(|r| l.weeks[r.0] >= s && l.weeks[r.0] < e && r.0 < l.cur0).collect();
|
||
if test.iter().filter(|r| r.2).count() < 20 { continue; }
|
||
let fits = [
|
||
Fit::new(&l, &all, |t| t + win + l.stay < t0 && t + HIST >= t0, win, tau, Opt { era: false, hist: false }),
|
||
Fit::new(&l, &all, |t| t + win + l.stay < t0, win, tau, Opt { era: false, hist: false }),
|
||
Fit::new(&l, &all, |t| t + win + l.stay < t0, win, tau, Opt { era: true, hist: false }),
|
||
Fit::new(&l, &all, |t| t + win + l.stay < t0 && t + HIST >= t0, win, tau, Opt { era: false, hist: true }),
|
||
Fit::new(&l, &all, |t| t + win + l.stay < t0, win, tau, Opt { era: true, hist: true }),
|
||
];
|
||
tests.push(Test { name: year.to_string(), t: test.iter().map(|r| r.0).collect(), y: test.iter().map(|r| r.2).collect(),
|
||
p: fits.iter().map(|f| test.iter().map(|r| f.p(&l, r.1, r.0, win, tau)).collect()).collect() });
|
||
eprintln!("long: {year} done");
|
||
}
|
||
// (2) the current panel: second half, refit every 4 weeks. 1-year = current panel only.
|
||
let mut cur = Test { name: format!("current panel, origins from {}", l.weeks[l.cur0 + 26]), t: vec![], y: vec![], p: vec![vec![]; 5] };
|
||
let mut t0 = l.cur0 + 26;
|
||
while t0 + win < l.w() {
|
||
let t1 = (t0 + 4).min(l.w());
|
||
let fits = [
|
||
Fit::new(&l, &all, |t| t >= l.cur0 && t + win + l.stay < t0, win, tau, Opt { era: false, hist: false }),
|
||
Fit::new(&l, &all, |t| t + win + l.stay < t0, win, tau, Opt { era: false, hist: false }),
|
||
Fit::new(&l, &all, |t| t + win + l.stay < t0, win, tau, Opt { era: true, hist: false }),
|
||
Fit::new(&l, &all, |t| t >= l.cur0 && t + win + l.stay < t0, win, tau, Opt { era: false, hist: true }),
|
||
Fit::new(&l, &all, |t| t + win + l.stay < t0, win, tau, Opt { era: true, hist: true }),
|
||
];
|
||
for r in all.iter().filter(|r| r.0 >= t0 && r.0 < t1) {
|
||
cur.t.push(r.0); cur.y.push(r.2);
|
||
for (m, f) in fits.iter().enumerate() { cur.p[m].push(f.p(&l, r.1, r.0, win, tau)); }
|
||
}
|
||
eprintln!("long: current from {} done", l.weeks[t0]);
|
||
t0 = t1;
|
||
}
|
||
tests.push(cur);
|
||
let _ = writeln!(md, "## Out of sample\n");
|
||
report(&mut md, &models, &tests);
|
||
|
||
let f = Fit::new(&l, &all, |t| t + win + l.stay < l.w(), win, tau, Opt { era: true, hist: true });
|
||
let _ = writeln!(md, "\n## Coefficients (all years + era + older history, every row)\n\n| feature | β |\n|---|---|");
|
||
for (n, b) in NAMES.iter().zip(&f.b) { let _ = writeln!(md, "| {n} | {b:+.3} |"); }
|
||
Ok(md)
|
||
}
|
||
|
||
/// Comebacks: every sustained add, by what the artist did before it, and how much release data
|
||
/// we have around it; then add rates after a release for quiet artists, former favourites or not.
|
||
pub fn comebacks(db: &Db, p: &Panel, tau: f64) -> Result<String> {
|
||
const Q: usize = 26; // "quiet" = under τ in each of the 26 weeks before
|
||
let l = Long::load(db, p)?;
|
||
let na = l.artists.len();
|
||
let on = |a: usize, w: usize| l.y(a, w) >= tau;
|
||
let same_era = |a: usize, b: usize| (a < l.cur0) == (b < l.cur0);
|
||
// Sustained add at week s: on at s, on in 2 of s+1..s+3, under τ the 8 weeks before, all observed.
|
||
let is_add = |a: usize, s: usize| -> bool {
|
||
s >= OFF && s + 3 < l.w() && same_era(s - OFF, s + 3) && (s - OFF..=s + 3).all(|w| l.obs(a, w))
|
||
&& on(a, s) && (s + 1..=s + 3).filter(|&w| on(a, w)).count() >= 2 && (s - OFF..s).all(|w| !on(a, w))
|
||
};
|
||
// What the artist was before week s.
|
||
#[derive(Clone, Copy, PartialEq)]
|
||
enum Kind { Active, Favourite, Seen, NoOld, Unknown }
|
||
let kind = |a: usize, s: usize| -> Kind {
|
||
if s < Q { return Kind::Unknown; }
|
||
let recent: Vec<usize> = (s - Q..s).filter(|&w| l.obs(a, w)).collect();
|
||
if recent.len() < Q * 4 / 5 { return Kind::Unknown; }
|
||
if recent.iter().any(|&w| on(a, w)) { return Kind::Active; }
|
||
let old: Vec<usize> = (0..s - Q).filter(|&w| l.obs(a, w)).collect();
|
||
if old.is_empty() { return Kind::NoOld; }
|
||
if old.iter().filter(|&&w| on(a, w)).count() >= 4 { Kind::Favourite } else { Kind::Seen }
|
||
};
|
||
let names = ["active in the last 26 weeks (back after 8-25 weeks off)", "quiet 26w, former favourite (4+ weeks on DR before)",
|
||
"quiet 26w, played before but never on", "quiet 26w, no older data", "too little data to tell"];
|
||
let idx = |k: Kind| match k { Kind::Active => 0, Kind::Favourite => 1, Kind::Seen => 2, Kind::NoOld => 3, Kind::Unknown => 4 };
|
||
let near = |a: usize, s: usize, before: i64, after: i64| -> (bool, bool) {
|
||
let d = l.weeks[s];
|
||
let rel = l.releases[a].as_ref().is_some_and(|r| r.iter().any(|x| *x >= d - Duration::weeks(before) && *x <= d + Duration::days(7 * after + 6)));
|
||
let nmf = l.nmf[a].iter().any(|(x, _)| *x >= d - Duration::weeks(before) && *x <= d + Duration::days(7 * after + 6));
|
||
(rel, nmf)
|
||
};
|
||
|
||
let mut md = String::new();
|
||
let _ = writeln!(md, "# Comebacks (τ = {tau} total DR plays a week; an add = on, then on again in 2 of the next 3 weeks, after 8 weeks under τ)\n");
|
||
for (label, lo, hi) in [("archive 2019-24", 0, l.cur0), ("current panel 2025-26", l.cur0, l.w())] {
|
||
// [kind][era]: adds, discography known, release -8w..+1w, NMF -8w..+1w, either
|
||
let mut c = [[0usize; 5]; 5];
|
||
let mut ex: Vec<Vec<String>> = vec![Vec::new(); 5];
|
||
for a in 0..na { for s in lo.max(OFF)..hi { if is_add(a, s) {
|
||
let k = idx(kind(a, s));
|
||
let (r, n) = near(a, s, 8, 1);
|
||
c[k][0] += 1; c[k][1] += l.releases[a].is_some() as usize; c[k][2] += r as usize; c[k][3] += n as usize; c[k][4] += (r || n) as usize;
|
||
if ex[k].len() < 12 { ex[k].push(format!("{} ({})", l.artists[a].name, l.weeks[s])); }
|
||
} } }
|
||
let total: usize = c.iter().map(|r| r[0]).sum();
|
||
let _ = writeln!(md, "## Adds, {label}: {total}\n\n| before the add | adds | share | discography known | release 8w before–1w after | NMF DK 8w before–1w after | release or NMF |\n|---|---|---|---|---|---|---|");
|
||
for k in 0..5 {
|
||
let r = c[k]; let pc = |x: usize| if r[0] > 0 { format!("{:.0}%", 100.0 * x as f64 / r[0] as f64) } else { "—".into() };
|
||
let _ = writeln!(md, "| {} | {} | {} | {} | {} | {} | {} |", names[k], r[0], if total > 0 { format!("{:.0}%", 100.0 * r[0] as f64 / total as f64) } else { "—".into() }, pc(r[1]), pc(r[2]), pc(r[3]), pc(r[4]));
|
||
}
|
||
let _ = writeln!(md, "\nExamples, former favourites: {}\n", ex[1].join(", "));
|
||
}
|
||
|
||
// Releases by quiet artists (Deezer, none in the 6 weeks before): add within the release week + 4?
|
||
let _ = writeln!(md, "## After a release, quiet artists (under τ for 26 weeks): add in the release week or the 4 after\n");
|
||
let _ = writeln!(md, "| period | artist | releases | adds | rate (90% CI) |\n|---|---|---|---|---|");
|
||
for (label, lo, hi) in [("archive 2019-24", 0, l.cur0), ("current panel 2025-26", l.cur0, l.w())] {
|
||
let mut c = [[0usize; 2]; 5];
|
||
for a in 0..na {
|
||
let Some(rel) = &l.releases[a] else { continue };
|
||
let mut prev: Option<NaiveDate> = None;
|
||
for d in rel {
|
||
let fresh = prev.is_none_or(|p| (*d - p).num_weeks() >= 6);
|
||
prev = Some(*d);
|
||
if !fresh { continue; }
|
||
let Some(s0) = l.weeks.iter().position(|w| *w <= *d && *d < *w + Duration::weeks(1)) else { continue };
|
||
if s0 < lo || s0 + 4 + 3 >= hi || !(s0..=s0 + 7).all(|w| l.obs(a, w)) { continue; }
|
||
let k = kind(a, s0);
|
||
if k == Kind::Active || k == Kind::Unknown { continue; }
|
||
c[idx(k)][0] += 1;
|
||
c[idx(k)][1] += (s0..=s0 + 4).any(|s| is_add(a, s)) as usize;
|
||
}
|
||
}
|
||
for k in 1..4 {
|
||
let (n, x) = (c[k][0] as f64, c[k][1] as f64);
|
||
if n == 0.0 { continue; }
|
||
// Wilson 90% interval
|
||
let (ph, z) = (x / n, 1.645f64);
|
||
let den = 1.0 + z * z / n;
|
||
let mid = (ph + z * z / (2.0 * n)) / den;
|
||
let half = z * (ph * (1.0 - ph) / n + z * z / (4.0 * n * n)).sqrt() / den;
|
||
let _ = writeln!(md, "| {label} | {} | {} | {} | {:.1}% ({:.1}–{:.1}%) |", names[k], n, x, 100.0 * ph, 100.0 * (mid - half).max(0.0), 100.0 * (mid + half));
|
||
}
|
||
}
|
||
let looked = (0..na).filter(|&a| l.releases[a].is_some()).count();
|
||
let _ = writeln!(md, "\nDiscographies (Deezer) known for {looked} of {na} artists ({} of the {} in today's pool).", (0..p.a()).filter(|&a| l.releases[a].is_some()).count(), p.a());
|
||
Ok(md)
|
||
}
|