Release-week calendar: adjustable window (2-12 weeks, one release-event model each), time-since features moved to the release week, 'too soon' instead of special cases, season index (IFPI 2002-24) in the window; site: Best week tab, live ranking for any release week and window, single new-song view
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+99
-60
@@ -40,12 +40,12 @@ pub const WAVE_NAMES: &[&str] = &["DR: weeks since last add 13-25", "DR: last ad
|
||||
|
||||
/// Weeks since the artist's last add on any DR channel (as of week t; None = none in the panel),
|
||||
/// and the IFPI wave position.
|
||||
pub fn wave_feats(p: &Panel, r: &Rot, a: usize, t: usize) -> [f64; WAVE] {
|
||||
pub fn wave_feats(p: &Panel, r: &Rot, a: usize, t: usize, shift: usize) -> [f64; WAVE] {
|
||||
if off("wave") { return [0.0; WAVE]; }
|
||||
let since = (0..=t).find(|&i| r.add_any(a, t - i));
|
||||
let since = (0..=t).find(|&i| r.add_any(a, t - i)).map(|s| s + shift);
|
||||
let ch = p.chart(a, t);
|
||||
let f = |b: bool| b as i32 as f64;
|
||||
let air = ch.air_since as f64;
|
||||
let air = ch.air_since as f64 + shift as f64;
|
||||
let known = ch.air_since != u16::MAX;
|
||||
let ratio_ok = known && ch.air_gap > 0 && { let x = air / ch.air_gap as f64; (0.5..=1.25).contains(&x) };
|
||||
[f(since.is_some_and(|s| (13..=25).contains(&s))), f(since.is_some_and(|s| s >= 26)), f(since.is_none()),
|
||||
@@ -54,11 +54,17 @@ pub fn wave_feats(p: &Panel, r: &Rot, a: usize, t: usize) -> [f64; WAVE] {
|
||||
|
||||
/// What the forecast assumes about the window: None = what actually happened (training and
|
||||
/// backtests), Some = a scenario (the ranking, the what-if).
|
||||
#[derive(Clone, Copy, Default)]
|
||||
pub struct Scenario { pub release: Option<bool>, pub nmf: Option<bool> }
|
||||
/// `win`: the window length in weeks (the add must come within it); `shift`: how many weeks after
|
||||
/// the origin the release is assumed, so time-since features (last release, last add, lists, NMF)
|
||||
/// are measured at the release week rather than today.
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct Scenario { pub release: Option<bool>, pub nmf: Option<bool>, pub shift: usize, pub win: usize }
|
||||
impl Scenario {
|
||||
pub const ACTUAL: Scenario = Scenario { release: None, nmf: None };
|
||||
pub fn new(release: bool, nmf: bool) -> Scenario { Scenario { release: Some(release), nmf: Some(nmf) } }
|
||||
pub const ACTUAL: Scenario = Scenario { release: None, nmf: None, shift: 0, win: W };
|
||||
pub fn actual(win: usize) -> Scenario { Scenario { win, ..Self::ACTUAL } }
|
||||
pub fn new(release: bool, nmf: bool) -> Scenario { Scenario { release: Some(release), nmf: Some(nmf), shift: 0, win: W } }
|
||||
pub fn at(self, shift: usize) -> Scenario { Scenario { shift, ..self } }
|
||||
pub fn window(self, win: usize) -> Scenario { Scenario { win, ..self } }
|
||||
}
|
||||
|
||||
/// New Music Friday Denmark: placed in the window (and in its top 10), placements in the past
|
||||
@@ -67,9 +73,18 @@ pub const NMF: usize = 4;
|
||||
pub const NMF_NAMES: &[&str] = &["NMF Denmark in the window", "NMF Denmark top 10 in the window", "log NMF Denmark placements last 52w",
|
||||
"share of releases on NMF Denmark (2 years)"];
|
||||
|
||||
fn window(p: &Panel, t: usize, h: usize) -> (chrono::NaiveDate, chrono::NaiveDate) {
|
||||
fn window(p: &Panel, t: usize, h: usize, win: usize) -> (chrono::NaiveDate, chrono::NaiveDate) {
|
||||
let w0 = p.weeks[t] + chrono::Duration::weeks(h as i64 + 1);
|
||||
(w0, w0 + chrono::Duration::weeks(W as i64))
|
||||
(w0, w0 + chrono::Duration::weeks(win as i64))
|
||||
}
|
||||
|
||||
/// A release in week t0+h+1 is "too soon" if the artist's latest release (as of t0) is less than
|
||||
/// CLEAN_WEEKS before it: the release-event model only learned from releases with a clear gap.
|
||||
pub fn too_soon(p: &Panel, a: usize, t0: usize, h: usize) -> bool {
|
||||
let Some(dates) = p.releases[a].as_ref() else { return false };
|
||||
let end_t = p.weeks[t0] + chrono::Duration::days(6);
|
||||
let start = p.weeks[t0] + chrono::Duration::weeks(h as i64 + 1);
|
||||
dates.iter().rev().find(|d| **d <= end_t).is_some_and(|d| (start - *d).num_days() < 7 * CLEAN_WEEKS as i64)
|
||||
}
|
||||
|
||||
/// Did the artist release something (Deezer discography) in DR week w (Monday to Sunday)?
|
||||
@@ -86,11 +101,6 @@ pub fn clean_release(p: &Panel, a: usize, w: usize) -> bool {
|
||||
released_in_week(p, a, w) && (w.saturating_sub(CLEAN_WEEKS)..w).all(|k| !released_in_week(p, a, k))
|
||||
}
|
||||
|
||||
/// Did the artist release something in the CLEAN_WEEKS weeks up to and including week t?
|
||||
pub fn recent_release(p: &Panel, a: usize, t: usize) -> bool {
|
||||
((t + 1).saturating_sub(CLEAN_WEEKS)..=t).any(|k| released_in_week(p, a, k))
|
||||
}
|
||||
|
||||
/// P(an artist's release is placed on NMF Denmark), from the past two years (shrunk).
|
||||
pub fn p_nmf(p: &Panel, a: usize, t: usize) -> f64 {
|
||||
let end_t = p.weeks[t] + chrono::Duration::days(6);
|
||||
@@ -102,7 +112,8 @@ pub fn p_nmf(p: &Panel, a: usize, t: usize) -> f64 {
|
||||
pub fn nmf_feats(p: &Panel, a: usize, t: usize, h: usize, sc: Scenario) -> [f64; NMF] {
|
||||
if off("nmf") { return [0.0; NMF]; }
|
||||
let end_t = p.weeks[t] + chrono::Duration::days(6);
|
||||
let (w0, w1) = window(p, t, h);
|
||||
let asof = end_t + chrono::Duration::weeks(sc.shift as i64);
|
||||
let (w0, w1) = window(p, t, h, sc.win);
|
||||
let placed = p.nmf_dk[a].iter().filter(|(d, _)| *d >= w0 && *d < w1);
|
||||
let (in_win, top10) = match sc.nmf {
|
||||
None => { let v: Vec<_> = placed.collect(); (!v.is_empty(), v.iter().any(|(_, pos)| pos.is_some_and(|x| x <= 10))) }
|
||||
@@ -112,7 +123,7 @@ pub fn nmf_feats(p: &Panel, a: usize, t: usize, h: usize, sc: Scenario) -> [f64;
|
||||
};
|
||||
let past: Vec<&(chrono::NaiveDate, Option<i64>)> = p.nmf_dk[a].iter().filter(|(d, _)| *d <= end_t).collect();
|
||||
let top10_share = if past.is_empty() { 0.0 } else { past.iter().filter(|(_, pos)| pos.is_some_and(|x| x <= 10)).count() as f64 / past.len() as f64 };
|
||||
let last_year = past.iter().filter(|(d, _)| (end_t - *d).num_days() <= 364).count() as f64;
|
||||
let last_year = past.iter().filter(|(d, _)| (asof - *d).num_days() <= 364).count() as f64;
|
||||
let f = |b: bool| b as i32 as f64;
|
||||
[f(in_win), if sc.nmf == Some(true) { top10_share } else { f(top10) }, last_year.ln_1p(), p_nmf(p, a, t)]
|
||||
}
|
||||
@@ -122,15 +133,26 @@ pub const LONG: usize = 4;
|
||||
pub const LONG_NAMES: &[&str] = &["log archive plays per 100 sampled hours (2012-21)", "log DR lists with the artist before t (2023-)",
|
||||
"DR list recency 26w", "DR lists known before t at all"];
|
||||
|
||||
pub fn long_feats(p: &Panel, a: usize, t: usize) -> [f64; LONG] {
|
||||
pub fn long_feats(p: &Panel, a: usize, t: usize, shift: usize) -> [f64; LONG] {
|
||||
if off("long") { return [0.0; LONG]; }
|
||||
let end_t = p.weeks[t] + chrono::Duration::days(6);
|
||||
// A list published on a Friday is known from then.
|
||||
let before: Vec<&chrono::NaiveDate> = p.listed_on[a].iter().filter(|d| **d <= end_t).collect();
|
||||
let since = before.last().map(|d| (end_t - **d).num_days() as f64 / 7.0);
|
||||
let since = before.last().map(|d| (end_t - **d).num_days() as f64 / 7.0 + shift as f64);
|
||||
[p.archive_rate[a].ln_1p(), (before.len() as f64).ln_1p(), since.map(|s| (-s / 26.0).exp()).unwrap_or(0.0), (!before.is_empty()) as i32 as f64]
|
||||
}
|
||||
|
||||
/// Season: the mean log season index over the window's weeks (new Airplay Top-20 songs by week of
|
||||
/// the year, 2002–2024).
|
||||
pub const SEASON: usize = 1;
|
||||
pub const SEASON_NAMES: &[&str] = &["season index of the window (log, IFPI 2002-24)"];
|
||||
pub fn season_feat(p: &Panel, t: usize, h: usize, win: usize) -> [f64; SEASON] {
|
||||
if off("season") { return [0.0]; }
|
||||
let start = p.weeks[t] + chrono::Duration::weeks(h as i64 + 1);
|
||||
let v: f64 = (0..win).map(|k| { let d = start + chrono::Duration::weeks(k as i64); p.season[chrono::Datelike::iso_week(&d).week() as usize].ln() }).sum::<f64>() / win as f64;
|
||||
[v]
|
||||
}
|
||||
|
||||
/// Release covariates (Deezer discography), appended to both row kinds.
|
||||
pub const REL: usize = 6;
|
||||
pub const REL_NAMES: &[&str] = &["discography known", "release recency 4w", "release recency 12w", "log releases last 52w",
|
||||
@@ -144,10 +166,10 @@ pub fn rel_feats(p: &Panel, a: usize, t: usize, h: usize, sc: Scenario, in_rot_n
|
||||
if off("rel") { return [0.0; REL]; }
|
||||
let Some(dates) = p.releases[a].as_ref() else { return [0.0; REL] };
|
||||
let end_t = p.weeks[t] + chrono::Duration::days(6);
|
||||
let since = dates.iter().rev().find(|d| **d <= end_t).map(|d| (end_t - *d).num_days() as f64 / 7.0);
|
||||
let last_year = dates.iter().filter(|d| **d <= end_t && (end_t - **d).num_days() <= 364).count() as f64;
|
||||
let w0 = p.weeks[t] + chrono::Duration::weeks(h as i64 + 1);
|
||||
let w1 = w0 + chrono::Duration::weeks(W as i64);
|
||||
let asof = end_t + chrono::Duration::weeks(sc.shift as i64);
|
||||
let since = dates.iter().rev().find(|d| **d <= end_t).map(|d| (asof - *d).num_days() as f64 / 7.0);
|
||||
let last_year = dates.iter().filter(|d| **d <= end_t && (asof - **d).num_days() <= 364).count() as f64;
|
||||
let (w0, w1) = window(p, t, h, sc.win);
|
||||
let in_window = rel.unwrap_or_else(|| dates.iter().any(|d| *d >= w0 && *d < w1));
|
||||
let f = |b: bool| b as i32 as f64;
|
||||
[1.0, since.map(|s| (-s / 4.0).exp()).unwrap_or(0.0), since.map(|s| (-s / 12.0).exp()).unwrap_or(0.0), last_year.ln_1p(),
|
||||
@@ -168,7 +190,7 @@ pub const EXTRA: &[&str] = &["in rotation on channel now", "in rotation anywhere
|
||||
"run on channel = 1 week", "run = 2 weeks", "run = 3-5 weeks", "run >= 6 weeks",
|
||||
"near threshold (>= half of tau, not in rotation)", "plays / tau (capped 3, /3)",
|
||||
"weeks since in rotation on channel (/12)"];
|
||||
pub const PC: usize = P + 10 + REL + WAVE + LONG + NMF;
|
||||
pub const PC: usize = P + 10 + REL + WAVE + LONG + NMF + SEASON;
|
||||
|
||||
/// Artist-level ("any channel") rows.
|
||||
pub const ANY_NAMES: &[&str] = &[
|
||||
@@ -178,7 +200,7 @@ pub const ANY_NAMES: &[&str] = &[
|
||||
"log Track Top-40 hits", "log Airplay Top-20 hits", "Danish", "group", "female", "career years (/50)", "debut unknown",
|
||||
"Christmas in window",
|
||||
];
|
||||
pub const PA: usize = 21 + REL + WAVE + LONG + NMF;
|
||||
pub const PA: usize = 21 + REL + WAVE + LONG + NMF + SEASON;
|
||||
/// Positions of the chart covariates (for the what-if): (t40 score, air score, recency 6w, recency 26w).
|
||||
pub const CHART_IDX_CHANNEL: [usize; 4] = [10, 11, 12, 13];
|
||||
pub const CHART_IDX_ANY: [usize; 4] = [9, 10, 11, 12];
|
||||
@@ -207,8 +229,10 @@ impl Rot {
|
||||
pub fn add(&self, a: usize, c: usize, w: usize) -> bool { w >= GAP && self.on(a, c, w) && (1..=GAP).all(|k| !self.on(a, c, w - k)) }
|
||||
pub fn add_any(&self, a: usize, w: usize) -> bool { w >= GAP && self.any(a, w) && (1..=GAP).all(|k| !self.any(a, w - k)) }
|
||||
/// An add in weeks t+h+1 ..= t+h+W (all inside the panel).
|
||||
pub fn add_in_window(&self, a: usize, c: usize, t: usize, h: usize) -> bool { (t + h + 1..=t + h + W).any(|w| self.add(a, c, w)) }
|
||||
pub fn add_any_in_window(&self, a: usize, t: usize, h: usize) -> bool { (t + h + 1..=t + h + W).any(|w| self.add_any(a, w)) }
|
||||
pub fn add_in_window(&self, a: usize, c: usize, t: usize, h: usize) -> bool { self.add_in(a, c, t, h, W) }
|
||||
pub fn add_any_in_window(&self, a: usize, t: usize, h: usize) -> bool { self.add_any_in(a, t, h, W) }
|
||||
pub fn add_in(&self, a: usize, c: usize, t: usize, h: usize, win: usize) -> bool { (t + h + 1..=t + h + win).any(|w| self.add(a, c, w)) }
|
||||
pub fn add_any_in(&self, a: usize, t: usize, h: usize, win: usize) -> bool { (t + h + 1..=t + h + win).any(|w| self.add_any(a, w)) }
|
||||
}
|
||||
|
||||
/// Station groups (P4 once) where the artist is in rotation in week t.
|
||||
@@ -226,11 +250,11 @@ fn run_and_since(flag: impl Fn(usize) -> bool, t: usize) -> (f64, f64) {
|
||||
(run, since)
|
||||
}
|
||||
|
||||
fn christmas_in_window(p: &Panel, t: usize, h: usize) -> bool { (t + h + 1..=t + h + W).any(|w| features::christmas(p, w)) }
|
||||
fn christmas_in_window(p: &Panel, t: usize, h: usize, win: usize) -> bool { (t + h + 1..=t + h + win).any(|w| features::christmas(p, w)) }
|
||||
|
||||
pub fn row_channel(p: &Panel, r: &Rot, a: usize, c: usize, t: usize, h: usize, sc: Scenario, out: &mut [f64]) {
|
||||
features::row(p, a, c, t, t + h + 1, &mut out[..P]);
|
||||
out[21] = christmas_in_window(p, t, h) as i32 as f64;
|
||||
out[21] = christmas_in_window(p, t, h, sc.win) as i32 as f64;
|
||||
let n_on = groups_on(r, a, t);
|
||||
let (run, since) = run_and_since(|w| r.on(a, c, w), t);
|
||||
let weeks = (run * LAGS as f64).round() as usize;
|
||||
@@ -241,9 +265,10 @@ pub fn row_channel(p: &Panel, r: &Rot, a: usize, c: usize, t: usize, h: usize, s
|
||||
f(weeks == 1), f(weeks == 2), f((3..=5).contains(&weeks)), f(weeks >= 6),
|
||||
f(!on && ratio >= 0.5), ratio.min(3.0) / 3.0, since]);
|
||||
out[P + 10..P + 10 + REL].copy_from_slice(&rel_feats(p, a, t, h, sc, on));
|
||||
out[P + 10 + REL..P + 10 + REL + WAVE].copy_from_slice(&wave_feats(p, r, a, t));
|
||||
out[P + 10 + REL + WAVE..P + 10 + REL + WAVE + LONG].copy_from_slice(&long_feats(p, a, t));
|
||||
out[P + 10 + REL + WAVE + LONG..PC].copy_from_slice(&nmf_feats(p, a, t, h, sc));
|
||||
out[P + 10 + REL..P + 10 + REL + WAVE].copy_from_slice(&wave_feats(p, r, a, t, sc.shift));
|
||||
out[P + 10 + REL + WAVE..P + 10 + REL + WAVE + LONG].copy_from_slice(&long_feats(p, a, t, sc.shift));
|
||||
out[P + 10 + REL + WAVE + LONG..PC - SEASON].copy_from_slice(&nmf_feats(p, a, t, h, sc));
|
||||
out[PC - SEASON..PC].copy_from_slice(&season_feat(p, t, h, sc.win));
|
||||
}
|
||||
|
||||
pub fn row_any(p: &Panel, r: &Rot, a: usize, t: usize, h: usize, sc: Scenario, out: &mut [f64]) {
|
||||
@@ -264,18 +289,21 @@ pub fn row_any(p: &Panel, r: &Rot, a: usize, t: usize, h: usize, sc: Scenario, o
|
||||
recency(6.0), recency(26.0), (ch.t40_hits as f64).ln_1p(), (ch.air_hits as f64).ln_1p(),
|
||||
art.danish as i32 as f64, art.group as i32 as f64, art.female as i32 as f64,
|
||||
art.debut.map(|d| ((year - d).clamp(0, 50)) as f64 / 50.0).unwrap_or(0.0), art.debut.is_none() as i32 as f64,
|
||||
christmas_in_window(p, t, h) as i32 as f64,
|
||||
christmas_in_window(p, t, h, sc.win) as i32 as f64,
|
||||
];
|
||||
out[..21].copy_from_slice(&v);
|
||||
out[21..21 + REL].copy_from_slice(&rel_feats(p, a, t, h, sc, r.any(a, t)));
|
||||
out[21 + REL..21 + REL + WAVE].copy_from_slice(&wave_feats(p, r, a, t));
|
||||
out[21 + REL + WAVE..21 + REL + WAVE + LONG].copy_from_slice(&long_feats(p, a, t));
|
||||
out[21 + REL + WAVE + LONG..PA].copy_from_slice(&nmf_feats(p, a, t, h, sc));
|
||||
out[21 + REL..21 + REL + WAVE].copy_from_slice(&wave_feats(p, r, a, t, sc.shift));
|
||||
out[21 + REL + WAVE..21 + REL + WAVE + LONG].copy_from_slice(&long_feats(p, a, t, sc.shift));
|
||||
out[21 + REL + WAVE + LONG..PA - SEASON].copy_from_slice(&nmf_feats(p, a, t, h, sc));
|
||||
out[PA - SEASON..PA].copy_from_slice(&season_feat(p, t, h, sc.win));
|
||||
}
|
||||
|
||||
/// Logit models for one h, trained on origins whose whole window is observed by `last_week`.
|
||||
pub struct AddModel {
|
||||
pub h: usize,
|
||||
/// Window length in weeks (the add must come within it).
|
||||
pub win: usize,
|
||||
pub channel: Vec<f64>,
|
||||
pub any: Vec<f64>,
|
||||
/// Platt recalibration (a, b): p' = sigmoid(a + b * logit(p)); identity when there wasn't
|
||||
@@ -290,8 +318,9 @@ const CAL_ORIGINS: usize = 6;
|
||||
fn logit(p: f64) -> f64 { let p = p.clamp(1e-6, 1.0 - 1e-6); (p / (1.0 - p)).ln() }
|
||||
fn platt(ab: (f64, f64), p: f64) -> f64 { glm::sigmoid(ab.0 + ab.1 * logit(p)).clamp(1e-6, 1.0 - 1e-6) }
|
||||
|
||||
fn origins(h: usize, last_week: usize) -> std::ops::RangeInclusive<usize> {
|
||||
let hi = last_week.saturating_sub(h + W);
|
||||
fn origins(h: usize, last_week: usize) -> std::ops::RangeInclusive<usize> { origins_w(h, last_week, W) }
|
||||
fn origins_w(h: usize, last_week: usize, win: usize) -> std::ops::RangeInclusive<usize> {
|
||||
let hi = last_week.saturating_sub(h + win);
|
||||
LAGS - 1..=hi
|
||||
}
|
||||
|
||||
@@ -299,32 +328,34 @@ impl AddModel {
|
||||
/// The logits on every window closed by `last_week`, then a recalibration: an inner model fitted
|
||||
/// on windows closed before the last CAL_ORIGINS origins predicts those origins, and a Platt fit
|
||||
/// maps its probabilities to what happened. (The raw fits were overconfident above ~0.4.)
|
||||
pub fn fit(p: &Panel, r: &Rot, h: usize, last_week: usize) -> AddModel { Self::fit_with(p, r, h, last_week, false) }
|
||||
pub fn fit(p: &Panel, r: &Rot, h: usize, last_week: usize) -> AddModel { Self::fit_with(p, r, h, last_week, false, W) }
|
||||
|
||||
/// The release-event model: "if the artist releases a new song in week t+1, will DR add it
|
||||
/// within 8 weeks?" Trained only on artist-weeks where the artist did release in week t+1, with
|
||||
/// the artist's state the week before (earlier releases and NMF placements count as history;
|
||||
/// the new song's own NMF placement is the scenario variable).
|
||||
pub fn fit_events(p: &Panel, r: &Rot, last_week: usize) -> AddModel { Self::fit_with(p, r, 0, last_week, true) }
|
||||
pub fn fit_events(p: &Panel, r: &Rot, last_week: usize) -> AddModel { Self::fit_with(p, r, 0, last_week, true, W) }
|
||||
/// The release-event model for a window of `win` weeks.
|
||||
pub fn fit_events_w(p: &Panel, r: &Rot, last_week: usize, win: usize) -> AddModel { Self::fit_with(p, r, 0, last_week, true, win) }
|
||||
|
||||
fn fit_with(p: &Panel, r: &Rot, h: usize, last_week: usize, events: bool) -> AddModel {
|
||||
let mut m = Self::fit_raw(p, r, h, last_week, events);
|
||||
let hi = last_week.saturating_sub(h + W);
|
||||
fn fit_with(p: &Panel, r: &Rot, h: usize, last_week: usize, events: bool, win: usize) -> AddModel {
|
||||
let mut m = Self::fit_raw(p, r, h, last_week, events, win);
|
||||
let hi = last_week.saturating_sub(h + win);
|
||||
if hi + 1 >= CAL_ORIGINS + LAGS {
|
||||
let c0 = hi + 1 - CAL_ORIGINS;
|
||||
// the inner model may only use windows that closed before the first calibration origin
|
||||
if c0 >= LAGS - 1 + h + W + 4 {
|
||||
let inner = Self::fit_raw(p, r, h, c0, events);
|
||||
if c0 >= LAGS - 1 + h + win + 4 {
|
||||
let inner = Self::fit_raw(p, r, h, c0, events, win);
|
||||
let (mut dc, mut da) = (Design::new(2), Design::new(2));
|
||||
for t in c0..=hi {
|
||||
for a in 0..p.a() {
|
||||
if events && !clean_release(p, a, t + 1) { continue; }
|
||||
let (pc, pa) = inner.predict(p, r, a, t, Scenario::ACTUAL, None);
|
||||
let (pc, pa) = inner.predict(p, r, a, t, Scenario::actual(win), None);
|
||||
for c in 0..C {
|
||||
if pc[c] < 1e-4 && !r.add_in_window(a, c, t, h) { continue; } // certain zeros carry no information
|
||||
dc.push(&[1.0, logit(pc[c])], r.add_in_window(a, c, t, h) as i32 as f64, 1.0);
|
||||
if pc[c] < 1e-4 && !r.add_in(a, c, t, h, win) { continue; } // certain zeros carry no information
|
||||
dc.push(&[1.0, logit(pc[c])], r.add_in(a, c, t, h, win) as i32 as f64, 1.0);
|
||||
}
|
||||
da.push(&[1.0, logit(pa)], r.add_any_in_window(a, t, h) as i32 as f64, 1.0);
|
||||
da.push(&[1.0, logit(pa)], r.add_any_in(a, t, h, win) as i32 as f64, 1.0);
|
||||
}
|
||||
}
|
||||
let (bc, ba) = (glm::logit(&dc, 0.0), glm::logit(&da, 0.0));
|
||||
@@ -336,38 +367,46 @@ impl AddModel {
|
||||
m
|
||||
}
|
||||
|
||||
fn fit_raw(p: &Panel, r: &Rot, h: usize, last_week: usize, events: bool) -> AddModel {
|
||||
fn fit_raw(p: &Panel, r: &Rot, h: usize, last_week: usize, events: bool, win: usize) -> AddModel {
|
||||
let (mut dc, mut da) = (Design::new(PC), Design::new(PA));
|
||||
let (mut x, mut xa) = (vec![0.0; PC], vec![0.0; PA]);
|
||||
for t in origins(h, last_week) {
|
||||
let sc = Scenario::actual(win);
|
||||
for t in origins_w(h, last_week, win) {
|
||||
for a in 0..p.a() {
|
||||
if events && !clean_release(p, a, t + 1) { continue; }
|
||||
for c in 0..C {
|
||||
// Channels where the artist hasn't been heard for 12 weeks: keep a quarter, weighted 4.
|
||||
let quiet = (0..LAGS).all(|i| p.y(a, c, t - i) == 0.0);
|
||||
let wgt = if quiet && !events { if mix((a as u64) << 32 ^ (c as u64) << 20 ^ t as u64 ^ 0xadd) % 4 != 0 { continue; } 4.0 } else { 1.0 };
|
||||
row_channel(p, r, a, c, t, h, Scenario::ACTUAL, &mut x);
|
||||
dc.push(&x, r.add_in_window(a, c, t, h) as i32 as f64, wgt);
|
||||
row_channel(p, r, a, c, t, h, sc, &mut x);
|
||||
dc.push(&x, r.add_in(a, c, t, h, win) as i32 as f64, wgt);
|
||||
}
|
||||
row_any(p, r, a, t, h, Scenario::ACTUAL, &mut xa);
|
||||
da.push(&xa, r.add_any_in_window(a, t, h) as i32 as f64, 1.0);
|
||||
row_any(p, r, a, t, h, sc, &mut xa);
|
||||
da.push(&xa, r.add_any_in(a, t, h, win) as i32 as f64, 1.0);
|
||||
}
|
||||
}
|
||||
AddModel { h, channel: glm::logit(&dc, 1.0), any: glm::logit(&da, 1.0), platt_channel: (0.0, 1.0), platt_any: (0.0, 1.0) }
|
||||
AddModel { h, win, channel: glm::logit(&dc, 1.0), any: glm::logit(&da, 1.0), platt_channel: (0.0, 1.0), platt_any: (0.0, 1.0) }
|
||||
}
|
||||
|
||||
/// (P(add) per channel, P(add on any channel)) from origin t.
|
||||
/// `sc`: what the window holds (Scenario::ACTUAL in backtests).
|
||||
pub fn predict(&self, p: &Panel, r: &Rot, a: usize, t: usize, sc: Scenario, chart: Option<&dyn Fn(&mut [f64], [usize; 4])>) -> ([f64; C], f64) {
|
||||
self.predict_at(p, r, a, t, self.h, sc, chart)
|
||||
}
|
||||
|
||||
/// As `predict`, for a window starting h weeks after t (the release-event model: a release in
|
||||
/// week t+h+1, with the time-since features moved forward by `sc.shift`).
|
||||
pub fn predict_at(&self, p: &Panel, r: &Rot, a: usize, t: usize, h: usize, sc: Scenario, chart: Option<&dyn Fn(&mut [f64], [usize; 4])>) -> ([f64; C], f64) {
|
||||
let sc = sc.window(self.win);
|
||||
let mut x = vec![0.0; PC];
|
||||
let mut pc = [0.0; C];
|
||||
for c in 0..C {
|
||||
row_channel(p, r, a, c, t, self.h, sc, &mut x);
|
||||
row_channel(p, r, a, c, t, h, sc, &mut x);
|
||||
if let Some(f) = chart { f(&mut x, CHART_IDX_CHANNEL); }
|
||||
pc[c] = platt(self.platt_channel, glm::sigmoid(glm::dot(&self.channel, &x)));
|
||||
}
|
||||
let mut xa = vec![0.0; PA];
|
||||
row_any(p, r, a, t, self.h, sc, &mut xa);
|
||||
row_any(p, r, a, t, h, sc, &mut xa);
|
||||
if let Some(f) = chart { f(&mut xa, CHART_IDX_ANY); }
|
||||
// Coherence: "any station" can't exceed what the stations allow together (P4 counted once,
|
||||
// as its best region): at most 1 - prod(1 - p) over the station groups.
|
||||
@@ -510,9 +549,9 @@ pub fn backtest(p: &Panel, tau: &[u32; C], horizons: &[usize], start: usize, md:
|
||||
let m = AddModel::fit(p, &r, 0, p.w() - 1);
|
||||
writeln!(md, "## Coefficients, window starting now (h = 0), all closed windows\n\n| Any-channel feature | Coefficient |\n|---|---|")?;
|
||||
for (n, b) in ANY_NAMES.iter().zip(&m.any) { writeln!(md, "| {n} | {b:+.3} |")?; }
|
||||
for (n, b) in REL_NAMES.iter().chain(WAVE_NAMES).chain(LONG_NAMES).chain(NMF_NAMES).zip(&m.any[21..]) { writeln!(md, "| {n} | {b:+.3} |")?; }
|
||||
for (n, b) in REL_NAMES.iter().chain(WAVE_NAMES).chain(LONG_NAMES).chain(NMF_NAMES).chain(SEASON_NAMES).zip(&m.any[21..]) { writeln!(md, "| {n} | {b:+.3} |")?; }
|
||||
writeln!(md, "\n| Per-channel feature | Coefficient |\n|---|---|")?;
|
||||
for (n, b) in features::NAMES.iter().chain(EXTRA).chain(REL_NAMES).chain(WAVE_NAMES).chain(LONG_NAMES).chain(NMF_NAMES).zip(&m.channel) { writeln!(md, "| {n} | {b:+.3} |")?; }
|
||||
for (n, b) in features::NAMES.iter().chain(EXTRA).chain(REL_NAMES).chain(WAVE_NAMES).chain(LONG_NAMES).chain(NMF_NAMES).chain(SEASON_NAMES).zip(&m.channel) { writeln!(md, "| {n} | {b:+.3} |")?; }
|
||||
writeln!(md)?;
|
||||
Ok(rows)
|
||||
}
|
||||
@@ -596,6 +635,6 @@ pub fn backtest_events(p: &Panel, tau: &[u32; C], start: usize, md: &mut String)
|
||||
writeln!(md, "\nSkill of the event model, any channel: {:.3} ({:.3} to {:.3}) vs in-rotation-now, {:.3} ({:.3} to {:.3}) vs last-12-weeks level.\n", s1.0, s1.1, s1.2, s2.0, s2.1, s2.2)?;
|
||||
let m = AddModel::fit_events(p, &r, p.w() - 1);
|
||||
writeln!(md, "| Any-channel feature (all events) | Coefficient |\n|---|---|")?;
|
||||
for (n, b) in ANY_NAMES.iter().chain(REL_NAMES).chain(WAVE_NAMES).chain(LONG_NAMES).chain(NMF_NAMES).zip(&m.any) { writeln!(md, "| {n} | {b:+.3} |")?; }
|
||||
for (n, b) in ANY_NAMES.iter().chain(REL_NAMES).chain(WAVE_NAMES).chain(LONG_NAMES).chain(NMF_NAMES).chain(SEASON_NAMES).zip(&m.any) { writeln!(md, "| {n} | {b:+.3} |")?; }
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
+27
-1
@@ -81,6 +81,9 @@ pub struct Panel {
|
||||
pub listed_on: Vec<Vec<NaiveDate>>,
|
||||
/// New Music Friday Denmark placements per artist: (Friday, position if known), sorted.
|
||||
pub nmf_dk: Vec<Vec<(NaiveDate, Option<i64>)>>,
|
||||
/// Season index per ISO week (1..=53): new songs reaching the Airplay Top-20 that week of the
|
||||
/// year, relative to an average week, 2002–2024 (before the DR panel), smoothed over 5 weeks.
|
||||
pub season: [f64; 54],
|
||||
}
|
||||
|
||||
impl Panel {
|
||||
@@ -147,7 +150,8 @@ impl Panel {
|
||||
let releases = load_releases(db, &id_of, artists.len())?;
|
||||
let (archive_rate, listed_on) = load_long_history(db, &id_of, &names, artists.len())?;
|
||||
let nmf_dk = load_nmf_dk(db, &id_of, &artists)?;
|
||||
Ok(Panel { weeks, artists, y, chart, list_tau, releases, archive_rate, listed_on, nmf_dk })
|
||||
let season = load_season(db)?;
|
||||
Ok(Panel { weeks, artists, y, chart, list_tau, releases, archive_rate, listed_on, nmf_dk, season })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -310,3 +314,25 @@ fn load_nmf_dk(db: &Db, id_of: &HashMap<String, usize>, artists: &[Artist]) -> R
|
||||
for v in out.iter_mut() { v.sort(); v.dedup_by_key(|x| x.0); }
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn load_season(db: &Db) -> Result<[f64; 54]> {
|
||||
let mut first: HashMap<String, (i32, u32)> = HashMap::new();
|
||||
for r in db.conn.prepare("SELECT year, week, artist, title FROM hitlisten_charts WHERE chart = 'airplay' AND year BETWEEN 2002 AND 2024")?
|
||||
.query_map([], |r| Ok((r.get::<_, i32>(0)?, r.get::<_, u32>(1)?, r.get::<_, String>(2)?, r.get::<_, String>(3)?)))? {
|
||||
let (y, w, a, t) = r?;
|
||||
let k = format!("{}|{}", key(&a), key(t.split(" (").next().unwrap_or(&t)));
|
||||
let e = first.entry(k).or_insert((y, w));
|
||||
if (y, w) < *e { *e = (y, w); }
|
||||
}
|
||||
let mut n = [0.0f64; 54];
|
||||
for (_, w) in first.values() { if (1..=53).contains(w) { n[*w as usize] += 1.0; } }
|
||||
n[53] = n[52]; // week 53 is rare; treat like 52
|
||||
let mut out = [1.0f64; 54];
|
||||
let mean = n[1..=52].iter().sum::<f64>() / 52.0;
|
||||
if mean == 0.0 { return Ok(out); }
|
||||
for w in 1..=53usize {
|
||||
let s: f64 = (-2i32..=2).map(|d| { let k = ((w as i32 - 1 + d).rem_euclid(52) + 1) as usize; n[k] }).sum::<f64>() / 5.0;
|
||||
out[w] = (s / mean).max(0.2);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
+14
-11
@@ -118,12 +118,15 @@ pub fn run(db: &Db, p: &Panel, max_h: usize, show_h: usize) -> Result<Vec<Line>>
|
||||
// window start: the artist's state now, the new song's NMF placement blended from history.
|
||||
let ev = AddModel::fit_events(p, &rot, p.w() - 1);
|
||||
let t0 = p.w() - 1;
|
||||
let event: Vec<([f64; C], f64, f64, f64, bool)> = (0..p.a()).map(|a| {
|
||||
// A new song released in week t0+h+1: time-since features moved forward to then; none if too
|
||||
// soon after the artist's last release.
|
||||
let event = |a: usize, h: usize| -> Option<([f64; C], f64, f64, f64)> {
|
||||
if super::adds::too_soon(p, a, t0, h) { return None; }
|
||||
let q = super::adds::p_nmf(p, a, t0);
|
||||
let (pcn, pan) = ev.predict(p, &rot, a, t0, Scenario::new(true, true), None);
|
||||
let (pcr, par) = ev.predict(p, &rot, a, t0, Scenario::new(true, false), None);
|
||||
(std::array::from_fn(|c| q * pcn[c] + (1.0 - q) * pcr[c]), q * pan + (1.0 - q) * par, pan, q, super::adds::recent_release(p, a, t0))
|
||||
}).collect();
|
||||
let (pcn, pan) = ev.predict_at(p, &rot, a, t0, h, Scenario::new(true, true).at(h), None);
|
||||
let (pcr, par) = ev.predict_at(p, &rot, a, t0, h, Scenario::new(true, false).at(h), None);
|
||||
Some((std::array::from_fn(|c| q * pcn[c] + (1.0 - q) * pcr[c]), q * pan + (1.0 - q) * par, pan, q))
|
||||
};
|
||||
for h in 0..=max_h {
|
||||
let m = AddModel::fit(p, &rot, h, p.w() - 1);
|
||||
let start = (origin + Duration::weeks(h as i64 + 1)).to_string();
|
||||
@@ -137,19 +140,19 @@ pub fn run(db: &Db, p: &Panel, max_h: usize, show_h: usize) -> Result<Vec<Line>>
|
||||
let (pc0, pa0) = m.predict(p, &rot, a, t0, Scenario::new(false, false), None);
|
||||
let pr = super::adds::p_release(p, a, t0);
|
||||
let mix = |p1: f64, p0: f64| match pr { Some(r) => r * p1 + (1.0 - r) * p0, None => p0 };
|
||||
let (ec, ea, ean, q, recent) = event[a];
|
||||
let ev_a = event(a, h);
|
||||
let none = Option::<f64>::None;
|
||||
let mut put = |target: &str, ch: &str, prob: f64| ins.execute(params![made, VERSION, target, format!("{id}:{ch}"), h as i64,
|
||||
origin.to_string(), start, prob, none, none, none]);
|
||||
put("add_any", "any", mix(pa1, pa0))?;
|
||||
put("add_any_rel", "any", ea)?;
|
||||
put("add_any_nmf", "any", ean)?;
|
||||
put("p_nmf", "any", q)?;
|
||||
if recent { put("recent_release", "any", 1.0)?; }
|
||||
match ev_a {
|
||||
Some((_, ea, ean, q)) => { put("add_any_rel", "any", ea)?; put("add_any_nmf", "any", ean)?; put("p_nmf", "any", q)?; }
|
||||
None => { put("too_soon", "any", 1.0)?; }
|
||||
}
|
||||
if let Some(r) = pr { put("p_release", "any", r)?; }
|
||||
for c in 0..C {
|
||||
put("add", CHANNELS[c], mix(pc1[c], pc0[c]))?;
|
||||
put("add_rel", CHANNELS[c], ec[c])?;
|
||||
if let Some((ec, ..)) = ev_a { put("add_rel", CHANNELS[c], ec[c])?; }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user