Models, first layer: artist × DR channel × week panel with IFPI chart history; NB and logit direct h-step models, Kalman artist form, persistence/popularity baselines; walk-forward backtest (docs/BACKTEST.md) and the ranking (forecasts table)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-15 02:08:48 +00:00
co-authored by Claude Opus 5.5
parent 6272fbd75d
commit eededc7571
12 changed files with 1318 additions and 1 deletions
+185
View File
@@ -0,0 +1,185 @@
//! The artist × DR channel × week panel the first models train on (doc: "The model we can build
//! now"). Plays come from the DR artist pages (52 Monday-to-Sunday weeks per artist); the
//! week still running when the pages were read is left out. IFPI chart history is joined to
//! the DR artists by name, as known at the time: chart week k counts as known from the end of
//! week k + 1 (Hitlisten publishes with about a week's delay).
use crate::db::Db;
use anyhow::{bail, Result};
use chrono::{Duration, NaiveDate, Weekday};
use serde_json::Value;
use std::collections::{BTreeSet, HashMap};
/// The DR music channels with one play count per artist and week (the P4 total is left out:
/// it double counts the ten regional P4s; P1 and P2 are talk and classical).
pub const CHANNELS: [&str; 14] = [
"p3", "p4kbh", "p4sjaelland", "p4fyn", "p4syd", "p4trekanten", "p4vest", "p4aarhus", "p4nord",
"p4bornholm", "p4esbjerg", "p5", "p6beat", "p8jazz",
];
pub const C: usize = CHANNELS.len();
pub fn is_p4(c: usize) -> bool { CHANNELS[c].starts_with("p4") }
pub struct Artist {
pub id: String,
pub name: String,
pub danish: bool,
pub group: bool,
pub female: bool,
pub debut: Option<i32>,
}
/// An artist's chart status as known at the end of a DR week.
#[derive(Clone, Copy)]
pub struct Chart {
/// Best position on the latest Track Top-40 known (0 = not on it).
pub t40_rank: u8,
/// Best position on the latest Airplay Top-20 known (0 = not on it).
pub air_rank: u8,
/// Weeks since the artist's latest new entry on either chart (u16::MAX = never).
pub weeks_since_entry: u16,
/// Distinct songs on the Track Top-40 / Airplay Top-20 so far.
pub t40_hits: u16,
pub air_hits: u16,
}
pub struct Panel {
/// Monday of each complete week, oldest first.
pub weeks: Vec<NaiveDate>,
pub artists: Vec<Artist>,
y: Vec<f32>,
chart: Vec<Chart>,
}
impl Panel {
pub fn w(&self) -> usize { self.weeks.len() }
pub fn a(&self) -> usize { self.artists.len() }
#[inline]
pub fn y(&self, a: usize, c: usize, w: usize) -> f64 { self.y[(a * C + c) * self.weeks.len() + w] as f64 }
#[inline]
pub fn chart(&self, a: usize, w: usize) -> Chart { self.chart[a * self.weeks.len() + w] }
pub fn load(db: &Db) -> Result<Panel> {
// Weeks: every Monday in the artist pages whose week had ended before the earliest page was read.
let first_read: String = db.conn.query_row("SELECT min(fetched_at) FROM dr_artists", [], |r| r.get(0))?;
let read_day = NaiveDate::parse_from_str(&first_read[..10], "%Y-%m-%d")?;
let mut weeks: Vec<NaiveDate> = db.conn.prepare("SELECT DISTINCT week_start FROM dr_artist_weeks ORDER BY week_start")?
.query_map([], |r| r.get::<_, String>(0))?
.filter_map(|s| NaiveDate::parse_from_str(&s.ok()?, "%Y-%m-%d").ok())
.filter(|d| *d + Duration::days(7) <= read_day)
.collect();
weeks.dedup();
if weeks.len() < 30 { bail!("only {} complete weeks of DR artist history; run `airplay collect --only dr --backfill`", weeks.len()); }
let week_of: HashMap<NaiveDate, usize> = weeks.iter().enumerate().map(|(i, d)| (*d, i)).collect();
let mut artists = Vec::new();
let mut names: HashMap<String, Vec<usize>> = HashMap::new();
let mut st = db.conn.prepare("SELECT artist_id, name, raw FROM dr_artists ORDER BY artist_id")?;
let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?)))?;
for r in rows {
let (id, name, raw) = r?;
let v: Value = serde_json::from_str(&raw).unwrap_or(Value::Null);
let a = &v["artist"];
let idx = artists.len();
let mut keys = BTreeSet::from([key(&name)]);
for n in a["names"].as_array().into_iter().flatten() {
if let Some(s) = n["name"].as_str() { keys.insert(key(s)); }
}
for k in keys { if !k.is_empty() { names.entry(k).or_default().push(idx); } }
artists.push(Artist {
id,
name,
danish: a["country"].as_str() == Some("Danmark"),
group: a["type"].as_str().is_some_and(|t| t != "person"),
female: a["gender"].as_str() == Some("kvinde"),
debut: a["lifespan"]["beginYear"].as_i64().map(|y| y as i32),
});
}
let id_of: HashMap<String, usize> = artists.iter().enumerate().map(|(i, a)| (a.id.clone(), i)).collect();
let ch_of: HashMap<&str, usize> = CHANNELS.iter().enumerate().map(|(i, c)| (*c, i)).collect();
let nw = weeks.len();
let mut y = vec![0f32; artists.len() * C * nw];
let mut st = db.conn.prepare("SELECT artist_id, week_start, channel, plays FROM dr_artist_weeks")?;
let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, i64>(3)?)))?;
for r in rows {
let (id, ws, ch, plays) = r?;
let (Some(&a), Some(&c)) = (id_of.get(&id), ch_of.get(ch.as_str())) else { continue };
let Ok(d) = NaiveDate::parse_from_str(&ws, "%Y-%m-%d") else { continue };
let Some(&w) = week_of.get(&d) else { continue };
y[(a * C + c) * nw + w] = plays as f32;
}
let chart = chart_states(db, &weeks, &names, artists.len())?;
Ok(Panel { weeks, artists, y, chart })
}
}
/// Lower-case letters and digits (æ, ø, å kept), for matching names.
pub fn key(s: &str) -> String {
s.to_lowercase().chars().filter(|c| c.is_alphanumeric()).collect()
}
/// The artists named in a chart's artist field ("ANNIKA FEAT. LAMIN" -> annika, lamin).
fn chart_artists(s: &str, names: &HashMap<String, Vec<usize>>) -> Vec<usize> {
let mut found: Vec<usize> = names.get(&key(s)).cloned().unwrap_or_default();
let mut parts = vec![s.to_lowercase()];
for sep in [" feat. ", " feat ", " ft. ", " ft ", " featuring ", " & ", ", ", " x ", " og ", " med ", " vs. ", " vs ", " + ", " / "] {
parts = parts.iter().flat_map(|p| p.split(sep).map(str::to_string).collect::<Vec<_>>()).collect();
}
if parts.len() > 1 {
for p in parts { if let Some(v) = names.get(&key(&p)) { found.extend(v); } }
}
found.sort_unstable();
found.dedup();
found
}
fn chart_states(db: &Db, weeks: &[NaiveDate], names: &HashMap<String, Vec<usize>>, n_artists: usize) -> Result<Vec<Chart>> {
struct Entry { monday: NaiveDate, t40: bool, rank: u8, new: bool, song: String }
let mut by_artist: Vec<Vec<Entry>> = (0..n_artists).map(|_| Vec::new()).collect();
let mut mondays: [BTreeSet<NaiveDate>; 2] = [BTreeSet::new(), BTreeSet::new()];
let mut st = db.conn.prepare(
"SELECT chart, year, week, position, coalesce(last_week, ''), weeks_on_chart, artist, title
FROM hitlisten_charts WHERE chart IN ('t40', 'airplay')")?;
let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i32>(1)?, r.get::<_, u32>(2)?, r.get::<_, i64>(3)?,
r.get::<_, String>(4)?, r.get::<_, Option<i64>>(5)?, r.get::<_, String>(6)?, r.get::<_, String>(7)?)))?;
for r in rows {
let (chart, year, week, pos, last, weeks_on, artist, title) = r?;
let Some(monday) = NaiveDate::from_isoywd_opt(year, week, Weekday::Mon) else { continue };
let t40 = chart == "t40";
mondays[t40 as usize].insert(monday);
let new = weeks_on == Some(1) || last.trim().eq_ignore_ascii_case("NY");
for a in chart_artists(&artist, names) {
by_artist[a].push(Entry { monday, t40, rank: pos.clamp(0, 255) as u8, new, song: key(&title) });
}
}
let nw = weeks.len();
let mut out = Vec::with_capacity(n_artists * nw);
// The latest chart week of each kind known at the end of each DR week.
let latest = |t40: bool, cutoff: NaiveDate| mondays[t40 as usize].range(..=cutoff).next_back().copied();
let cutoffs: Vec<NaiveDate> = weeks.iter().map(|d| *d - Duration::days(7)).collect();
let latest_t40: Vec<Option<NaiveDate>> = cutoffs.iter().map(|c| latest(true, *c)).collect();
let latest_air: Vec<Option<NaiveDate>> = cutoffs.iter().map(|c| latest(false, *c)).collect();
for entries in &by_artist {
for w in 0..nw {
let cutoff = cutoffs[w];
let mut s = Chart { t40_rank: 0, air_rank: 0, weeks_since_entry: u16::MAX, t40_hits: 0, air_hits: 0 };
let (mut t40_songs, mut air_songs) = (BTreeSet::new(), BTreeSet::new());
for e in entries.iter().filter(|e| e.monday <= cutoff) {
if e.t40 { t40_songs.insert(&e.song); } else { air_songs.insert(&e.song); }
let best = |cur: u8| if cur == 0 { e.rank } else { cur.min(e.rank) };
if e.t40 && Some(e.monday) == latest_t40[w] { s.t40_rank = best(s.t40_rank); }
if !e.t40 && Some(e.monday) == latest_air[w] { s.air_rank = best(s.air_rank); }
if e.new {
let since = ((cutoff - e.monday).num_days() / 7).clamp(0, u16::MAX as i64 - 1) as u16;
s.weeks_since_entry = s.weeks_since_entry.min(since);
}
}
s.t40_hits = t40_songs.len() as u16;
s.air_hits = air_songs.len() as u16;
out.push(s);
}
}
Ok(out)
}