Models, first layer: artist × DR channel × week panel with IFPI chart history; NB and logit direct h-step models, Kalman artist form, persistence/popularity baselines; walk-forward backtest (docs/BACKTEST.md) and the ranking (forecasts table)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,185 @@
|
||||
//! The artist × DR channel × week panel the first models train on (doc: "The model we can build
|
||||
//! now"). Plays come from the DR artist pages (52 Monday-to-Sunday weeks per artist); the
|
||||
//! week still running when the pages were read is left out. IFPI chart history is joined to
|
||||
//! the DR artists by name, as known at the time: chart week k counts as known from the end of
|
||||
//! week k + 1 (Hitlisten publishes with about a week's delay).
|
||||
|
||||
use crate::db::Db;
|
||||
use anyhow::{bail, Result};
|
||||
use chrono::{Duration, NaiveDate, Weekday};
|
||||
use serde_json::Value;
|
||||
use std::collections::{BTreeSet, HashMap};
|
||||
|
||||
/// The DR music channels with one play count per artist and week (the P4 total is left out:
|
||||
/// it double counts the ten regional P4s; P1 and P2 are talk and classical).
|
||||
pub const CHANNELS: [&str; 14] = [
|
||||
"p3", "p4kbh", "p4sjaelland", "p4fyn", "p4syd", "p4trekanten", "p4vest", "p4aarhus", "p4nord",
|
||||
"p4bornholm", "p4esbjerg", "p5", "p6beat", "p8jazz",
|
||||
];
|
||||
pub const C: usize = CHANNELS.len();
|
||||
|
||||
pub fn is_p4(c: usize) -> bool { CHANNELS[c].starts_with("p4") }
|
||||
|
||||
pub struct Artist {
|
||||
pub id: String,
|
||||
pub name: String,
|
||||
pub danish: bool,
|
||||
pub group: bool,
|
||||
pub female: bool,
|
||||
pub debut: Option<i32>,
|
||||
}
|
||||
|
||||
/// An artist's chart status as known at the end of a DR week.
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct Chart {
|
||||
/// Best position on the latest Track Top-40 known (0 = not on it).
|
||||
pub t40_rank: u8,
|
||||
/// Best position on the latest Airplay Top-20 known (0 = not on it).
|
||||
pub air_rank: u8,
|
||||
/// Weeks since the artist's latest new entry on either chart (u16::MAX = never).
|
||||
pub weeks_since_entry: u16,
|
||||
/// Distinct songs on the Track Top-40 / Airplay Top-20 so far.
|
||||
pub t40_hits: u16,
|
||||
pub air_hits: u16,
|
||||
}
|
||||
|
||||
pub struct Panel {
|
||||
/// Monday of each complete week, oldest first.
|
||||
pub weeks: Vec<NaiveDate>,
|
||||
pub artists: Vec<Artist>,
|
||||
y: Vec<f32>,
|
||||
chart: Vec<Chart>,
|
||||
}
|
||||
|
||||
impl Panel {
|
||||
pub fn w(&self) -> usize { self.weeks.len() }
|
||||
pub fn a(&self) -> usize { self.artists.len() }
|
||||
#[inline]
|
||||
pub fn y(&self, a: usize, c: usize, w: usize) -> f64 { self.y[(a * C + c) * self.weeks.len() + w] as f64 }
|
||||
#[inline]
|
||||
pub fn chart(&self, a: usize, w: usize) -> Chart { self.chart[a * self.weeks.len() + w] }
|
||||
|
||||
pub fn load(db: &Db) -> Result<Panel> {
|
||||
// Weeks: every Monday in the artist pages whose week had ended before the earliest page was read.
|
||||
let first_read: String = db.conn.query_row("SELECT min(fetched_at) FROM dr_artists", [], |r| r.get(0))?;
|
||||
let read_day = NaiveDate::parse_from_str(&first_read[..10], "%Y-%m-%d")?;
|
||||
let mut weeks: Vec<NaiveDate> = db.conn.prepare("SELECT DISTINCT week_start FROM dr_artist_weeks ORDER BY week_start")?
|
||||
.query_map([], |r| r.get::<_, String>(0))?
|
||||
.filter_map(|s| NaiveDate::parse_from_str(&s.ok()?, "%Y-%m-%d").ok())
|
||||
.filter(|d| *d + Duration::days(7) <= read_day)
|
||||
.collect();
|
||||
weeks.dedup();
|
||||
if weeks.len() < 30 { bail!("only {} complete weeks of DR artist history; run `airplay collect --only dr --backfill`", weeks.len()); }
|
||||
let week_of: HashMap<NaiveDate, usize> = weeks.iter().enumerate().map(|(i, d)| (*d, i)).collect();
|
||||
|
||||
let mut artists = Vec::new();
|
||||
let mut names: HashMap<String, Vec<usize>> = HashMap::new();
|
||||
let mut st = db.conn.prepare("SELECT artist_id, name, raw FROM dr_artists ORDER BY artist_id")?;
|
||||
let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?)))?;
|
||||
for r in rows {
|
||||
let (id, name, raw) = r?;
|
||||
let v: Value = serde_json::from_str(&raw).unwrap_or(Value::Null);
|
||||
let a = &v["artist"];
|
||||
let idx = artists.len();
|
||||
let mut keys = BTreeSet::from([key(&name)]);
|
||||
for n in a["names"].as_array().into_iter().flatten() {
|
||||
if let Some(s) = n["name"].as_str() { keys.insert(key(s)); }
|
||||
}
|
||||
for k in keys { if !k.is_empty() { names.entry(k).or_default().push(idx); } }
|
||||
artists.push(Artist {
|
||||
id,
|
||||
name,
|
||||
danish: a["country"].as_str() == Some("Danmark"),
|
||||
group: a["type"].as_str().is_some_and(|t| t != "person"),
|
||||
female: a["gender"].as_str() == Some("kvinde"),
|
||||
debut: a["lifespan"]["beginYear"].as_i64().map(|y| y as i32),
|
||||
});
|
||||
}
|
||||
let id_of: HashMap<String, usize> = artists.iter().enumerate().map(|(i, a)| (a.id.clone(), i)).collect();
|
||||
let ch_of: HashMap<&str, usize> = CHANNELS.iter().enumerate().map(|(i, c)| (*c, i)).collect();
|
||||
|
||||
let nw = weeks.len();
|
||||
let mut y = vec![0f32; artists.len() * C * nw];
|
||||
let mut st = db.conn.prepare("SELECT artist_id, week_start, channel, plays FROM dr_artist_weeks")?;
|
||||
let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, i64>(3)?)))?;
|
||||
for r in rows {
|
||||
let (id, ws, ch, plays) = r?;
|
||||
let (Some(&a), Some(&c)) = (id_of.get(&id), ch_of.get(ch.as_str())) else { continue };
|
||||
let Ok(d) = NaiveDate::parse_from_str(&ws, "%Y-%m-%d") else { continue };
|
||||
let Some(&w) = week_of.get(&d) else { continue };
|
||||
y[(a * C + c) * nw + w] = plays as f32;
|
||||
}
|
||||
|
||||
let chart = chart_states(db, &weeks, &names, artists.len())?;
|
||||
Ok(Panel { weeks, artists, y, chart })
|
||||
}
|
||||
}
|
||||
|
||||
/// Lower-case letters and digits (æ, ø, å kept), for matching names.
|
||||
pub fn key(s: &str) -> String {
|
||||
s.to_lowercase().chars().filter(|c| c.is_alphanumeric()).collect()
|
||||
}
|
||||
|
||||
/// The artists named in a chart's artist field ("ANNIKA FEAT. LAMIN" -> annika, lamin).
|
||||
fn chart_artists(s: &str, names: &HashMap<String, Vec<usize>>) -> Vec<usize> {
|
||||
let mut found: Vec<usize> = names.get(&key(s)).cloned().unwrap_or_default();
|
||||
let mut parts = vec![s.to_lowercase()];
|
||||
for sep in [" feat. ", " feat ", " ft. ", " ft ", " featuring ", " & ", ", ", " x ", " og ", " med ", " vs. ", " vs ", " + ", " / "] {
|
||||
parts = parts.iter().flat_map(|p| p.split(sep).map(str::to_string).collect::<Vec<_>>()).collect();
|
||||
}
|
||||
if parts.len() > 1 {
|
||||
for p in parts { if let Some(v) = names.get(&key(&p)) { found.extend(v); } }
|
||||
}
|
||||
found.sort_unstable();
|
||||
found.dedup();
|
||||
found
|
||||
}
|
||||
|
||||
fn chart_states(db: &Db, weeks: &[NaiveDate], names: &HashMap<String, Vec<usize>>, n_artists: usize) -> Result<Vec<Chart>> {
|
||||
struct Entry { monday: NaiveDate, t40: bool, rank: u8, new: bool, song: String }
|
||||
let mut by_artist: Vec<Vec<Entry>> = (0..n_artists).map(|_| Vec::new()).collect();
|
||||
let mut mondays: [BTreeSet<NaiveDate>; 2] = [BTreeSet::new(), BTreeSet::new()];
|
||||
let mut st = db.conn.prepare(
|
||||
"SELECT chart, year, week, position, coalesce(last_week, ''), weeks_on_chart, artist, title
|
||||
FROM hitlisten_charts WHERE chart IN ('t40', 'airplay')")?;
|
||||
let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i32>(1)?, r.get::<_, u32>(2)?, r.get::<_, i64>(3)?,
|
||||
r.get::<_, String>(4)?, r.get::<_, Option<i64>>(5)?, r.get::<_, String>(6)?, r.get::<_, String>(7)?)))?;
|
||||
for r in rows {
|
||||
let (chart, year, week, pos, last, weeks_on, artist, title) = r?;
|
||||
let Some(monday) = NaiveDate::from_isoywd_opt(year, week, Weekday::Mon) else { continue };
|
||||
let t40 = chart == "t40";
|
||||
mondays[t40 as usize].insert(monday);
|
||||
let new = weeks_on == Some(1) || last.trim().eq_ignore_ascii_case("NY");
|
||||
for a in chart_artists(&artist, names) {
|
||||
by_artist[a].push(Entry { monday, t40, rank: pos.clamp(0, 255) as u8, new, song: key(&title) });
|
||||
}
|
||||
}
|
||||
let nw = weeks.len();
|
||||
let mut out = Vec::with_capacity(n_artists * nw);
|
||||
// The latest chart week of each kind known at the end of each DR week.
|
||||
let latest = |t40: bool, cutoff: NaiveDate| mondays[t40 as usize].range(..=cutoff).next_back().copied();
|
||||
let cutoffs: Vec<NaiveDate> = weeks.iter().map(|d| *d - Duration::days(7)).collect();
|
||||
let latest_t40: Vec<Option<NaiveDate>> = cutoffs.iter().map(|c| latest(true, *c)).collect();
|
||||
let latest_air: Vec<Option<NaiveDate>> = cutoffs.iter().map(|c| latest(false, *c)).collect();
|
||||
for entries in &by_artist {
|
||||
for w in 0..nw {
|
||||
let cutoff = cutoffs[w];
|
||||
let mut s = Chart { t40_rank: 0, air_rank: 0, weeks_since_entry: u16::MAX, t40_hits: 0, air_hits: 0 };
|
||||
let (mut t40_songs, mut air_songs) = (BTreeSet::new(), BTreeSet::new());
|
||||
for e in entries.iter().filter(|e| e.monday <= cutoff) {
|
||||
if e.t40 { t40_songs.insert(&e.song); } else { air_songs.insert(&e.song); }
|
||||
let best = |cur: u8| if cur == 0 { e.rank } else { cur.min(e.rank) };
|
||||
if e.t40 && Some(e.monday) == latest_t40[w] { s.t40_rank = best(s.t40_rank); }
|
||||
if !e.t40 && Some(e.monday) == latest_air[w] { s.air_rank = best(s.air_rank); }
|
||||
if e.new {
|
||||
let since = ((cutoff - e.monday).num_days() / 7).clamp(0, u16::MAX as i64 - 1) as u16;
|
||||
s.weeks_since_entry = s.weeks_since_entry.min(since);
|
||||
}
|
||||
}
|
||||
s.t40_hits = t40_songs.len() as u16;
|
||||
s.air_hits = air_songs.len() as u16;
|
||||
out.push(s);
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
Reference in New Issue
Block a user