//! The artist × DR channel × week panel the first models train on (doc: "The model we can build //! now"). Plays come from the DR artist pages (52 Monday-to-Sunday weeks per artist); the //! week still running when the pages were read is left out. IFPI chart history is joined to //! the DR artists by name, as known at the time: chart week k counts as known from the end of //! week k + 1 (Hitlisten publishes with about a week's delay). use crate::db::Db; use anyhow::{bail, Result}; use chrono::{Duration, NaiveDate, Weekday}; use serde_json::Value; use std::collections::{BTreeSet, HashMap}; /// The DR music channels with one play count per artist and week (the P4 total is left out: /// it double counts the ten regional P4s; P1 and P2 are talk and classical). pub const CHANNELS: [&str; 14] = [ "p3", "p4kbh", "p4sjaelland", "p4fyn", "p4syd", "p4trekanten", "p4vest", "p4aarhus", "p4nord", "p4bornholm", "p4esbjerg", "p5", "p6beat", "p8jazz", ]; pub const C: usize = CHANNELS.len(); pub fn is_p4(c: usize) -> bool { CHANNELS[c].starts_with("p4") } pub struct Artist { pub id: String, pub name: String, pub danish: bool, pub group: bool, pub female: bool, pub debut: Option, } /// An artist's chart status as known at the end of a DR week. #[derive(Clone, Copy)] pub struct Chart { /// Best position on the latest Track Top-40 known (0 = not on it). pub t40_rank: u8, /// Best position on the latest Airplay Top-20 known (0 = not on it). pub air_rank: u8, /// Weeks since the artist's latest new entry on either chart (u16::MAX = never). pub weeks_since_entry: u16, /// Distinct songs on the Track Top-40 / Airplay Top-20 so far. pub t40_hits: u16, pub air_hits: u16, } pub struct Panel { /// Monday of each complete week, oldest first. pub weeks: Vec, pub artists: Vec, y: Vec, chart: Vec, } impl Panel { pub fn w(&self) -> usize { self.weeks.len() } pub fn a(&self) -> usize { self.artists.len() } #[inline] pub fn y(&self, a: usize, c: usize, w: usize) -> f64 { self.y[(a * C + c) * self.weeks.len() + w] as f64 } #[inline] pub fn chart(&self, a: usize, w: usize) -> Chart { self.chart[a * self.weeks.len() + w] } pub fn load(db: &Db) -> Result { // Weeks: every Monday in the artist pages whose week had ended before the earliest page was read. let first_read: String = db.conn.query_row("SELECT min(fetched_at) FROM dr_artists", [], |r| r.get(0))?; let read_day = NaiveDate::parse_from_str(&first_read[..10], "%Y-%m-%d")?; let mut weeks: Vec = db.conn.prepare("SELECT DISTINCT week_start FROM dr_artist_weeks ORDER BY week_start")? .query_map([], |r| r.get::<_, String>(0))? .filter_map(|s| NaiveDate::parse_from_str(&s.ok()?, "%Y-%m-%d").ok()) .filter(|d| *d + Duration::days(7) <= read_day) .collect(); weeks.dedup(); if weeks.len() < 30 { bail!("only {} complete weeks of DR artist history; run `airplay collect --only dr --backfill`", weeks.len()); } let week_of: HashMap = weeks.iter().enumerate().map(|(i, d)| (*d, i)).collect(); let mut artists = Vec::new(); let mut names: HashMap> = HashMap::new(); let mut st = db.conn.prepare("SELECT artist_id, name, raw FROM dr_artists ORDER BY artist_id")?; let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?)))?; for r in rows { let (id, name, raw) = r?; let v: Value = serde_json::from_str(&raw).unwrap_or(Value::Null); let a = &v["artist"]; let idx = artists.len(); let mut keys = BTreeSet::from([key(&name)]); for n in a["names"].as_array().into_iter().flatten() { if let Some(s) = n["name"].as_str() { keys.insert(key(s)); } } for k in keys { if !k.is_empty() { names.entry(k).or_default().push(idx); } } artists.push(Artist { id, name, danish: a["country"].as_str() == Some("Danmark"), group: a["type"].as_str().is_some_and(|t| t != "person"), female: a["gender"].as_str() == Some("kvinde"), debut: a["lifespan"]["beginYear"].as_i64().map(|y| y as i32), }); } let id_of: HashMap = artists.iter().enumerate().map(|(i, a)| (a.id.clone(), i)).collect(); let ch_of: HashMap<&str, usize> = CHANNELS.iter().enumerate().map(|(i, c)| (*c, i)).collect(); let nw = weeks.len(); let mut y = vec![0f32; artists.len() * C * nw]; let mut st = db.conn.prepare("SELECT artist_id, week_start, channel, plays FROM dr_artist_weeks")?; let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, i64>(3)?)))?; for r in rows { let (id, ws, ch, plays) = r?; let (Some(&a), Some(&c)) = (id_of.get(&id), ch_of.get(ch.as_str())) else { continue }; let Ok(d) = NaiveDate::parse_from_str(&ws, "%Y-%m-%d") else { continue }; let Some(&w) = week_of.get(&d) else { continue }; y[(a * C + c) * nw + w] = plays as f32; } let chart = chart_states(db, &weeks, &names, artists.len())?; Ok(Panel { weeks, artists, y, chart }) } } /// Lower-case letters and digits (æ, ø, å kept), for matching names. pub fn key(s: &str) -> String { s.to_lowercase().chars().filter(|c| c.is_alphanumeric()).collect() } /// The artists named in a chart's artist field ("ANNIKA FEAT. LAMIN" -> annika, lamin). fn chart_artists(s: &str, names: &HashMap>) -> Vec { let mut found: Vec = names.get(&key(s)).cloned().unwrap_or_default(); let mut parts = vec![s.to_lowercase()]; for sep in [" feat. ", " feat ", " ft. ", " ft ", " featuring ", " & ", ", ", " x ", " og ", " med ", " vs. ", " vs ", " + ", " / "] { parts = parts.iter().flat_map(|p| p.split(sep).map(str::to_string).collect::>()).collect(); } if parts.len() > 1 { for p in parts { if let Some(v) = names.get(&key(&p)) { found.extend(v); } } } found.sort_unstable(); found.dedup(); found } fn chart_states(db: &Db, weeks: &[NaiveDate], names: &HashMap>, n_artists: usize) -> Result> { struct Entry { monday: NaiveDate, t40: bool, rank: u8, new: bool, song: String } let mut by_artist: Vec> = (0..n_artists).map(|_| Vec::new()).collect(); let mut mondays: [BTreeSet; 2] = [BTreeSet::new(), BTreeSet::new()]; let mut st = db.conn.prepare( "SELECT chart, year, week, position, coalesce(last_week, ''), weeks_on_chart, artist, title FROM hitlisten_charts WHERE chart IN ('t40', 'airplay')")?; let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i32>(1)?, r.get::<_, u32>(2)?, r.get::<_, i64>(3)?, r.get::<_, String>(4)?, r.get::<_, Option>(5)?, r.get::<_, String>(6)?, r.get::<_, String>(7)?)))?; for r in rows { let (chart, year, week, pos, last, weeks_on, artist, title) = r?; let Some(monday) = NaiveDate::from_isoywd_opt(year, week, Weekday::Mon) else { continue }; let t40 = chart == "t40"; mondays[t40 as usize].insert(monday); let new = weeks_on == Some(1) || last.trim().eq_ignore_ascii_case("NY"); for a in chart_artists(&artist, names) { by_artist[a].push(Entry { monday, t40, rank: pos.clamp(0, 255) as u8, new, song: key(&title) }); } } let nw = weeks.len(); let mut out = Vec::with_capacity(n_artists * nw); // The latest chart week of each kind known at the end of each DR week. let latest = |t40: bool, cutoff: NaiveDate| mondays[t40 as usize].range(..=cutoff).next_back().copied(); let cutoffs: Vec = weeks.iter().map(|d| *d - Duration::days(7)).collect(); let latest_t40: Vec> = cutoffs.iter().map(|c| latest(true, *c)).collect(); let latest_air: Vec> = cutoffs.iter().map(|c| latest(false, *c)).collect(); for entries in &by_artist { for w in 0..nw { let cutoff = cutoffs[w]; let mut s = Chart { t40_rank: 0, air_rank: 0, weeks_since_entry: u16::MAX, t40_hits: 0, air_hits: 0 }; let (mut t40_songs, mut air_songs) = (BTreeSet::new(), BTreeSet::new()); for e in entries.iter().filter(|e| e.monday <= cutoff) { if e.t40 { t40_songs.insert(&e.song); } else { air_songs.insert(&e.song); } let best = |cur: u8| if cur == 0 { e.rank } else { cur.min(e.rank) }; if e.t40 && Some(e.monday) == latest_t40[w] { s.t40_rank = best(s.t40_rank); } if !e.t40 && Some(e.monday) == latest_air[w] { s.air_rank = best(s.air_rank); } if e.new { let since = ((cutoff - e.monday).num_days() / 7).clamp(0, u16::MAX as i64 - 1) as u16; s.weeks_since_entry = s.weeks_since_entry.min(since); } } s.t40_hits = t40_songs.len() as u16; s.air_hits = air_songs.len() as u16; out.push(s); } } Ok(out) }