Files
Airplay-Predictor/src/model/panel.rs
T

186 lines
9.3 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! The artist × DR channel × week panel the first models train on (doc: "The model we can build
//! now"). Plays come from the DR artist pages (52 Monday-to-Sunday weeks per artist); the
//! week still running when the pages were read is left out. IFPI chart history is joined to
//! the DR artists by name, as known at the time: chart week k counts as known from the end of
//! week k + 1 (Hitlisten publishes with about a week's delay).
use crate::db::Db;
use anyhow::{bail, Result};
use chrono::{Duration, NaiveDate, Weekday};
use serde_json::Value;
use std::collections::{BTreeSet, HashMap};
/// The DR music channels with one play count per artist and week (the P4 total is left out:
/// it double counts the ten regional P4s; P1 and P2 are talk and classical).
pub const CHANNELS: [&str; 14] = [
"p3", "p4kbh", "p4sjaelland", "p4fyn", "p4syd", "p4trekanten", "p4vest", "p4aarhus", "p4nord",
"p4bornholm", "p4esbjerg", "p5", "p6beat", "p8jazz",
];
pub const C: usize = CHANNELS.len();
pub fn is_p4(c: usize) -> bool { CHANNELS[c].starts_with("p4") }
pub struct Artist {
pub id: String,
pub name: String,
pub danish: bool,
pub group: bool,
pub female: bool,
pub debut: Option<i32>,
}
/// An artist's chart status as known at the end of a DR week.
#[derive(Clone, Copy)]
pub struct Chart {
/// Best position on the latest Track Top-40 known (0 = not on it).
pub t40_rank: u8,
/// Best position on the latest Airplay Top-20 known (0 = not on it).
pub air_rank: u8,
/// Weeks since the artist's latest new entry on either chart (u16::MAX = never).
pub weeks_since_entry: u16,
/// Distinct songs on the Track Top-40 / Airplay Top-20 so far.
pub t40_hits: u16,
pub air_hits: u16,
}
pub struct Panel {
/// Monday of each complete week, oldest first.
pub weeks: Vec<NaiveDate>,
pub artists: Vec<Artist>,
y: Vec<f32>,
chart: Vec<Chart>,
}
impl Panel {
pub fn w(&self) -> usize { self.weeks.len() }
pub fn a(&self) -> usize { self.artists.len() }
#[inline]
pub fn y(&self, a: usize, c: usize, w: usize) -> f64 { self.y[(a * C + c) * self.weeks.len() + w] as f64 }
#[inline]
pub fn chart(&self, a: usize, w: usize) -> Chart { self.chart[a * self.weeks.len() + w] }
pub fn load(db: &Db) -> Result<Panel> {
// Weeks: every Monday in the artist pages whose week had ended before the earliest page was read.
let first_read: String = db.conn.query_row("SELECT min(fetched_at) FROM dr_artists", [], |r| r.get(0))?;
let read_day = NaiveDate::parse_from_str(&first_read[..10], "%Y-%m-%d")?;
let mut weeks: Vec<NaiveDate> = db.conn.prepare("SELECT DISTINCT week_start FROM dr_artist_weeks ORDER BY week_start")?
.query_map([], |r| r.get::<_, String>(0))?
.filter_map(|s| NaiveDate::parse_from_str(&s.ok()?, "%Y-%m-%d").ok())
.filter(|d| *d + Duration::days(7) <= read_day)
.collect();
weeks.dedup();
if weeks.len() < 30 { bail!("only {} complete weeks of DR artist history; run `airplay collect --only dr --backfill`", weeks.len()); }
let week_of: HashMap<NaiveDate, usize> = weeks.iter().enumerate().map(|(i, d)| (*d, i)).collect();
let mut artists = Vec::new();
let mut names: HashMap<String, Vec<usize>> = HashMap::new();
let mut st = db.conn.prepare("SELECT artist_id, name, raw FROM dr_artists ORDER BY artist_id")?;
let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?)))?;
for r in rows {
let (id, name, raw) = r?;
let v: Value = serde_json::from_str(&raw).unwrap_or(Value::Null);
let a = &v["artist"];
let idx = artists.len();
let mut keys = BTreeSet::from([key(&name)]);
for n in a["names"].as_array().into_iter().flatten() {
if let Some(s) = n["name"].as_str() { keys.insert(key(s)); }
}
for k in keys { if !k.is_empty() { names.entry(k).or_default().push(idx); } }
artists.push(Artist {
id,
name,
danish: a["country"].as_str() == Some("Danmark"),
group: a["type"].as_str().is_some_and(|t| t != "person"),
female: a["gender"].as_str() == Some("kvinde"),
debut: a["lifespan"]["beginYear"].as_i64().map(|y| y as i32),
});
}
let id_of: HashMap<String, usize> = artists.iter().enumerate().map(|(i, a)| (a.id.clone(), i)).collect();
let ch_of: HashMap<&str, usize> = CHANNELS.iter().enumerate().map(|(i, c)| (*c, i)).collect();
let nw = weeks.len();
let mut y = vec![0f32; artists.len() * C * nw];
let mut st = db.conn.prepare("SELECT artist_id, week_start, channel, plays FROM dr_artist_weeks")?;
let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, i64>(3)?)))?;
for r in rows {
let (id, ws, ch, plays) = r?;
let (Some(&a), Some(&c)) = (id_of.get(&id), ch_of.get(ch.as_str())) else { continue };
let Ok(d) = NaiveDate::parse_from_str(&ws, "%Y-%m-%d") else { continue };
let Some(&w) = week_of.get(&d) else { continue };
y[(a * C + c) * nw + w] = plays as f32;
}
let chart = chart_states(db, &weeks, &names, artists.len())?;
Ok(Panel { weeks, artists, y, chart })
}
}
/// Lower-case letters and digits (æ, ø, å kept), for matching names.
pub fn key(s: &str) -> String {
s.to_lowercase().chars().filter(|c| c.is_alphanumeric()).collect()
}
/// The artists named in a chart's artist field ("ANNIKA FEAT. LAMIN" -> annika, lamin).
fn chart_artists(s: &str, names: &HashMap<String, Vec<usize>>) -> Vec<usize> {
let mut found: Vec<usize> = names.get(&key(s)).cloned().unwrap_or_default();
let mut parts = vec![s.to_lowercase()];
for sep in [" feat. ", " feat ", " ft. ", " ft ", " featuring ", " & ", ", ", " x ", " og ", " med ", " vs. ", " vs ", " + ", " / "] {
parts = parts.iter().flat_map(|p| p.split(sep).map(str::to_string).collect::<Vec<_>>()).collect();
}
if parts.len() > 1 {
for p in parts { if let Some(v) = names.get(&key(&p)) { found.extend(v); } }
}
found.sort_unstable();
found.dedup();
found
}
fn chart_states(db: &Db, weeks: &[NaiveDate], names: &HashMap<String, Vec<usize>>, n_artists: usize) -> Result<Vec<Chart>> {
struct Entry { monday: NaiveDate, t40: bool, rank: u8, new: bool, song: String }
let mut by_artist: Vec<Vec<Entry>> = (0..n_artists).map(|_| Vec::new()).collect();
let mut mondays: [BTreeSet<NaiveDate>; 2] = [BTreeSet::new(), BTreeSet::new()];
let mut st = db.conn.prepare(
"SELECT chart, year, week, position, coalesce(last_week, ''), weeks_on_chart, artist, title
FROM hitlisten_charts WHERE chart IN ('t40', 'airplay')")?;
let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i32>(1)?, r.get::<_, u32>(2)?, r.get::<_, i64>(3)?,
r.get::<_, String>(4)?, r.get::<_, Option<i64>>(5)?, r.get::<_, String>(6)?, r.get::<_, String>(7)?)))?;
for r in rows {
let (chart, year, week, pos, last, weeks_on, artist, title) = r?;
let Some(monday) = NaiveDate::from_isoywd_opt(year, week, Weekday::Mon) else { continue };
let t40 = chart == "t40";
mondays[t40 as usize].insert(monday);
let new = weeks_on == Some(1) || last.trim().eq_ignore_ascii_case("NY");
for a in chart_artists(&artist, names) {
by_artist[a].push(Entry { monday, t40, rank: pos.clamp(0, 255) as u8, new, song: key(&title) });
}
}
let nw = weeks.len();
let mut out = Vec::with_capacity(n_artists * nw);
// The latest chart week of each kind known at the end of each DR week.
let latest = |t40: bool, cutoff: NaiveDate| mondays[t40 as usize].range(..=cutoff).next_back().copied();
let cutoffs: Vec<NaiveDate> = weeks.iter().map(|d| *d - Duration::days(7)).collect();
let latest_t40: Vec<Option<NaiveDate>> = cutoffs.iter().map(|c| latest(true, *c)).collect();
let latest_air: Vec<Option<NaiveDate>> = cutoffs.iter().map(|c| latest(false, *c)).collect();
for entries in &by_artist {
for w in 0..nw {
let cutoff = cutoffs[w];
let mut s = Chart { t40_rank: 0, air_rank: 0, weeks_since_entry: u16::MAX, t40_hits: 0, air_hits: 0 };
let (mut t40_songs, mut air_songs) = (BTreeSet::new(), BTreeSet::new());
for e in entries.iter().filter(|e| e.monday <= cutoff) {
if e.t40 { t40_songs.insert(&e.song); } else { air_songs.insert(&e.song); }
let best = |cur: u8| if cur == 0 { e.rank } else { cur.min(e.rank) };
if e.t40 && Some(e.monday) == latest_t40[w] { s.t40_rank = best(s.t40_rank); }
if !e.t40 && Some(e.monday) == latest_air[w] { s.air_rank = best(s.air_rank); }
if e.new {
let since = ((cutoff - e.monday).num_days() / 7).clamp(0, u16::MAX as i64 - 1) as u16;
s.weeks_since_entry = s.weeks_since_entry.min(since);
}
}
s.t40_hits = t40_songs.len() as u16;
s.air_hits = air_songs.len() as u16;
out.push(s);
}
}
Ok(out)
}