469 lines
26 KiB
Rust
469 lines
26 KiB
Rust
//! The artist × DR channel × week panel the first models train on (doc: "The model we can build
|
||
//! now"). Plays come from the DR artist pages (52 Monday-to-Sunday weeks per artist); the
|
||
//! week still running when the pages were read is left out. IFPI chart history is joined to
|
||
//! the DR artists by name, as known at the time: chart week k counts as known from the end of
|
||
//! week k + 1 (Hitlisten publishes with about a week's delay).
|
||
|
||
use crate::db::Db;
|
||
use anyhow::{bail, Result};
|
||
use chrono::{Duration, NaiveDate, Weekday};
|
||
use serde_json::Value;
|
||
use std::collections::{BTreeSet, HashMap};
|
||
|
||
/// The DR music channels with one play count per artist and week (the P4 total is left out:
|
||
/// it double counts the ten regional P4s; P1 and P2 are talk and classical).
|
||
pub const CHANNELS: [&str; 14] = [
|
||
"p3", "p4kbh", "p4sjaelland", "p4fyn", "p4syd", "p4trekanten", "p4vest", "p4aarhus", "p4nord",
|
||
"p4bornholm", "p4esbjerg", "p5", "p6beat", "p8jazz",
|
||
];
|
||
pub const C: usize = CHANNELS.len();
|
||
|
||
pub fn is_p4(c: usize) -> bool { CHANNELS[c].starts_with("p4") }
|
||
|
||
/// Station groups: P3, P4 (the ten regional P4s broadcast mostly the same national programme,
|
||
/// so they count as one), P5, P6 Beat, P8 Jazz.
|
||
pub const GROUPS: usize = 5;
|
||
pub fn group(c: usize) -> usize { match CHANNELS[c] { "p3" => 0, "p5" => 2, "p6beat" => 3, "p8jazz" => 4, _ => 1 } }
|
||
|
||
impl Panel {
|
||
/// The artist's plays in week w across station groups (P4 = the mean of the regions), optionally
|
||
/// leaving out one group.
|
||
pub fn group_total(&self, a: usize, w: usize, skip: Option<usize>) -> f64 {
|
||
let (mut p4, mut n4, mut tot) = (0.0, 0.0, 0.0);
|
||
for c in 0..C {
|
||
if Some(group(c)) == skip { continue; }
|
||
if is_p4(c) { p4 += self.y(a, c, w); n4 += 1.0; } else { tot += self.y(a, c, w); }
|
||
}
|
||
tot + if n4 > 0.0 { p4 / n4 } else { 0.0 }
|
||
}
|
||
}
|
||
|
||
#[derive(Clone)]
|
||
pub struct Artist {
|
||
pub id: String,
|
||
pub name: String,
|
||
pub danish: bool,
|
||
pub group: bool,
|
||
pub female: bool,
|
||
pub debut: Option<i32>,
|
||
}
|
||
|
||
/// An artist's chart status as known at the end of a DR week.
|
||
#[derive(Clone, Copy)]
|
||
pub struct Chart {
|
||
/// Best position on the latest Track Top-40 known (0 = not on it).
|
||
pub t40_rank: u8,
|
||
/// Best position on the latest Airplay Top-20 known (0 = not on it).
|
||
pub air_rank: u8,
|
||
/// Weeks since the artist's latest new entry on either chart (u16::MAX = never).
|
||
pub weeks_since_entry: u16,
|
||
/// Distinct songs on the Track Top-40 / Airplay Top-20 so far.
|
||
pub t40_hits: u16,
|
||
pub air_hits: u16,
|
||
/// Airplay Top-20 "waves": weeks since the artist's latest new song on the Airplay Top-20
|
||
/// (u16::MAX = never), and the median gap between their earlier new songs (0 = fewer than 2 gaps).
|
||
pub air_since: u16,
|
||
pub air_gap: u16,
|
||
}
|
||
|
||
#[derive(Clone)]
|
||
pub struct Panel {
|
||
/// Monday of each complete week, oldest first.
|
||
pub weeks: Vec<NaiveDate>,
|
||
pub artists: Vec<Artist>,
|
||
y: Vec<f32>,
|
||
chart: Vec<Chart>,
|
||
/// Rotation thresholds fitted to DR's rotation lists, if calibrated.
|
||
pub list_tau: Option<[u32; C]>,
|
||
/// Release dates per artist (Deezer discography: singles, EPs, albums), sorted; None = not looked up / not found.
|
||
pub releases: Vec<Option<Vec<NaiveDate>>>,
|
||
/// Long history: the artist's plays per 100 archived hours on DR (Wayback, 2012–2021), and the
|
||
/// dates of DR's real rotation lists (2023–) that had the artist on any channel.
|
||
pub archive_rate: Vec<f64>,
|
||
pub listed_on: Vec<Vec<NaiveDate>>,
|
||
/// New Music Friday Denmark placements per artist: (Friday, position if known), sorted.
|
||
pub nmf_dk: Vec<Vec<(NaiveDate, Option<i64>)>>,
|
||
/// All-time DR plays per station group (P3, P4, P5, P6, P8) from the artist page's most played
|
||
/// tracks (P4: the national count, or the regional average when only regions are given).
|
||
pub alltime: Vec<[f64; GROUPS]>,
|
||
/// How much DR plays the artist's genre on each station group, relative to all music: ln of
|
||
/// (genre's share of plays on the group / the group's share of all plays); 0 = genre unknown.
|
||
pub genre_fit: Vec<[f64; GROUPS]>,
|
||
/// The first week (index) at which the artist was knowable: their first release, DR play,
|
||
/// chart entry, DR list, archived play or NMF placement (0 = before the panel; w() = never).
|
||
pub first_known: Vec<usize>,
|
||
/// Season index per ISO week (1..=53): new songs reaching the Airplay Top-20 that week of the
|
||
/// year, relative to an average week, 2002–2024 (before the DR panel), smoothed over 5 weeks.
|
||
pub season: [f64; 54],
|
||
}
|
||
|
||
impl Panel {
|
||
pub fn w(&self) -> usize { self.weeks.len() }
|
||
pub fn a(&self) -> usize { self.artists.len() }
|
||
#[inline]
|
||
pub fn y(&self, a: usize, c: usize, w: usize) -> f64 { self.y[(a * C + c) * self.weeks.len() + w] as f64 }
|
||
#[inline]
|
||
pub fn chart(&self, a: usize, w: usize) -> Chart { self.chart[a * self.weeks.len() + w] }
|
||
|
||
/// Was the artist knowable by the end of week t (so a forecast made then could include them)?
|
||
#[inline]
|
||
pub fn known(&self, a: usize, t: usize) -> bool { self.first_known[a] <= t }
|
||
|
||
/// The panel as it was known at the end of week t: plays after t removed (and taken out of the
|
||
/// all-time counts), releases, NMF placements and lists after t dropped. For the leak test.
|
||
pub fn truncated(&self, t: usize) -> Panel {
|
||
let mut q = self.clone();
|
||
let nw = self.w();
|
||
let end_t = self.weeks[t] + Duration::days(6);
|
||
for a in 0..self.a() {
|
||
for g in 0..GROUPS {
|
||
let cs: Vec<usize> = (0..C).filter(|&c| group(c) == g).collect();
|
||
let after: f64 = (t + 1..nw).map(|w| cs.iter().map(|&c| self.y(a, c, w)).sum::<f64>() / cs.len() as f64).sum();
|
||
q.alltime[a][g] = (self.alltime[a][g] - after).max(0.0);
|
||
}
|
||
for c in 0..C { for w in t + 1..nw { q.y[(a * C + c) * nw + w] = 0.0; } }
|
||
if let Some(r) = q.releases[a].as_mut() { r.retain(|d| *d <= end_t); }
|
||
q.nmf_dk[a].retain(|(d, _)| *d <= end_t);
|
||
q.listed_on[a].retain(|d| *d <= end_t);
|
||
}
|
||
q
|
||
}
|
||
|
||
pub fn load(db: &Db) -> Result<Panel> {
|
||
// Weeks: every Monday in the artist pages whose week had ended before the earliest page was read.
|
||
let first_read: String = db.conn.query_row("SELECT min(fetched_at) FROM dr_artists", [], |r| r.get(0))?;
|
||
let read_day = NaiveDate::parse_from_str(&first_read[..10], "%Y-%m-%d")?;
|
||
let mut weeks: Vec<NaiveDate> = db.conn.prepare("SELECT DISTINCT week_start FROM dr_artist_weeks ORDER BY week_start")?
|
||
.query_map([], |r| r.get::<_, String>(0))?
|
||
.filter_map(|s| NaiveDate::parse_from_str(&s.ok()?, "%Y-%m-%d").ok())
|
||
.filter(|d| *d + Duration::days(7) <= read_day)
|
||
.collect();
|
||
weeks.dedup();
|
||
if weeks.len() < 30 { bail!("only {} complete weeks of DR artist history; run `airplay collect --only dr --backfill`", weeks.len()); }
|
||
let week_of: HashMap<NaiveDate, usize> = weeks.iter().enumerate().map(|(i, d)| (*d, i)).collect();
|
||
|
||
let mut artists = Vec::new();
|
||
let mut names: HashMap<String, Vec<usize>> = HashMap::new();
|
||
let mut st = db.conn.prepare("SELECT artist_id, name, raw FROM dr_artists ORDER BY artist_id")?;
|
||
let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?)))?;
|
||
for r in rows {
|
||
let (id, name, raw) = r?;
|
||
let v: Value = serde_json::from_str(&raw).unwrap_or(Value::Null);
|
||
let a = &v["artist"];
|
||
let idx = artists.len();
|
||
let mut keys = BTreeSet::from([key(&name)]);
|
||
for n in a["names"].as_array().into_iter().flatten() {
|
||
if let Some(s) = n["name"].as_str() { keys.insert(key(s)); }
|
||
}
|
||
for k in keys { if !k.is_empty() { names.entry(k).or_default().push(idx); } }
|
||
artists.push(Artist {
|
||
id,
|
||
name,
|
||
danish: a["country"].as_str() == Some("Danmark"),
|
||
group: a["type"].as_str().is_some_and(|t| t != "person"),
|
||
female: a["gender"].as_str() == Some("kvinde"),
|
||
debut: a["lifespan"]["beginYear"].as_i64().map(|y| y as i32),
|
||
});
|
||
}
|
||
let id_of: HashMap<String, usize> = artists.iter().enumerate().map(|(i, a)| (a.id.clone(), i)).collect();
|
||
let ch_of: HashMap<&str, usize> = CHANNELS.iter().enumerate().map(|(i, c)| (*c, i)).collect();
|
||
|
||
let nw = weeks.len();
|
||
let mut y = vec![0f32; artists.len() * C * nw];
|
||
let mut st = db.conn.prepare("SELECT artist_id, week_start, channel, plays FROM dr_artist_weeks")?;
|
||
let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, i64>(3)?)))?;
|
||
for r in rows {
|
||
let (id, ws, ch, plays) = r?;
|
||
let (Some(&a), Some(&c)) = (id_of.get(&id), ch_of.get(ch.as_str())) else { continue };
|
||
let Ok(d) = NaiveDate::parse_from_str(&ws, "%Y-%m-%d") else { continue };
|
||
let Some(&w) = week_of.get(&d) else { continue };
|
||
y[(a * C + c) * nw + w] = plays as f32;
|
||
}
|
||
|
||
let chart = chart_states(db, &weeks, &names, artists.len())?;
|
||
let list_tau = super::calibrate::stored(db)?;
|
||
let releases = load_releases(db, &id_of, artists.len())?;
|
||
let (archive_rate, listed_on) = load_long_history(db, &id_of, &names, artists.len())?;
|
||
let nmf_dk = load_nmf_dk(db, &id_of, &artists)?;
|
||
let season = load_season(db)?;
|
||
let (alltime, genre_fit) = load_affinity(db, &id_of, artists.len())?;
|
||
let mut p = Panel { weeks, artists, y, chart, list_tau, releases, archive_rate, listed_on, nmf_dk, season, alltime, genre_fit, first_known: Vec::new() };
|
||
p.first_known = first_known(db, &p, &id_of)?;
|
||
Ok(p)
|
||
}
|
||
}
|
||
|
||
/// Lower-case letters and digits (æ, ø, å kept), for matching names.
|
||
pub fn key(s: &str) -> String {
|
||
s.to_lowercase().chars().filter(|c| c.is_alphanumeric()).collect()
|
||
}
|
||
|
||
/// The artists named in a chart's artist field ("ANNIKA FEAT. LAMIN" -> annika, lamin).
|
||
fn chart_artists(s: &str, names: &HashMap<String, Vec<usize>>) -> Vec<usize> {
|
||
let mut found: Vec<usize> = names.get(&key(s)).cloned().unwrap_or_default();
|
||
let mut parts = vec![s.to_lowercase()];
|
||
for sep in [" feat. ", " feat ", " ft. ", " ft ", " featuring ", " & ", ", ", " x ", " og ", " med ", " vs. ", " vs ", " + ", " / "] {
|
||
parts = parts.iter().flat_map(|p| p.split(sep).map(str::to_string).collect::<Vec<_>>()).collect();
|
||
}
|
||
if parts.len() > 1 {
|
||
for p in parts { if let Some(v) = names.get(&key(&p)) { found.extend(v); } }
|
||
}
|
||
found.sort_unstable();
|
||
found.dedup();
|
||
found
|
||
}
|
||
|
||
pub(crate) fn chart_states(db: &Db, weeks: &[NaiveDate], names: &HashMap<String, Vec<usize>>, n_artists: usize) -> Result<Vec<Chart>> {
|
||
struct Entry { monday: NaiveDate, t40: bool, rank: u8, new: bool, song: String }
|
||
let mut by_artist: Vec<Vec<Entry>> = (0..n_artists).map(|_| Vec::new()).collect();
|
||
let mut mondays: [BTreeSet<NaiveDate>; 2] = [BTreeSet::new(), BTreeSet::new()];
|
||
let mut st = db.conn.prepare(
|
||
"SELECT chart, year, week, position, coalesce(last_week, ''), weeks_on_chart, artist, title
|
||
FROM hitlisten_charts WHERE chart IN ('t40', 'airplay')")?;
|
||
let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i32>(1)?, r.get::<_, u32>(2)?, r.get::<_, i64>(3)?,
|
||
r.get::<_, String>(4)?, r.get::<_, Option<i64>>(5)?, r.get::<_, String>(6)?, r.get::<_, String>(7)?)))?;
|
||
for r in rows {
|
||
let (chart, year, week, pos, last, weeks_on, artist, title) = r?;
|
||
let Some(monday) = NaiveDate::from_isoywd_opt(year, week, Weekday::Mon) else { continue };
|
||
let t40 = chart == "t40";
|
||
mondays[t40 as usize].insert(monday);
|
||
let new = weeks_on == Some(1) || last.trim().eq_ignore_ascii_case("NY");
|
||
for a in chart_artists(&artist, names) {
|
||
by_artist[a].push(Entry { monday, t40, rank: pos.clamp(0, 255) as u8, new, song: key(&title) });
|
||
}
|
||
}
|
||
let nw = weeks.len();
|
||
let mut out = Vec::with_capacity(n_artists * nw);
|
||
// The latest chart week of each kind known at the end of each DR week.
|
||
let latest = |t40: bool, cutoff: NaiveDate| mondays[t40 as usize].range(..=cutoff).next_back().copied();
|
||
let cutoffs: Vec<NaiveDate> = weeks.iter().map(|d| *d - Duration::days(7)).collect();
|
||
let latest_t40: Vec<Option<NaiveDate>> = cutoffs.iter().map(|c| latest(true, *c)).collect();
|
||
let latest_air: Vec<Option<NaiveDate>> = cutoffs.iter().map(|c| latest(false, *c)).collect();
|
||
for entries in &by_artist {
|
||
for w in 0..nw {
|
||
let cutoff = cutoffs[w];
|
||
let mut s = Chart { t40_rank: 0, air_rank: 0, weeks_since_entry: u16::MAX, t40_hits: 0, air_hits: 0, air_since: u16::MAX, air_gap: 0 };
|
||
let mut air_firsts: BTreeSet<NaiveDate> = BTreeSet::new();
|
||
let (mut t40_songs, mut air_songs) = (BTreeSet::new(), BTreeSet::new());
|
||
for e in entries.iter().filter(|e| e.monday <= cutoff) {
|
||
if e.t40 { t40_songs.insert(&e.song); } else { air_songs.insert(&e.song); }
|
||
if !e.t40 && e.new { air_firsts.insert(e.monday); }
|
||
let best = |cur: u8| if cur == 0 { e.rank } else { cur.min(e.rank) };
|
||
if e.t40 && Some(e.monday) == latest_t40[w] { s.t40_rank = best(s.t40_rank); }
|
||
if !e.t40 && Some(e.monday) == latest_air[w] { s.air_rank = best(s.air_rank); }
|
||
if e.new {
|
||
let since = ((cutoff - e.monday).num_days() / 7).clamp(0, u16::MAX as i64 - 1) as u16;
|
||
s.weeks_since_entry = s.weeks_since_entry.min(since);
|
||
}
|
||
}
|
||
s.t40_hits = t40_songs.len() as u16;
|
||
s.air_hits = air_songs.len() as u16;
|
||
// Waves: new songs at least 2 weeks apart.
|
||
let mut waves: Vec<NaiveDate> = Vec::new();
|
||
for d in &air_firsts { if waves.last().is_none_or(|l| (*d - *l).num_days() >= 14) { waves.push(*d); } }
|
||
if let Some(last) = waves.last() { s.air_since = ((cutoff - *last).num_days() / 7).clamp(0, u16::MAX as i64 - 1) as u16; }
|
||
if waves.len() >= 3 {
|
||
let mut gaps: Vec<i64> = waves.windows(2).map(|w| (w[1] - w[0]).num_days() / 7).collect();
|
||
gaps.sort();
|
||
s.air_gap = gaps[gaps.len() / 2].clamp(1, 2000) as u16;
|
||
}
|
||
out.push(s);
|
||
}
|
||
}
|
||
Ok(out)
|
||
}
|
||
|
||
pub(crate) fn load_releases(db: &Db, id_of: &HashMap<String, usize>, n: usize) -> Result<Vec<Option<Vec<NaiveDate>>>> {
|
||
let mut out: Vec<Option<Vec<NaiveDate>>> = vec![None; n];
|
||
let exists: i64 = db.conn.query_row("SELECT count(*) FROM sqlite_master WHERE name = 'deezer_artists'", [], |r| r.get(0))?;
|
||
if exists == 0 { return Ok(out); }
|
||
for r in db.conn.prepare("SELECT dr_artist_id FROM deezer_artists WHERE deezer_artist_id IS NOT NULL")?.query_map([], |r| r.get::<_, String>(0))? {
|
||
if let Some(&a) = id_of.get(&r?) { out[a] = Some(Vec::new()); }
|
||
}
|
||
for r in db.conn.prepare(
|
||
"SELECT d.dr_artist_id, r.release_date FROM deezer_artists d JOIN deezer_releases r ON r.deezer_artist_id = d.deezer_artist_id
|
||
WHERE coalesce(r.record_type, '') != 'compile' AND r.release_date > '1900'")?
|
||
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? {
|
||
let (id, d) = r?;
|
||
if let (Some(&a), Ok(d)) = (id_of.get(&id), NaiveDate::parse_from_str(&d, "%Y-%m-%d")) {
|
||
if let Some(v) = out[a].as_mut() { v.push(d); }
|
||
}
|
||
}
|
||
for v in out.iter_mut().flatten() { v.sort(); v.dedup(); }
|
||
Ok(out)
|
||
}
|
||
|
||
fn load_long_history(db: &Db, id_of: &HashMap<String, usize>, names: &HashMap<String, Vec<usize>>, n: usize) -> Result<(Vec<f64>, Vec<Vec<NaiveDate>>)> {
|
||
let mut rate = vec![0.0; n];
|
||
let has = |t: &str| -> Result<bool> { Ok(db.conn.query_row("SELECT count(*) FROM sqlite_master WHERE name = ?1", [t], |r| r.get::<_, i64>(0))? > 0) };
|
||
if has("archive_plays")? {
|
||
let hours: f64 = db.conn.query_row("SELECT count(DISTINCT station || substr(played_at, 1, 13)) FROM archive_plays", [], |r| r.get(0))?;
|
||
let mut st = db.conn.prepare("SELECT artist_ids FROM archive_plays WHERE artist_ids != ''")?;
|
||
let rows = st.query_map([], |r| r.get::<_, String>(0))?;
|
||
for r in rows {
|
||
for id in r?.split(',') { if let Some(&a) = id_of.get(id) { rate[a] += 1.0; } }
|
||
}
|
||
if hours > 0.0 { for v in rate.iter_mut() { *v *= 100.0 / hours; } }
|
||
}
|
||
let mut listed: Vec<Vec<NaiveDate>> = vec![Vec::new(); n];
|
||
if has("rotation_lists")? {
|
||
for r in db.conn.prepare("SELECT DISTINCT list_date, artist FROM rotation_lists")?.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? {
|
||
let (d, artist) = r?;
|
||
let Ok(d) = NaiveDate::parse_from_str(&d, "%Y-%m-%d") else { continue };
|
||
let list_artist = artist.replace(" [+] ", ", ");
|
||
for a in chart_artists(&list_artist, names) { listed[a].push(d); }
|
||
}
|
||
}
|
||
for v in listed.iter_mut() { v.sort(); v.dedup(); }
|
||
Ok((rate, listed))
|
||
}
|
||
|
||
/// NMF Denmark placements per DR artist: the artist's SuperFridayChart placements, confirmed on the
|
||
/// track page (country Denmark, with position), or for Danish acts not yet read on the track page,
|
||
/// one playlist the size of NMF Denmark that Friday.
|
||
pub(crate) fn load_nmf_dk(db: &Db, id_of: &HashMap<String, usize>, artists: &[Artist]) -> Result<Vec<Vec<(NaiveDate, Option<i64>)>>> {
|
||
let mut out: Vec<Vec<(NaiveDate, Option<i64>)>> = vec![Vec::new(); artists.len()];
|
||
let has = |t: &str| -> Result<bool> { Ok(db.conn.query_row("SELECT count(*) FROM sqlite_master WHERE name = ?1", [t], |r| r.get::<_, i64>(0))? > 0) };
|
||
if !has("nmf_artist_tracks")? || !has("nmf_additions")? { return Ok(out); }
|
||
let mut confirmed: HashMap<(String, String), Option<i64>> = HashMap::new();
|
||
let mut read: std::collections::HashSet<(String, String)> = std::collections::HashSet::new();
|
||
let mut size: HashMap<String, f64> = HashMap::new();
|
||
for r in db.conn.prepare("SELECT isrc, friday, country, position, followers FROM nmf_additions")?
|
||
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, Option<i64>>(3)?, r.get::<_, Option<f64>>(4)?)))? {
|
||
let (isrc, f, country, pos, fo) = r?;
|
||
read.insert((isrc.clone(), f.clone()));
|
||
if country.eq_ignore_ascii_case("denmark") {
|
||
if let Some(fo) = fo { let e = size.entry(f.clone()).or_insert(0.0); *e = e.max(fo); }
|
||
confirmed.insert((isrc, f), pos);
|
||
}
|
||
}
|
||
for r in db.conn.prepare(
|
||
"SELECT l.dr_artist_id, t.isrc, t.friday, coalesce(t.playlists, 0), coalesce(t.followers, 0)
|
||
FROM nmf_artist_tracks t JOIN nmf_artist_lookups l ON l.spotify_artist_id = t.spotify_artist_id")?
|
||
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, i64>(3)?, r.get::<_, f64>(4)?)))? {
|
||
let (id, isrc, f, pl, fo) = r?;
|
||
let Some(&a) = id_of.get(&id) else { continue };
|
||
let Ok(d) = NaiveDate::parse_from_str(&f, "%Y-%m-%d") else { continue };
|
||
let key = (isrc, f.clone());
|
||
if let Some(pos) = confirmed.get(&key) { out[a].push((d, *pos)); continue; }
|
||
if read.contains(&key) { continue; } // read: not on NMF Denmark
|
||
if artists[a].danish && pl == 1 && size.get(&f).is_some_and(|n| (fo - n).abs() <= 0.02 * n) { out[a].push((d, None)); }
|
||
}
|
||
for v in out.iter_mut() { v.sort(); v.dedup_by_key(|x| x.0); }
|
||
Ok(out)
|
||
}
|
||
|
||
fn load_season(db: &Db) -> Result<[f64; 54]> {
|
||
let mut first: HashMap<String, (i32, u32)> = HashMap::new();
|
||
for r in db.conn.prepare("SELECT year, week, artist, title FROM hitlisten_charts WHERE chart = 'airplay' AND year BETWEEN 2002 AND 2024")?
|
||
.query_map([], |r| Ok((r.get::<_, i32>(0)?, r.get::<_, u32>(1)?, r.get::<_, String>(2)?, r.get::<_, String>(3)?)))? {
|
||
let (y, w, a, t) = r?;
|
||
let k = format!("{}|{}", key(&a), key(t.split(" (").next().unwrap_or(&t)));
|
||
let e = first.entry(k).or_insert((y, w));
|
||
if (y, w) < *e { *e = (y, w); }
|
||
}
|
||
let mut n = [0.0f64; 54];
|
||
for (_, w) in first.values() { if (1..=53).contains(w) { n[*w as usize] += 1.0; } }
|
||
n[53] = n[52]; // week 53 is rare; treat like 52
|
||
let mut out = [1.0f64; 54];
|
||
let mean = n[1..=52].iter().sum::<f64>() / 52.0;
|
||
if mean == 0.0 { return Ok(out); }
|
||
for w in 1..=53usize {
|
||
let s: f64 = (-2i32..=2).map(|d| { let k = ((w as i32 - 1 + d).rem_euclid(52) + 1) as usize; n[k] }).sum::<f64>() / 5.0;
|
||
out[w] = (s / mean).max(0.2);
|
||
}
|
||
Ok(out)
|
||
}
|
||
|
||
/// Broad genre from DR's description ("Sanger (Afrobeats/Pop/R&B)" -> the first genre that maps).
|
||
pub fn genre_bucket(description: &str) -> Option<usize> {
|
||
let inner = description.split('(').nth(1)?.split(')').next()?;
|
||
for tok in inner.split('/') {
|
||
let t = tok.trim().to_lowercase();
|
||
let b = if ["hip hop", "hiphop", "rap", "grime", "trap", "drill"].iter().any(|k| t.contains(k)) { 1 }
|
||
else if ["jazz", "fusion", "bebop", "swing"].iter().any(|k| t.contains(k)) { 4 }
|
||
else if ["klassisk", "filmmusik", "musical", "opera", "kor"].iter().any(|k| t.contains(k)) { 7 }
|
||
else if ["r&b", "soul", "funk", "gospel", "afrobeat", "dancehall", "reggae"].iter().any(|k| t.contains(k)) { 5 }
|
||
else if ["house", "techno", "elektronisk", "electronica", "electro", "dubstep", "dance", "disco"].iter().any(|k| t.contains(k)) { 3 }
|
||
else if ["rock", "punk", "metal", "indie", "new wave", "blues", "grunge"].iter().any(|k| t.contains(k)) { 2 }
|
||
else if ["folk", "country", "vise", "singer"].iter().any(|k| t.contains(k)) { 6 }
|
||
else if ["latin", "world", "verdens"].iter().any(|k| t.contains(k)) { 8 }
|
||
else if ["pop", "dansktop", "schlager", "underholdning"].iter().any(|k| t.contains(k)) { 0 }
|
||
else { continue };
|
||
return Some(b);
|
||
}
|
||
None
|
||
}
|
||
const GENRES: usize = 9;
|
||
|
||
fn load_affinity(db: &Db, id_of: &HashMap<String, usize>, n: usize) -> Result<(Vec<[f64; GROUPS]>, Vec<[f64; GROUPS]>)> {
|
||
let mut alltime = vec![[0.0f64; GROUPS]; n];
|
||
let mut genre: Vec<Option<usize>> = vec![None; n];
|
||
for r in db.conn.prepare("SELECT artist_id, raw FROM dr_artists")?.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? {
|
||
let (id, raw) = r?;
|
||
let Some(&a) = id_of.get(&id) else { continue };
|
||
let v: Value = serde_json::from_str(&raw).unwrap_or(Value::Null);
|
||
genre[a] = v["artist"]["description"].as_str().and_then(genre_bucket);
|
||
let (mut p4_nat, mut p4_reg, mut n_reg) = (0.0, 0.0, 0.0);
|
||
for t in v["mostPlayed"].as_array().into_iter().flatten() {
|
||
for s in t["statisticsByChannels"].as_array().into_iter().flatten() {
|
||
let cnt = s["count"].as_f64().unwrap_or(0.0);
|
||
match s["channel"]["slug"].as_str().unwrap_or("") {
|
||
"p3" => alltime[a][0] += cnt, "p5" => alltime[a][2] += cnt, "p6beat" => alltime[a][3] += cnt, "p8jazz" => alltime[a][4] += cnt,
|
||
"p4" => p4_nat += cnt,
|
||
s if s.starts_with("p4") => { p4_reg += cnt; n_reg += 1.0; }
|
||
_ => {}
|
||
}
|
||
}
|
||
}
|
||
alltime[a][1] = p4_nat.max(if n_reg > 0.0 { p4_reg / 10.0 } else { 0.0 });
|
||
}
|
||
// Genre x station group: share of the genre's plays on each group vs the group's share overall.
|
||
let mut by = [[0.0f64; GROUPS]; GENRES];
|
||
let mut all = [0.0f64; GROUPS];
|
||
for a in 0..n {
|
||
for g in 0..GROUPS { all[g] += alltime[a][g]; if let Some(b) = genre[a] { by[b][g] += alltime[a][g]; } }
|
||
}
|
||
let tot: f64 = all.iter().sum();
|
||
let mut fit = vec![[0.0f64; GROUPS]; n];
|
||
for a in 0..n {
|
||
let Some(b) = genre[a] else { continue };
|
||
let tb: f64 = by[b].iter().sum();
|
||
if tb <= 0.0 || tot <= 0.0 { continue; }
|
||
for g in 0..GROUPS {
|
||
fit[a][g] = (((by[b][g] + 1.0) / (tb + 5.0)) / ((all[g] + 1.0) / (tot + 5.0))).ln().clamp(-4.0, 2.0);
|
||
}
|
||
}
|
||
Ok((alltime, fit))
|
||
}
|
||
|
||
/// First knowable week per artist (see Panel::first_known).
|
||
fn first_known(db: &Db, p: &Panel, id_of: &HashMap<String, usize>) -> Result<Vec<usize>> {
|
||
let nw = p.w();
|
||
let week_of_date = |d: NaiveDate| -> usize {
|
||
if d < p.weeks[0] { 0 } else { (((d - p.weeks[0]).num_days() / 7) as usize).min(nw) }
|
||
};
|
||
let mut first = vec![nw; p.a()];
|
||
let mut old: HashMap<usize, NaiveDate> = HashMap::new();
|
||
let exists: i64 = db.conn.query_row("SELECT count(*) FROM sqlite_master WHERE name = 'dr_artist_history'", [], |r| r.get(0))?;
|
||
if exists > 0 {
|
||
for r in db.conn.prepare("SELECT artist_id, min(week_start) FROM dr_artist_history WHERE plays > 0 GROUP BY artist_id")?
|
||
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? {
|
||
let (id, d) = r?;
|
||
if let (Some(&a), Ok(d)) = (id_of.get(&id), NaiveDate::parse_from_str(&d, "%Y-%m-%d")) { old.insert(a, d); }
|
||
}
|
||
}
|
||
for a in 0..p.a() {
|
||
let mut f = nw;
|
||
if let Some(d) = p.releases[a].as_ref().and_then(|v| v.first()) { f = f.min(week_of_date(*d)); }
|
||
if let Some(w) = (0..nw).find(|&w| (0..C).any(|c| p.y(a, c, w) > 0.0)) { f = f.min(w); }
|
||
if let Some(w) = (0..nw).find(|&w| { let c = p.chart(a, w); c.t40_hits > 0 || c.air_hits > 0 }) { f = f.min(w); }
|
||
if let Some(d) = p.listed_on[a].first() { f = f.min(week_of_date(*d)); }
|
||
if let Some((d, _)) = p.nmf_dk[a].first() { f = f.min(week_of_date(*d)); }
|
||
if p.archive_rate[a] > 0.0 { f = 0; }
|
||
if let Some(d) = old.get(&a) { f = f.min(week_of_date(*d)); }
|
||
first[a] = f;
|
||
}
|
||
Ok(first)
|
||
}
|