Files
Airplay-Predictor/src/model/panel.rs
T

469 lines
26 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! The artist × DR channel × week panel the first models train on (doc: "The model we can build
//! now"). Plays come from the DR artist pages (52 Monday-to-Sunday weeks per artist); the
//! week still running when the pages were read is left out. IFPI chart history is joined to
//! the DR artists by name, as known at the time: chart week k counts as known from the end of
//! week k + 1 (Hitlisten publishes with about a week's delay).
use crate::db::Db;
use anyhow::{bail, Result};
use chrono::{Duration, NaiveDate, Weekday};
use serde_json::Value;
use std::collections::{BTreeSet, HashMap};
/// The DR music channels with one play count per artist and week (the P4 total is left out:
/// it double counts the ten regional P4s; P1 and P2 are talk and classical).
pub const CHANNELS: [&str; 14] = [
"p3", "p4kbh", "p4sjaelland", "p4fyn", "p4syd", "p4trekanten", "p4vest", "p4aarhus", "p4nord",
"p4bornholm", "p4esbjerg", "p5", "p6beat", "p8jazz",
];
pub const C: usize = CHANNELS.len();
pub fn is_p4(c: usize) -> bool { CHANNELS[c].starts_with("p4") }
/// Station groups: P3, P4 (the ten regional P4s broadcast mostly the same national programme,
/// so they count as one), P5, P6 Beat, P8 Jazz.
pub const GROUPS: usize = 5;
pub fn group(c: usize) -> usize { match CHANNELS[c] { "p3" => 0, "p5" => 2, "p6beat" => 3, "p8jazz" => 4, _ => 1 } }
impl Panel {
/// The artist's plays in week w across station groups (P4 = the mean of the regions), optionally
/// leaving out one group.
pub fn group_total(&self, a: usize, w: usize, skip: Option<usize>) -> f64 {
let (mut p4, mut n4, mut tot) = (0.0, 0.0, 0.0);
for c in 0..C {
if Some(group(c)) == skip { continue; }
if is_p4(c) { p4 += self.y(a, c, w); n4 += 1.0; } else { tot += self.y(a, c, w); }
}
tot + if n4 > 0.0 { p4 / n4 } else { 0.0 }
}
}
#[derive(Clone)]
pub struct Artist {
pub id: String,
pub name: String,
pub danish: bool,
pub group: bool,
pub female: bool,
pub debut: Option<i32>,
}
/// An artist's chart status as known at the end of a DR week.
#[derive(Clone, Copy)]
pub struct Chart {
/// Best position on the latest Track Top-40 known (0 = not on it).
pub t40_rank: u8,
/// Best position on the latest Airplay Top-20 known (0 = not on it).
pub air_rank: u8,
/// Weeks since the artist's latest new entry on either chart (u16::MAX = never).
pub weeks_since_entry: u16,
/// Distinct songs on the Track Top-40 / Airplay Top-20 so far.
pub t40_hits: u16,
pub air_hits: u16,
/// Airplay Top-20 "waves": weeks since the artist's latest new song on the Airplay Top-20
/// (u16::MAX = never), and the median gap between their earlier new songs (0 = fewer than 2 gaps).
pub air_since: u16,
pub air_gap: u16,
}
#[derive(Clone)]
pub struct Panel {
/// Monday of each complete week, oldest first.
pub weeks: Vec<NaiveDate>,
pub artists: Vec<Artist>,
y: Vec<f32>,
chart: Vec<Chart>,
/// Rotation thresholds fitted to DR's rotation lists, if calibrated.
pub list_tau: Option<[u32; C]>,
/// Release dates per artist (Deezer discography: singles, EPs, albums), sorted; None = not looked up / not found.
pub releases: Vec<Option<Vec<NaiveDate>>>,
/// Long history: the artist's plays per 100 archived hours on DR (Wayback, 2012–2021), and the
/// dates of DR's real rotation lists (2023–) that had the artist on any channel.
pub archive_rate: Vec<f64>,
pub listed_on: Vec<Vec<NaiveDate>>,
/// New Music Friday Denmark placements per artist: (Friday, position if known), sorted.
pub nmf_dk: Vec<Vec<(NaiveDate, Option<i64>)>>,
/// All-time DR plays per station group (P3, P4, P5, P6, P8) from the artist page's most played
/// tracks (P4: the national count, or the regional average when only regions are given).
pub alltime: Vec<[f64; GROUPS]>,
/// How much DR plays the artist's genre on each station group, relative to all music: ln of
/// (genre's share of plays on the group / the group's share of all plays); 0 = genre unknown.
pub genre_fit: Vec<[f64; GROUPS]>,
/// The first week (index) at which the artist was knowable: their first release, DR play,
/// chart entry, DR list, archived play or NMF placement (0 = before the panel; w() = never).
pub first_known: Vec<usize>,
/// Season index per ISO week (1..=53): new songs reaching the Airplay Top-20 that week of the
/// year, relative to an average week, 2002–2024 (before the DR panel), smoothed over 5 weeks.
pub season: [f64; 54],
}
impl Panel {
pub fn w(&self) -> usize { self.weeks.len() }
pub fn a(&self) -> usize { self.artists.len() }
#[inline]
pub fn y(&self, a: usize, c: usize, w: usize) -> f64 { self.y[(a * C + c) * self.weeks.len() + w] as f64 }
#[inline]
pub fn chart(&self, a: usize, w: usize) -> Chart { self.chart[a * self.weeks.len() + w] }
/// Was the artist knowable by the end of week t (so a forecast made then could include them)?
#[inline]
pub fn known(&self, a: usize, t: usize) -> bool { self.first_known[a] <= t }
/// The panel as it was known at the end of week t: plays after t removed (and taken out of the
/// all-time counts), releases, NMF placements and lists after t dropped. For the leak test.
pub fn truncated(&self, t: usize) -> Panel {
let mut q = self.clone();
let nw = self.w();
let end_t = self.weeks[t] + Duration::days(6);
for a in 0..self.a() {
for g in 0..GROUPS {
let cs: Vec<usize> = (0..C).filter(|&c| group(c) == g).collect();
let after: f64 = (t + 1..nw).map(|w| cs.iter().map(|&c| self.y(a, c, w)).sum::<f64>() / cs.len() as f64).sum();
q.alltime[a][g] = (self.alltime[a][g] - after).max(0.0);
}
for c in 0..C { for w in t + 1..nw { q.y[(a * C + c) * nw + w] = 0.0; } }
if let Some(r) = q.releases[a].as_mut() { r.retain(|d| *d <= end_t); }
q.nmf_dk[a].retain(|(d, _)| *d <= end_t);
q.listed_on[a].retain(|d| *d <= end_t);
}
q
}
pub fn load(db: &Db) -> Result<Panel> {
// Weeks: every Monday in the artist pages whose week had ended before the earliest page was read.
let first_read: String = db.conn.query_row("SELECT min(fetched_at) FROM dr_artists", [], |r| r.get(0))?;
let read_day = NaiveDate::parse_from_str(&first_read[..10], "%Y-%m-%d")?;
let mut weeks: Vec<NaiveDate> = db.conn.prepare("SELECT DISTINCT week_start FROM dr_artist_weeks ORDER BY week_start")?
.query_map([], |r| r.get::<_, String>(0))?
.filter_map(|s| NaiveDate::parse_from_str(&s.ok()?, "%Y-%m-%d").ok())
.filter(|d| *d + Duration::days(7) <= read_day)
.collect();
weeks.dedup();
if weeks.len() < 30 { bail!("only {} complete weeks of DR artist history; run `airplay collect --only dr --backfill`", weeks.len()); }
let week_of: HashMap<NaiveDate, usize> = weeks.iter().enumerate().map(|(i, d)| (*d, i)).collect();
let mut artists = Vec::new();
let mut names: HashMap<String, Vec<usize>> = HashMap::new();
let mut st = db.conn.prepare("SELECT artist_id, name, raw FROM dr_artists ORDER BY artist_id")?;
let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?)))?;
for r in rows {
let (id, name, raw) = r?;
let v: Value = serde_json::from_str(&raw).unwrap_or(Value::Null);
let a = &v["artist"];
let idx = artists.len();
let mut keys = BTreeSet::from([key(&name)]);
for n in a["names"].as_array().into_iter().flatten() {
if let Some(s) = n["name"].as_str() { keys.insert(key(s)); }
}
for k in keys { if !k.is_empty() { names.entry(k).or_default().push(idx); } }
artists.push(Artist {
id,
name,
danish: a["country"].as_str() == Some("Danmark"),
group: a["type"].as_str().is_some_and(|t| t != "person"),
female: a["gender"].as_str() == Some("kvinde"),
debut: a["lifespan"]["beginYear"].as_i64().map(|y| y as i32),
});
}
let id_of: HashMap<String, usize> = artists.iter().enumerate().map(|(i, a)| (a.id.clone(), i)).collect();
let ch_of: HashMap<&str, usize> = CHANNELS.iter().enumerate().map(|(i, c)| (*c, i)).collect();
let nw = weeks.len();
let mut y = vec![0f32; artists.len() * C * nw];
let mut st = db.conn.prepare("SELECT artist_id, week_start, channel, plays FROM dr_artist_weeks")?;
let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, i64>(3)?)))?;
for r in rows {
let (id, ws, ch, plays) = r?;
let (Some(&a), Some(&c)) = (id_of.get(&id), ch_of.get(ch.as_str())) else { continue };
let Ok(d) = NaiveDate::parse_from_str(&ws, "%Y-%m-%d") else { continue };
let Some(&w) = week_of.get(&d) else { continue };
y[(a * C + c) * nw + w] = plays as f32;
}
let chart = chart_states(db, &weeks, &names, artists.len())?;
let list_tau = super::calibrate::stored(db)?;
let releases = load_releases(db, &id_of, artists.len())?;
let (archive_rate, listed_on) = load_long_history(db, &id_of, &names, artists.len())?;
let nmf_dk = load_nmf_dk(db, &id_of, &artists)?;
let season = load_season(db)?;
let (alltime, genre_fit) = load_affinity(db, &id_of, artists.len())?;
let mut p = Panel { weeks, artists, y, chart, list_tau, releases, archive_rate, listed_on, nmf_dk, season, alltime, genre_fit, first_known: Vec::new() };
p.first_known = first_known(db, &p, &id_of)?;
Ok(p)
}
}
/// Lower-case letters and digits (æ, ø, å kept), for matching names.
pub fn key(s: &str) -> String {
s.to_lowercase().chars().filter(|c| c.is_alphanumeric()).collect()
}
/// The artists named in a chart's artist field ("ANNIKA FEAT. LAMIN" -> annika, lamin).
fn chart_artists(s: &str, names: &HashMap<String, Vec<usize>>) -> Vec<usize> {
let mut found: Vec<usize> = names.get(&key(s)).cloned().unwrap_or_default();
let mut parts = vec![s.to_lowercase()];
for sep in [" feat. ", " feat ", " ft. ", " ft ", " featuring ", " & ", ", ", " x ", " og ", " med ", " vs. ", " vs ", " + ", " / "] {
parts = parts.iter().flat_map(|p| p.split(sep).map(str::to_string).collect::<Vec<_>>()).collect();
}
if parts.len() > 1 {
for p in parts { if let Some(v) = names.get(&key(&p)) { found.extend(v); } }
}
found.sort_unstable();
found.dedup();
found
}
pub(crate) fn chart_states(db: &Db, weeks: &[NaiveDate], names: &HashMap<String, Vec<usize>>, n_artists: usize) -> Result<Vec<Chart>> {
struct Entry { monday: NaiveDate, t40: bool, rank: u8, new: bool, song: String }
let mut by_artist: Vec<Vec<Entry>> = (0..n_artists).map(|_| Vec::new()).collect();
let mut mondays: [BTreeSet<NaiveDate>; 2] = [BTreeSet::new(), BTreeSet::new()];
let mut st = db.conn.prepare(
"SELECT chart, year, week, position, coalesce(last_week, ''), weeks_on_chart, artist, title
FROM hitlisten_charts WHERE chart IN ('t40', 'airplay')")?;
let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i32>(1)?, r.get::<_, u32>(2)?, r.get::<_, i64>(3)?,
r.get::<_, String>(4)?, r.get::<_, Option<i64>>(5)?, r.get::<_, String>(6)?, r.get::<_, String>(7)?)))?;
for r in rows {
let (chart, year, week, pos, last, weeks_on, artist, title) = r?;
let Some(monday) = NaiveDate::from_isoywd_opt(year, week, Weekday::Mon) else { continue };
let t40 = chart == "t40";
mondays[t40 as usize].insert(monday);
let new = weeks_on == Some(1) || last.trim().eq_ignore_ascii_case("NY");
for a in chart_artists(&artist, names) {
by_artist[a].push(Entry { monday, t40, rank: pos.clamp(0, 255) as u8, new, song: key(&title) });
}
}
let nw = weeks.len();
let mut out = Vec::with_capacity(n_artists * nw);
// The latest chart week of each kind known at the end of each DR week.
let latest = |t40: bool, cutoff: NaiveDate| mondays[t40 as usize].range(..=cutoff).next_back().copied();
let cutoffs: Vec<NaiveDate> = weeks.iter().map(|d| *d - Duration::days(7)).collect();
let latest_t40: Vec<Option<NaiveDate>> = cutoffs.iter().map(|c| latest(true, *c)).collect();
let latest_air: Vec<Option<NaiveDate>> = cutoffs.iter().map(|c| latest(false, *c)).collect();
for entries in &by_artist {
for w in 0..nw {
let cutoff = cutoffs[w];
let mut s = Chart { t40_rank: 0, air_rank: 0, weeks_since_entry: u16::MAX, t40_hits: 0, air_hits: 0, air_since: u16::MAX, air_gap: 0 };
let mut air_firsts: BTreeSet<NaiveDate> = BTreeSet::new();
let (mut t40_songs, mut air_songs) = (BTreeSet::new(), BTreeSet::new());
for e in entries.iter().filter(|e| e.monday <= cutoff) {
if e.t40 { t40_songs.insert(&e.song); } else { air_songs.insert(&e.song); }
if !e.t40 && e.new { air_firsts.insert(e.monday); }
let best = |cur: u8| if cur == 0 { e.rank } else { cur.min(e.rank) };
if e.t40 && Some(e.monday) == latest_t40[w] { s.t40_rank = best(s.t40_rank); }
if !e.t40 && Some(e.monday) == latest_air[w] { s.air_rank = best(s.air_rank); }
if e.new {
let since = ((cutoff - e.monday).num_days() / 7).clamp(0, u16::MAX as i64 - 1) as u16;
s.weeks_since_entry = s.weeks_since_entry.min(since);
}
}
s.t40_hits = t40_songs.len() as u16;
s.air_hits = air_songs.len() as u16;
// Waves: new songs at least 2 weeks apart.
let mut waves: Vec<NaiveDate> = Vec::new();
for d in &air_firsts { if waves.last().is_none_or(|l| (*d - *l).num_days() >= 14) { waves.push(*d); } }
if let Some(last) = waves.last() { s.air_since = ((cutoff - *last).num_days() / 7).clamp(0, u16::MAX as i64 - 1) as u16; }
if waves.len() >= 3 {
let mut gaps: Vec<i64> = waves.windows(2).map(|w| (w[1] - w[0]).num_days() / 7).collect();
gaps.sort();
s.air_gap = gaps[gaps.len() / 2].clamp(1, 2000) as u16;
}
out.push(s);
}
}
Ok(out)
}
pub(crate) fn load_releases(db: &Db, id_of: &HashMap<String, usize>, n: usize) -> Result<Vec<Option<Vec<NaiveDate>>>> {
let mut out: Vec<Option<Vec<NaiveDate>>> = vec![None; n];
let exists: i64 = db.conn.query_row("SELECT count(*) FROM sqlite_master WHERE name = 'deezer_artists'", [], |r| r.get(0))?;
if exists == 0 { return Ok(out); }
for r in db.conn.prepare("SELECT dr_artist_id FROM deezer_artists WHERE deezer_artist_id IS NOT NULL")?.query_map([], |r| r.get::<_, String>(0))? {
if let Some(&a) = id_of.get(&r?) { out[a] = Some(Vec::new()); }
}
for r in db.conn.prepare(
"SELECT d.dr_artist_id, r.release_date FROM deezer_artists d JOIN deezer_releases r ON r.deezer_artist_id = d.deezer_artist_id
WHERE coalesce(r.record_type, '') != 'compile' AND r.release_date > '1900'")?
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? {
let (id, d) = r?;
if let (Some(&a), Ok(d)) = (id_of.get(&id), NaiveDate::parse_from_str(&d, "%Y-%m-%d")) {
if let Some(v) = out[a].as_mut() { v.push(d); }
}
}
for v in out.iter_mut().flatten() { v.sort(); v.dedup(); }
Ok(out)
}
fn load_long_history(db: &Db, id_of: &HashMap<String, usize>, names: &HashMap<String, Vec<usize>>, n: usize) -> Result<(Vec<f64>, Vec<Vec<NaiveDate>>)> {
let mut rate = vec![0.0; n];
let has = |t: &str| -> Result<bool> { Ok(db.conn.query_row("SELECT count(*) FROM sqlite_master WHERE name = ?1", [t], |r| r.get::<_, i64>(0))? > 0) };
if has("archive_plays")? {
let hours: f64 = db.conn.query_row("SELECT count(DISTINCT station || substr(played_at, 1, 13)) FROM archive_plays", [], |r| r.get(0))?;
let mut st = db.conn.prepare("SELECT artist_ids FROM archive_plays WHERE artist_ids != ''")?;
let rows = st.query_map([], |r| r.get::<_, String>(0))?;
for r in rows {
for id in r?.split(',') { if let Some(&a) = id_of.get(id) { rate[a] += 1.0; } }
}
if hours > 0.0 { for v in rate.iter_mut() { *v *= 100.0 / hours; } }
}
let mut listed: Vec<Vec<NaiveDate>> = vec![Vec::new(); n];
if has("rotation_lists")? {
for r in db.conn.prepare("SELECT DISTINCT list_date, artist FROM rotation_lists")?.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? {
let (d, artist) = r?;
let Ok(d) = NaiveDate::parse_from_str(&d, "%Y-%m-%d") else { continue };
let list_artist = artist.replace(" [+] ", ", ");
for a in chart_artists(&list_artist, names) { listed[a].push(d); }
}
}
for v in listed.iter_mut() { v.sort(); v.dedup(); }
Ok((rate, listed))
}
/// NMF Denmark placements per DR artist: the artist's SuperFridayChart placements, confirmed on the
/// track page (country Denmark, with position), or for Danish acts not yet read on the track page,
/// one playlist the size of NMF Denmark that Friday.
pub(crate) fn load_nmf_dk(db: &Db, id_of: &HashMap<String, usize>, artists: &[Artist]) -> Result<Vec<Vec<(NaiveDate, Option<i64>)>>> {
let mut out: Vec<Vec<(NaiveDate, Option<i64>)>> = vec![Vec::new(); artists.len()];
let has = |t: &str| -> Result<bool> { Ok(db.conn.query_row("SELECT count(*) FROM sqlite_master WHERE name = ?1", [t], |r| r.get::<_, i64>(0))? > 0) };
if !has("nmf_artist_tracks")? || !has("nmf_additions")? { return Ok(out); }
let mut confirmed: HashMap<(String, String), Option<i64>> = HashMap::new();
let mut read: std::collections::HashSet<(String, String)> = std::collections::HashSet::new();
let mut size: HashMap<String, f64> = HashMap::new();
for r in db.conn.prepare("SELECT isrc, friday, country, position, followers FROM nmf_additions")?
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, Option<i64>>(3)?, r.get::<_, Option<f64>>(4)?)))? {
let (isrc, f, country, pos, fo) = r?;
read.insert((isrc.clone(), f.clone()));
if country.eq_ignore_ascii_case("denmark") {
if let Some(fo) = fo { let e = size.entry(f.clone()).or_insert(0.0); *e = e.max(fo); }
confirmed.insert((isrc, f), pos);
}
}
for r in db.conn.prepare(
"SELECT l.dr_artist_id, t.isrc, t.friday, coalesce(t.playlists, 0), coalesce(t.followers, 0)
FROM nmf_artist_tracks t JOIN nmf_artist_lookups l ON l.spotify_artist_id = t.spotify_artist_id")?
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, i64>(3)?, r.get::<_, f64>(4)?)))? {
let (id, isrc, f, pl, fo) = r?;
let Some(&a) = id_of.get(&id) else { continue };
let Ok(d) = NaiveDate::parse_from_str(&f, "%Y-%m-%d") else { continue };
let key = (isrc, f.clone());
if let Some(pos) = confirmed.get(&key) { out[a].push((d, *pos)); continue; }
if read.contains(&key) { continue; } // read: not on NMF Denmark
if artists[a].danish && pl == 1 && size.get(&f).is_some_and(|n| (fo - n).abs() <= 0.02 * n) { out[a].push((d, None)); }
}
for v in out.iter_mut() { v.sort(); v.dedup_by_key(|x| x.0); }
Ok(out)
}
fn load_season(db: &Db) -> Result<[f64; 54]> {
let mut first: HashMap<String, (i32, u32)> = HashMap::new();
for r in db.conn.prepare("SELECT year, week, artist, title FROM hitlisten_charts WHERE chart = 'airplay' AND year BETWEEN 2002 AND 2024")?
.query_map([], |r| Ok((r.get::<_, i32>(0)?, r.get::<_, u32>(1)?, r.get::<_, String>(2)?, r.get::<_, String>(3)?)))? {
let (y, w, a, t) = r?;
let k = format!("{}|{}", key(&a), key(t.split(" (").next().unwrap_or(&t)));
let e = first.entry(k).or_insert((y, w));
if (y, w) < *e { *e = (y, w); }
}
let mut n = [0.0f64; 54];
for (_, w) in first.values() { if (1..=53).contains(w) { n[*w as usize] += 1.0; } }
n[53] = n[52]; // week 53 is rare; treat like 52
let mut out = [1.0f64; 54];
let mean = n[1..=52].iter().sum::<f64>() / 52.0;
if mean == 0.0 { return Ok(out); }
for w in 1..=53usize {
let s: f64 = (-2i32..=2).map(|d| { let k = ((w as i32 - 1 + d).rem_euclid(52) + 1) as usize; n[k] }).sum::<f64>() / 5.0;
out[w] = (s / mean).max(0.2);
}
Ok(out)
}
/// Broad genre from DR's description ("Sanger (Afrobeats/Pop/R&B)" -> the first genre that maps).
pub fn genre_bucket(description: &str) -> Option<usize> {
let inner = description.split('(').nth(1)?.split(')').next()?;
for tok in inner.split('/') {
let t = tok.trim().to_lowercase();
let b = if ["hip hop", "hiphop", "rap", "grime", "trap", "drill"].iter().any(|k| t.contains(k)) { 1 }
else if ["jazz", "fusion", "bebop", "swing"].iter().any(|k| t.contains(k)) { 4 }
else if ["klassisk", "filmmusik", "musical", "opera", "kor"].iter().any(|k| t.contains(k)) { 7 }
else if ["r&b", "soul", "funk", "gospel", "afrobeat", "dancehall", "reggae"].iter().any(|k| t.contains(k)) { 5 }
else if ["house", "techno", "elektronisk", "electronica", "electro", "dubstep", "dance", "disco"].iter().any(|k| t.contains(k)) { 3 }
else if ["rock", "punk", "metal", "indie", "new wave", "blues", "grunge"].iter().any(|k| t.contains(k)) { 2 }
else if ["folk", "country", "vise", "singer"].iter().any(|k| t.contains(k)) { 6 }
else if ["latin", "world", "verdens"].iter().any(|k| t.contains(k)) { 8 }
else if ["pop", "dansktop", "schlager", "underholdning"].iter().any(|k| t.contains(k)) { 0 }
else { continue };
return Some(b);
}
None
}
const GENRES: usize = 9;
fn load_affinity(db: &Db, id_of: &HashMap<String, usize>, n: usize) -> Result<(Vec<[f64; GROUPS]>, Vec<[f64; GROUPS]>)> {
let mut alltime = vec![[0.0f64; GROUPS]; n];
let mut genre: Vec<Option<usize>> = vec![None; n];
for r in db.conn.prepare("SELECT artist_id, raw FROM dr_artists")?.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? {
let (id, raw) = r?;
let Some(&a) = id_of.get(&id) else { continue };
let v: Value = serde_json::from_str(&raw).unwrap_or(Value::Null);
genre[a] = v["artist"]["description"].as_str().and_then(genre_bucket);
let (mut p4_nat, mut p4_reg, mut n_reg) = (0.0, 0.0, 0.0);
for t in v["mostPlayed"].as_array().into_iter().flatten() {
for s in t["statisticsByChannels"].as_array().into_iter().flatten() {
let cnt = s["count"].as_f64().unwrap_or(0.0);
match s["channel"]["slug"].as_str().unwrap_or("") {
"p3" => alltime[a][0] += cnt, "p5" => alltime[a][2] += cnt, "p6beat" => alltime[a][3] += cnt, "p8jazz" => alltime[a][4] += cnt,
"p4" => p4_nat += cnt,
s if s.starts_with("p4") => { p4_reg += cnt; n_reg += 1.0; }
_ => {}
}
}
}
alltime[a][1] = p4_nat.max(if n_reg > 0.0 { p4_reg / 10.0 } else { 0.0 });
}
// Genre x station group: share of the genre's plays on each group vs the group's share overall.
let mut by = [[0.0f64; GROUPS]; GENRES];
let mut all = [0.0f64; GROUPS];
for a in 0..n {
for g in 0..GROUPS { all[g] += alltime[a][g]; if let Some(b) = genre[a] { by[b][g] += alltime[a][g]; } }
}
let tot: f64 = all.iter().sum();
let mut fit = vec![[0.0f64; GROUPS]; n];
for a in 0..n {
let Some(b) = genre[a] else { continue };
let tb: f64 = by[b].iter().sum();
if tb <= 0.0 || tot <= 0.0 { continue; }
for g in 0..GROUPS {
fit[a][g] = (((by[b][g] + 1.0) / (tb + 5.0)) / ((all[g] + 1.0) / (tot + 5.0))).ln().clamp(-4.0, 2.0);
}
}
Ok((alltime, fit))
}
/// First knowable week per artist (see Panel::first_known).
fn first_known(db: &Db, p: &Panel, id_of: &HashMap<String, usize>) -> Result<Vec<usize>> {
let nw = p.w();
let week_of_date = |d: NaiveDate| -> usize {
if d < p.weeks[0] { 0 } else { (((d - p.weeks[0]).num_days() / 7) as usize).min(nw) }
};
let mut first = vec![nw; p.a()];
let mut old: HashMap<usize, NaiveDate> = HashMap::new();
let exists: i64 = db.conn.query_row("SELECT count(*) FROM sqlite_master WHERE name = 'dr_artist_history'", [], |r| r.get(0))?;
if exists > 0 {
for r in db.conn.prepare("SELECT artist_id, min(week_start) FROM dr_artist_history WHERE plays > 0 GROUP BY artist_id")?
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? {
let (id, d) = r?;
if let (Some(&a), Ok(d)) = (id_of.get(&id), NaiveDate::parse_from_str(&d, "%Y-%m-%d")) { old.insert(a, d); }
}
}
for a in 0..p.a() {
let mut f = nw;
if let Some(d) = p.releases[a].as_ref().and_then(|v| v.first()) { f = f.min(week_of_date(*d)); }
if let Some(w) = (0..nw).find(|&w| (0..C).any(|c| p.y(a, c, w) > 0.0)) { f = f.min(w); }
if let Some(w) = (0..nw).find(|&w| { let c = p.chart(a, w); c.t40_hits > 0 || c.air_hits > 0 }) { f = f.min(w); }
if let Some(d) = p.listed_on[a].first() { f = f.min(week_of_date(*d)); }
if let Some((d, _)) = p.nmf_dk[a].first() { f = f.min(week_of_date(*d)); }
if p.archive_rate[a] > 0.0 { f = 0; }
if let Some(d) = old.get(&a) { f = f.min(week_of_date(*d)); }
first[a] = f;
}
Ok(first)
}