//! The artist × DR channel × week panel the first models train on (doc: "The model we can build //! now"). Plays come from the DR artist pages (52 Monday-to-Sunday weeks per artist); the //! week still running when the pages were read is left out. IFPI chart history is joined to //! the DR artists by name, as known at the time: chart week k counts as known from the end of //! week k + 1 (Hitlisten publishes with about a week's delay). use crate::db::Db; use anyhow::{bail, Result}; use chrono::{Duration, NaiveDate, Weekday}; use serde_json::Value; use std::collections::{BTreeSet, HashMap}; /// The DR music channels with one play count per artist and week (the P4 total is left out: /// it double counts the ten regional P4s; P1 and P2 are talk and classical). pub const CHANNELS: [&str; 14] = [ "p3", "p4kbh", "p4sjaelland", "p4fyn", "p4syd", "p4trekanten", "p4vest", "p4aarhus", "p4nord", "p4bornholm", "p4esbjerg", "p5", "p6beat", "p8jazz", ]; pub const C: usize = CHANNELS.len(); pub fn is_p4(c: usize) -> bool { CHANNELS[c].starts_with("p4") } /// Station groups: P3, P4 (the ten regional P4s broadcast mostly the same national programme, /// so they count as one), P5, P6 Beat, P8 Jazz. pub const GROUPS: usize = 5; pub fn group(c: usize) -> usize { match CHANNELS[c] { "p3" => 0, "p5" => 2, "p6beat" => 3, "p8jazz" => 4, _ => 1 } } impl Panel { /// The artist's plays in week w across station groups (P4 = the mean of the regions), optionally /// leaving out one group. pub fn group_total(&self, a: usize, w: usize, skip: Option) -> f64 { let (mut p4, mut n4, mut tot) = (0.0, 0.0, 0.0); for c in 0..C { if Some(group(c)) == skip { continue; } if is_p4(c) { p4 += self.y(a, c, w); n4 += 1.0; } else { tot += self.y(a, c, w); } } tot + if n4 > 0.0 { p4 / n4 } else { 0.0 } } } #[derive(Clone)] pub struct Artist { pub id: String, pub name: String, pub danish: bool, pub group: bool, pub female: bool, pub debut: Option, } /// An artist's chart status as known at the end of a DR week. #[derive(Clone, Copy)] pub struct Chart { /// Best position on the latest Track Top-40 known (0 = not on it). pub t40_rank: u8, /// Best position on the latest Airplay Top-20 known (0 = not on it). pub air_rank: u8, /// Weeks since the artist's latest new entry on either chart (u16::MAX = never). pub weeks_since_entry: u16, /// Distinct songs on the Track Top-40 / Airplay Top-20 so far. pub t40_hits: u16, pub air_hits: u16, /// Airplay Top-20 "waves": weeks since the artist's latest new song on the Airplay Top-20 /// (u16::MAX = never), and the median gap between their earlier new songs (0 = fewer than 2 gaps). pub air_since: u16, pub air_gap: u16, } #[derive(Clone)] pub struct Panel { /// Monday of each complete week, oldest first. pub weeks: Vec, pub artists: Vec, y: Vec, chart: Vec, /// Rotation thresholds fitted to DR's rotation lists, if calibrated. pub list_tau: Option<[u32; C]>, /// Release dates per artist (Deezer discography: singles, EPs, albums), sorted; None = not looked up / not found. pub releases: Vec>>, /// Long history: the artist's plays per 100 archived hours on DR (Wayback, 2012–2021), and the /// dates of DR's real rotation lists (2023–) that had the artist on any channel. pub archive_rate: Vec, pub listed_on: Vec>, /// New Music Friday Denmark placements per artist: (Friday, position if known), sorted. pub nmf_dk: Vec)>>, /// All-time DR plays per station group (P3, P4, P5, P6, P8) from the artist page's most played /// tracks (P4: the national count, or the regional average when only regions are given). pub alltime: Vec<[f64; GROUPS]>, /// How much DR plays the artist's genre on each station group, relative to all music: ln of /// (genre's share of plays on the group / the group's share of all plays); 0 = genre unknown. pub genre_fit: Vec<[f64; GROUPS]>, /// The first week (index) at which the artist was knowable: their first release, DR play, /// chart entry, DR list, archived play or NMF placement (0 = before the panel; w() = never). pub first_known: Vec, /// Season index per ISO week (1..=53): new songs reaching the Airplay Top-20 that week of the /// year, relative to an average week, 2002–2024 (before the DR panel), smoothed over 5 weeks. pub season: [f64; 54], } impl Panel { pub fn w(&self) -> usize { self.weeks.len() } pub fn a(&self) -> usize { self.artists.len() } #[inline] pub fn y(&self, a: usize, c: usize, w: usize) -> f64 { self.y[(a * C + c) * self.weeks.len() + w] as f64 } #[inline] pub fn chart(&self, a: usize, w: usize) -> Chart { self.chart[a * self.weeks.len() + w] } /// Was the artist knowable by the end of week t (so a forecast made then could include them)? #[inline] pub fn known(&self, a: usize, t: usize) -> bool { self.first_known[a] <= t } /// The panel as it was known at the end of week t: plays after t removed (and taken out of the /// all-time counts), releases, NMF placements and lists after t dropped. For the leak test. pub fn truncated(&self, t: usize) -> Panel { let mut q = self.clone(); let nw = self.w(); let end_t = self.weeks[t] + Duration::days(6); for a in 0..self.a() { for g in 0..GROUPS { let cs: Vec = (0..C).filter(|&c| group(c) == g).collect(); let after: f64 = (t + 1..nw).map(|w| cs.iter().map(|&c| self.y(a, c, w)).sum::() / cs.len() as f64).sum(); q.alltime[a][g] = (self.alltime[a][g] - after).max(0.0); } for c in 0..C { for w in t + 1..nw { q.y[(a * C + c) * nw + w] = 0.0; } } if let Some(r) = q.releases[a].as_mut() { r.retain(|d| *d <= end_t); } q.nmf_dk[a].retain(|(d, _)| *d <= end_t); q.listed_on[a].retain(|d| *d <= end_t); } q } pub fn load(db: &Db) -> Result { // Weeks: every Monday in the artist pages whose week had ended before the earliest page was read. let first_read: String = db.conn.query_row("SELECT min(fetched_at) FROM dr_artists", [], |r| r.get(0))?; let read_day = NaiveDate::parse_from_str(&first_read[..10], "%Y-%m-%d")?; let mut weeks: Vec = db.conn.prepare("SELECT DISTINCT week_start FROM dr_artist_weeks ORDER BY week_start")? .query_map([], |r| r.get::<_, String>(0))? .filter_map(|s| NaiveDate::parse_from_str(&s.ok()?, "%Y-%m-%d").ok()) .filter(|d| *d + Duration::days(7) <= read_day) .collect(); weeks.dedup(); if weeks.len() < 30 { bail!("only {} complete weeks of DR artist history; run `airplay collect --only dr --backfill`", weeks.len()); } let week_of: HashMap = weeks.iter().enumerate().map(|(i, d)| (*d, i)).collect(); let mut artists = Vec::new(); let mut names: HashMap> = HashMap::new(); let mut st = db.conn.prepare("SELECT artist_id, name, raw FROM dr_artists ORDER BY artist_id")?; let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?)))?; for r in rows { let (id, name, raw) = r?; let v: Value = serde_json::from_str(&raw).unwrap_or(Value::Null); let a = &v["artist"]; let idx = artists.len(); let mut keys = BTreeSet::from([key(&name)]); for n in a["names"].as_array().into_iter().flatten() { if let Some(s) = n["name"].as_str() { keys.insert(key(s)); } } for k in keys { if !k.is_empty() { names.entry(k).or_default().push(idx); } } artists.push(Artist { id, name, danish: a["country"].as_str() == Some("Danmark"), group: a["type"].as_str().is_some_and(|t| t != "person"), female: a["gender"].as_str() == Some("kvinde"), debut: a["lifespan"]["beginYear"].as_i64().map(|y| y as i32), }); } let id_of: HashMap = artists.iter().enumerate().map(|(i, a)| (a.id.clone(), i)).collect(); let ch_of: HashMap<&str, usize> = CHANNELS.iter().enumerate().map(|(i, c)| (*c, i)).collect(); let nw = weeks.len(); let mut y = vec![0f32; artists.len() * C * nw]; let mut st = db.conn.prepare("SELECT artist_id, week_start, channel, plays FROM dr_artist_weeks")?; let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, i64>(3)?)))?; for r in rows { let (id, ws, ch, plays) = r?; let (Some(&a), Some(&c)) = (id_of.get(&id), ch_of.get(ch.as_str())) else { continue }; let Ok(d) = NaiveDate::parse_from_str(&ws, "%Y-%m-%d") else { continue }; let Some(&w) = week_of.get(&d) else { continue }; y[(a * C + c) * nw + w] = plays as f32; } let chart = chart_states(db, &weeks, &names, artists.len())?; let list_tau = super::calibrate::stored(db)?; let releases = load_releases(db, &id_of, artists.len())?; let (archive_rate, listed_on) = load_long_history(db, &id_of, &names, artists.len())?; let nmf_dk = load_nmf_dk(db, &id_of, &artists)?; let season = load_season(db)?; let (alltime, genre_fit) = load_affinity(db, &id_of, artists.len())?; let mut p = Panel { weeks, artists, y, chart, list_tau, releases, archive_rate, listed_on, nmf_dk, season, alltime, genre_fit, first_known: Vec::new() }; p.first_known = first_known(db, &p, &id_of)?; Ok(p) } } /// Lower-case letters and digits (æ, ø, å kept), for matching names. pub fn key(s: &str) -> String { s.to_lowercase().chars().filter(|c| c.is_alphanumeric()).collect() } /// The artists named in a chart's artist field ("ANNIKA FEAT. LAMIN" -> annika, lamin). fn chart_artists(s: &str, names: &HashMap>) -> Vec { let mut found: Vec = names.get(&key(s)).cloned().unwrap_or_default(); let mut parts = vec![s.to_lowercase()]; for sep in [" feat. ", " feat ", " ft. ", " ft ", " featuring ", " & ", ", ", " x ", " og ", " med ", " vs. ", " vs ", " + ", " / "] { parts = parts.iter().flat_map(|p| p.split(sep).map(str::to_string).collect::>()).collect(); } if parts.len() > 1 { for p in parts { if let Some(v) = names.get(&key(&p)) { found.extend(v); } } } found.sort_unstable(); found.dedup(); found } pub(crate) fn chart_states(db: &Db, weeks: &[NaiveDate], names: &HashMap>, n_artists: usize) -> Result> { struct Entry { monday: NaiveDate, t40: bool, rank: u8, new: bool, song: String } let mut by_artist: Vec> = (0..n_artists).map(|_| Vec::new()).collect(); let mut mondays: [BTreeSet; 2] = [BTreeSet::new(), BTreeSet::new()]; let mut st = db.conn.prepare( "SELECT chart, year, week, position, coalesce(last_week, ''), weeks_on_chart, artist, title FROM hitlisten_charts WHERE chart IN ('t40', 'airplay')")?; let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i32>(1)?, r.get::<_, u32>(2)?, r.get::<_, i64>(3)?, r.get::<_, String>(4)?, r.get::<_, Option>(5)?, r.get::<_, String>(6)?, r.get::<_, String>(7)?)))?; for r in rows { let (chart, year, week, pos, last, weeks_on, artist, title) = r?; let Some(monday) = NaiveDate::from_isoywd_opt(year, week, Weekday::Mon) else { continue }; let t40 = chart == "t40"; mondays[t40 as usize].insert(monday); let new = weeks_on == Some(1) || last.trim().eq_ignore_ascii_case("NY"); for a in chart_artists(&artist, names) { by_artist[a].push(Entry { monday, t40, rank: pos.clamp(0, 255) as u8, new, song: key(&title) }); } } let nw = weeks.len(); let mut out = Vec::with_capacity(n_artists * nw); // The latest chart week of each kind known at the end of each DR week. let latest = |t40: bool, cutoff: NaiveDate| mondays[t40 as usize].range(..=cutoff).next_back().copied(); let cutoffs: Vec = weeks.iter().map(|d| *d - Duration::days(7)).collect(); let latest_t40: Vec> = cutoffs.iter().map(|c| latest(true, *c)).collect(); let latest_air: Vec> = cutoffs.iter().map(|c| latest(false, *c)).collect(); for entries in &by_artist { for w in 0..nw { let cutoff = cutoffs[w]; let mut s = Chart { t40_rank: 0, air_rank: 0, weeks_since_entry: u16::MAX, t40_hits: 0, air_hits: 0, air_since: u16::MAX, air_gap: 0 }; let mut air_firsts: BTreeSet = BTreeSet::new(); let (mut t40_songs, mut air_songs) = (BTreeSet::new(), BTreeSet::new()); for e in entries.iter().filter(|e| e.monday <= cutoff) { if e.t40 { t40_songs.insert(&e.song); } else { air_songs.insert(&e.song); } if !e.t40 && e.new { air_firsts.insert(e.monday); } let best = |cur: u8| if cur == 0 { e.rank } else { cur.min(e.rank) }; if e.t40 && Some(e.monday) == latest_t40[w] { s.t40_rank = best(s.t40_rank); } if !e.t40 && Some(e.monday) == latest_air[w] { s.air_rank = best(s.air_rank); } if e.new { let since = ((cutoff - e.monday).num_days() / 7).clamp(0, u16::MAX as i64 - 1) as u16; s.weeks_since_entry = s.weeks_since_entry.min(since); } } s.t40_hits = t40_songs.len() as u16; s.air_hits = air_songs.len() as u16; // Waves: new songs at least 2 weeks apart. let mut waves: Vec = Vec::new(); for d in &air_firsts { if waves.last().is_none_or(|l| (*d - *l).num_days() >= 14) { waves.push(*d); } } if let Some(last) = waves.last() { s.air_since = ((cutoff - *last).num_days() / 7).clamp(0, u16::MAX as i64 - 1) as u16; } if waves.len() >= 3 { let mut gaps: Vec = waves.windows(2).map(|w| (w[1] - w[0]).num_days() / 7).collect(); gaps.sort(); s.air_gap = gaps[gaps.len() / 2].clamp(1, 2000) as u16; } out.push(s); } } Ok(out) } pub(crate) fn load_releases(db: &Db, id_of: &HashMap, n: usize) -> Result>>> { let mut out: Vec>> = vec![None; n]; let exists: i64 = db.conn.query_row("SELECT count(*) FROM sqlite_master WHERE name = 'deezer_artists'", [], |r| r.get(0))?; if exists == 0 { return Ok(out); } for r in db.conn.prepare("SELECT dr_artist_id FROM deezer_artists WHERE deezer_artist_id IS NOT NULL")?.query_map([], |r| r.get::<_, String>(0))? { if let Some(&a) = id_of.get(&r?) { out[a] = Some(Vec::new()); } } for r in db.conn.prepare( "SELECT d.dr_artist_id, r.release_date FROM deezer_artists d JOIN deezer_releases r ON r.deezer_artist_id = d.deezer_artist_id WHERE coalesce(r.record_type, '') != 'compile' AND r.release_date > '1900'")? .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? { let (id, d) = r?; if let (Some(&a), Ok(d)) = (id_of.get(&id), NaiveDate::parse_from_str(&d, "%Y-%m-%d")) { if let Some(v) = out[a].as_mut() { v.push(d); } } } for v in out.iter_mut().flatten() { v.sort(); v.dedup(); } Ok(out) } fn load_long_history(db: &Db, id_of: &HashMap, names: &HashMap>, n: usize) -> Result<(Vec, Vec>)> { let mut rate = vec![0.0; n]; let has = |t: &str| -> Result { Ok(db.conn.query_row("SELECT count(*) FROM sqlite_master WHERE name = ?1", [t], |r| r.get::<_, i64>(0))? > 0) }; if has("archive_plays")? { let hours: f64 = db.conn.query_row("SELECT count(DISTINCT station || substr(played_at, 1, 13)) FROM archive_plays", [], |r| r.get(0))?; let mut st = db.conn.prepare("SELECT artist_ids FROM archive_plays WHERE artist_ids != ''")?; let rows = st.query_map([], |r| r.get::<_, String>(0))?; for r in rows { for id in r?.split(',') { if let Some(&a) = id_of.get(id) { rate[a] += 1.0; } } } if hours > 0.0 { for v in rate.iter_mut() { *v *= 100.0 / hours; } } } let mut listed: Vec> = vec![Vec::new(); n]; if has("rotation_lists")? { for r in db.conn.prepare("SELECT DISTINCT list_date, artist FROM rotation_lists")?.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? { let (d, artist) = r?; let Ok(d) = NaiveDate::parse_from_str(&d, "%Y-%m-%d") else { continue }; let list_artist = artist.replace(" [+] ", ", "); for a in chart_artists(&list_artist, names) { listed[a].push(d); } } } for v in listed.iter_mut() { v.sort(); v.dedup(); } Ok((rate, listed)) } /// NMF Denmark placements per DR artist: the artist's SuperFridayChart placements, confirmed on the /// track page (country Denmark, with position), or for Danish acts not yet read on the track page, /// one playlist the size of NMF Denmark that Friday. pub(crate) fn load_nmf_dk(db: &Db, id_of: &HashMap, artists: &[Artist]) -> Result)>>> { let mut out: Vec)>> = vec![Vec::new(); artists.len()]; let has = |t: &str| -> Result { Ok(db.conn.query_row("SELECT count(*) FROM sqlite_master WHERE name = ?1", [t], |r| r.get::<_, i64>(0))? > 0) }; if !has("nmf_artist_tracks")? || !has("nmf_additions")? { return Ok(out); } let mut confirmed: HashMap<(String, String), Option> = HashMap::new(); let mut read: std::collections::HashSet<(String, String)> = std::collections::HashSet::new(); let mut size: HashMap = HashMap::new(); for r in db.conn.prepare("SELECT isrc, friday, country, position, followers FROM nmf_additions")? .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, Option>(3)?, r.get::<_, Option>(4)?)))? { let (isrc, f, country, pos, fo) = r?; read.insert((isrc.clone(), f.clone())); if country.eq_ignore_ascii_case("denmark") { if let Some(fo) = fo { let e = size.entry(f.clone()).or_insert(0.0); *e = e.max(fo); } confirmed.insert((isrc, f), pos); } } for r in db.conn.prepare( "SELECT l.dr_artist_id, t.isrc, t.friday, coalesce(t.playlists, 0), coalesce(t.followers, 0) FROM nmf_artist_tracks t JOIN nmf_artist_lookups l ON l.spotify_artist_id = t.spotify_artist_id")? .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, i64>(3)?, r.get::<_, f64>(4)?)))? { let (id, isrc, f, pl, fo) = r?; let Some(&a) = id_of.get(&id) else { continue }; let Ok(d) = NaiveDate::parse_from_str(&f, "%Y-%m-%d") else { continue }; let key = (isrc, f.clone()); if let Some(pos) = confirmed.get(&key) { out[a].push((d, *pos)); continue; } if read.contains(&key) { continue; } // read: not on NMF Denmark if artists[a].danish && pl == 1 && size.get(&f).is_some_and(|n| (fo - n).abs() <= 0.02 * n) { out[a].push((d, None)); } } for v in out.iter_mut() { v.sort(); v.dedup_by_key(|x| x.0); } Ok(out) } fn load_season(db: &Db) -> Result<[f64; 54]> { let mut first: HashMap = HashMap::new(); for r in db.conn.prepare("SELECT year, week, artist, title FROM hitlisten_charts WHERE chart = 'airplay' AND year BETWEEN 2002 AND 2024")? .query_map([], |r| Ok((r.get::<_, i32>(0)?, r.get::<_, u32>(1)?, r.get::<_, String>(2)?, r.get::<_, String>(3)?)))? { let (y, w, a, t) = r?; let k = format!("{}|{}", key(&a), key(t.split(" (").next().unwrap_or(&t))); let e = first.entry(k).or_insert((y, w)); if (y, w) < *e { *e = (y, w); } } let mut n = [0.0f64; 54]; for (_, w) in first.values() { if (1..=53).contains(w) { n[*w as usize] += 1.0; } } n[53] = n[52]; // week 53 is rare; treat like 52 let mut out = [1.0f64; 54]; let mean = n[1..=52].iter().sum::() / 52.0; if mean == 0.0 { return Ok(out); } for w in 1..=53usize { let s: f64 = (-2i32..=2).map(|d| { let k = ((w as i32 - 1 + d).rem_euclid(52) + 1) as usize; n[k] }).sum::() / 5.0; out[w] = (s / mean).max(0.2); } Ok(out) } /// Broad genre from DR's description ("Sanger (Afrobeats/Pop/R&B)" -> the first genre that maps). pub fn genre_bucket(description: &str) -> Option { let inner = description.split('(').nth(1)?.split(')').next()?; for tok in inner.split('/') { let t = tok.trim().to_lowercase(); let b = if ["hip hop", "hiphop", "rap", "grime", "trap", "drill"].iter().any(|k| t.contains(k)) { 1 } else if ["jazz", "fusion", "bebop", "swing"].iter().any(|k| t.contains(k)) { 4 } else if ["klassisk", "filmmusik", "musical", "opera", "kor"].iter().any(|k| t.contains(k)) { 7 } else if ["r&b", "soul", "funk", "gospel", "afrobeat", "dancehall", "reggae"].iter().any(|k| t.contains(k)) { 5 } else if ["house", "techno", "elektronisk", "electronica", "electro", "dubstep", "dance", "disco"].iter().any(|k| t.contains(k)) { 3 } else if ["rock", "punk", "metal", "indie", "new wave", "blues", "grunge"].iter().any(|k| t.contains(k)) { 2 } else if ["folk", "country", "vise", "singer"].iter().any(|k| t.contains(k)) { 6 } else if ["latin", "world", "verdens"].iter().any(|k| t.contains(k)) { 8 } else if ["pop", "dansktop", "schlager", "underholdning"].iter().any(|k| t.contains(k)) { 0 } else { continue }; return Some(b); } None } const GENRES: usize = 9; fn load_affinity(db: &Db, id_of: &HashMap, n: usize) -> Result<(Vec<[f64; GROUPS]>, Vec<[f64; GROUPS]>)> { let mut alltime = vec![[0.0f64; GROUPS]; n]; let mut genre: Vec> = vec![None; n]; for r in db.conn.prepare("SELECT artist_id, raw FROM dr_artists")?.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? { let (id, raw) = r?; let Some(&a) = id_of.get(&id) else { continue }; let v: Value = serde_json::from_str(&raw).unwrap_or(Value::Null); genre[a] = v["artist"]["description"].as_str().and_then(genre_bucket); let (mut p4_nat, mut p4_reg, mut n_reg) = (0.0, 0.0, 0.0); for t in v["mostPlayed"].as_array().into_iter().flatten() { for s in t["statisticsByChannels"].as_array().into_iter().flatten() { let cnt = s["count"].as_f64().unwrap_or(0.0); match s["channel"]["slug"].as_str().unwrap_or("") { "p3" => alltime[a][0] += cnt, "p5" => alltime[a][2] += cnt, "p6beat" => alltime[a][3] += cnt, "p8jazz" => alltime[a][4] += cnt, "p4" => p4_nat += cnt, s if s.starts_with("p4") => { p4_reg += cnt; n_reg += 1.0; } _ => {} } } } alltime[a][1] = p4_nat.max(if n_reg > 0.0 { p4_reg / 10.0 } else { 0.0 }); } // Genre x station group: share of the genre's plays on each group vs the group's share overall. let mut by = [[0.0f64; GROUPS]; GENRES]; let mut all = [0.0f64; GROUPS]; for a in 0..n { for g in 0..GROUPS { all[g] += alltime[a][g]; if let Some(b) = genre[a] { by[b][g] += alltime[a][g]; } } } let tot: f64 = all.iter().sum(); let mut fit = vec![[0.0f64; GROUPS]; n]; for a in 0..n { let Some(b) = genre[a] else { continue }; let tb: f64 = by[b].iter().sum(); if tb <= 0.0 || tot <= 0.0 { continue; } for g in 0..GROUPS { fit[a][g] = (((by[b][g] + 1.0) / (tb + 5.0)) / ((all[g] + 1.0) / (tot + 5.0))).ln().clamp(-4.0, 2.0); } } Ok((alltime, fit)) } /// First knowable week per artist (see Panel::first_known). fn first_known(db: &Db, p: &Panel, id_of: &HashMap) -> Result> { let nw = p.w(); let week_of_date = |d: NaiveDate| -> usize { if d < p.weeks[0] { 0 } else { (((d - p.weeks[0]).num_days() / 7) as usize).min(nw) } }; let mut first = vec![nw; p.a()]; let mut old: HashMap = HashMap::new(); let exists: i64 = db.conn.query_row("SELECT count(*) FROM sqlite_master WHERE name = 'dr_artist_history'", [], |r| r.get(0))?; if exists > 0 { for r in db.conn.prepare("SELECT artist_id, min(week_start) FROM dr_artist_history WHERE plays > 0 GROUP BY artist_id")? .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? { let (id, d) = r?; if let (Some(&a), Ok(d)) = (id_of.get(&id), NaiveDate::parse_from_str(&d, "%Y-%m-%d")) { old.insert(a, d); } } } for a in 0..p.a() { let mut f = nw; if let Some(d) = p.releases[a].as_ref().and_then(|v| v.first()) { f = f.min(week_of_date(*d)); } if let Some(w) = (0..nw).find(|&w| (0..C).any(|c| p.y(a, c, w) > 0.0)) { f = f.min(w); } if let Some(w) = (0..nw).find(|&w| { let c = p.chart(a, w); c.t40_hits > 0 || c.air_hits > 0 }) { f = f.min(w); } if let Some(d) = p.listed_on[a].first() { f = f.min(week_of_date(*d)); } if let Some((d, _)) = p.nmf_dk[a].first() { f = f.min(week_of_date(*d)); } if p.archive_rate[a] > 0.0 { f = 0; } if let Some(d) = old.get(&a) { f = f.min(week_of_date(*d)); } first[a] = f; } Ok(first) }