//! Rotation thresholds from the real lists. For every parsed rotation list whose week lies in //! the DR panel, compare the artists on the list (per channel; the national P4 list against each //! regional P4) with the artists' plays that week. Reports plays by category and the threshold //! that best separates listed from not-listed artists, and stores the thresholds in //! `rotation_thresholds`, which the models then use instead of the play-distribution guess. use super::panel::{key, Panel, C, CHANNELS}; use crate::db::Db; use anyhow::Result; use chrono::{Duration, NaiveDate}; use std::collections::{HashMap, HashSet}; use std::fmt::Write as _; pub fn create_tables(db: &Db) -> Result<()> { db.conn.execute_batch( "CREATE TABLE IF NOT EXISTS rotation_thresholds ( channel TEXT PRIMARY KEY, tau INTEGER NOT NULL, f1 REAL, listed INTEGER, weeks INTEGER, made_at TEXT NOT NULL );", )?; Ok(()) } /// Thresholds stored by `run`, if any. pub fn stored(db: &Db) -> Result> { create_tables(db)?; let mut tau = [0u32; C]; let mut n = 0; for r in db.conn.prepare("SELECT channel, tau FROM rotation_thresholds")?.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?)))? { let (ch, t) = r?; if let Some(c) = CHANNELS.iter().position(|x| *x == ch) { tau[c] = t as u32; n += 1; } } Ok(if n == C { Some(tau) } else { None }) } /// The artists credited in a list line ("A [+] B", "A feat. B", "A & B", "A, B"). fn names(s: &str) -> Vec { let mut parts = vec![s.to_lowercase()]; for sep in [" [+] ", " feat. ", " feat ", " ft. ", " ft ", " featuring ", " & ", ", ", " x ", " og ", " med ", " vs. ", " vs "] { parts = parts.iter().flat_map(|p| p.split(sep).map(str::to_string).collect::>()).collect(); } let mut out = vec![key(s)]; out.extend(parts.iter().map(|p| key(p))); out.retain(|k| !k.is_empty()); out } pub fn run(db: &Db, p: &Panel) -> Result { create_tables(db)?; let mut by_key: HashMap> = HashMap::new(); for (i, a) in p.artists.iter().enumerate() { by_key.entry(key(&a.name)).or_default().push(i); } let week_of: HashMap = p.weeks.iter().enumerate().map(|(i, d)| (*d, i)).collect(); // (list channel, week index) -> artist -> best category let mut listed: HashMap<(String, usize), HashMap> = HashMap::new(); let mut unmatched = 0usize; let mut total = 0usize; let rank = |c: &str| match c { "A" => 0, "B" => 1, "C" => 2, _ => 3 }; for r in db.conn.prepare("SELECT list_date, channel, category, artist FROM rotation_lists")? .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, String>(3)?)))? { let (date, ch, cat, artist) = r?; let Ok(d) = NaiveDate::parse_from_str(&date, "%Y-%m-%d") else { continue }; // A Friday list applies to the week starting the following Monday. let Some(&w) = week_of.get(&(d + Duration::days(3))) else { continue }; total += 1; let ids: HashSet = names(&artist).iter().filter_map(|k| by_key.get(k)).flatten().copied().collect(); if ids.is_empty() { unmatched += 1; continue; } let m = listed.entry((ch, w)).or_default(); for a in ids { let e = m.entry(a).or_insert_with(|| cat.clone()); if rank(&cat) < rank(e) { *e = cat.clone(); } } } let mut md = String::new(); writeln!(md, "# Rotation thresholds from DR's lists\n")?; let weeks: HashSet = listed.keys().map(|k| k.1).collect(); let mut wl: Vec = weeks.iter().map(|w| p.weeks[*w].to_string()).collect(); wl.sort(); writeln!(md, "Lists inside the DR panel: weeks starting {}. List lines in those weeks: {total}; artist not among the DR artists read: {unmatched}.\n", wl.join(", "))?; writeln!(md, "| Channel | τ (best F1) | F1 | Listed artist-weeks | Median plays A | B | C | Median plays, not listed but played | Old τ |\n|---|---|---|---|---|---|---|---|---|")?; let old = super::common::thresholds_from_plays(p, p.w()); let made = chrono::Utc::now().to_rfc3339(); let mut out_tau = [0u32; C]; for c in 0..C { let list_ch = if CHANNELS[c].starts_with("p4") { "p4" } else { CHANNELS[c] }; let mut by_cat: HashMap> = HashMap::new(); let mut not_listed: Vec = Vec::new(); let mut pairs: Vec<(f64, bool)> = Vec::new(); let mut n_weeks = 0; for &w in &weeks { let Some(m) = listed.get(&(list_ch.to_string(), w)) else { continue }; n_weeks += 1; for a in 0..p.a() { let y = p.y(a, c, w); match m.get(&a) { Some(cat) => { by_cat.entry(cat.clone()).or_default().push(y); pairs.push((y, true)); } None => { if y > 0.0 { not_listed.push(y); } pairs.push((y, false)); } } } } if n_weeks == 0 { out_tau[c] = old[c]; continue; } let med = |v: Option<&Vec>| v.filter(|v| !v.is_empty()).map(|v| { let mut v = v.clone(); v.sort_by(f64::total_cmp); v[v.len() / 2] }).map(|m| format!("{m:.0}")).unwrap_or("—".into()); // Best threshold by F1 of "plays >= tau" against "on the list". let (mut best, mut best_f1) = (old[c], 0.0); for tau in 1..=40u32 { let (mut tp, mut fp, mut fn_) = (0.0, 0.0, 0.0); for (y, l) in &pairs { let pred = *y >= tau as f64; match (pred, *l) { (true, true) => tp += 1.0, (true, false) => fp += 1.0, (false, true) => fn_ += 1.0, _ => {} } } let f1 = if tp > 0.0 { 2.0 * tp / (2.0 * tp + fp + fn_) } else { 0.0 }; if f1 > best_f1 { best_f1 = f1; best = tau; } } out_tau[c] = best; let n_listed = pairs.iter().filter(|x| x.1).count(); let mut nl = not_listed.clone(); nl.sort_by(f64::total_cmp); writeln!(md, "| {} | {best} | {best_f1:.2} | {n_listed} | {} | {} | {} | {} | {} |", CHANNELS[c], med(by_cat.get("A")), med(by_cat.get("B")), med(by_cat.get("C")), med(Some(&nl)), old[c])?; db.conn.execute("INSERT OR REPLACE INTO rotation_thresholds VALUES (?1, ?2, ?3, ?4, ?5, ?6)", rusqlite::params![CHANNELS[c], best as i64, best_f1, n_listed as i64, n_weeks as i64, made])?; } writeln!(md, "\nThe models now read these thresholds (table `rotation_thresholds`). Listed artist-weeks count an artist once per week even with several songs listed; plays are the artist's total that week, including off-list plays.\n")?; Ok(md) }