Rotation thresholds fitted to DR's own lists (P3 23, P4 ~14, P6 16 plays/week: ~3x the old guess); IFPI 2010-2026 walk-forward test bed and route/lead-lag analysis; site and docs updated
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,124 @@
|
||||
//! Rotation thresholds from the real lists. For every parsed rotation list whose week lies in
|
||||
//! the DR panel, compare the artists on the list (per channel; the national P4 list against each
|
||||
//! regional P4) with the artists' plays that week. Reports plays by category and the threshold
|
||||
//! that best separates listed from not-listed artists, and stores the thresholds in
|
||||
//! `rotation_thresholds`, which the models then use instead of the play-distribution guess.
|
||||
|
||||
use super::panel::{key, Panel, C, CHANNELS};
|
||||
use crate::db::Db;
|
||||
use anyhow::Result;
|
||||
use chrono::{Duration, NaiveDate};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fmt::Write as _;
|
||||
|
||||
pub fn create_tables(db: &Db) -> Result<()> {
|
||||
db.conn.execute_batch(
|
||||
"CREATE TABLE IF NOT EXISTS rotation_thresholds (
|
||||
channel TEXT PRIMARY KEY, tau INTEGER NOT NULL, f1 REAL, listed INTEGER, weeks INTEGER, made_at TEXT NOT NULL
|
||||
);",
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Thresholds stored by `run`, if any.
|
||||
pub fn stored(db: &Db) -> Result<Option<[u32; C]>> {
|
||||
create_tables(db)?;
|
||||
let mut tau = [0u32; C];
|
||||
let mut n = 0;
|
||||
for r in db.conn.prepare("SELECT channel, tau FROM rotation_thresholds")?.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?)))? {
|
||||
let (ch, t) = r?;
|
||||
if let Some(c) = CHANNELS.iter().position(|x| *x == ch) { tau[c] = t as u32; n += 1; }
|
||||
}
|
||||
Ok(if n == C { Some(tau) } else { None })
|
||||
}
|
||||
|
||||
/// The artists credited in a list line ("A [+] B", "A feat. B", "A & B", "A, B").
|
||||
fn names(s: &str) -> Vec<String> {
|
||||
let mut parts = vec![s.to_lowercase()];
|
||||
for sep in [" [+] ", " feat. ", " feat ", " ft. ", " ft ", " featuring ", " & ", ", ", " x ", " og ", " med ", " vs. ", " vs "] {
|
||||
parts = parts.iter().flat_map(|p| p.split(sep).map(str::to_string).collect::<Vec<_>>()).collect();
|
||||
}
|
||||
let mut out = vec![key(s)];
|
||||
out.extend(parts.iter().map(|p| key(p)));
|
||||
out.retain(|k| !k.is_empty());
|
||||
out
|
||||
}
|
||||
|
||||
pub fn run(db: &Db, p: &Panel) -> Result<String> {
|
||||
create_tables(db)?;
|
||||
let mut by_key: HashMap<String, Vec<usize>> = HashMap::new();
|
||||
for (i, a) in p.artists.iter().enumerate() { by_key.entry(key(&a.name)).or_default().push(i); }
|
||||
let week_of: HashMap<NaiveDate, usize> = p.weeks.iter().enumerate().map(|(i, d)| (*d, i)).collect();
|
||||
|
||||
// (list channel, week index) -> artist -> best category
|
||||
let mut listed: HashMap<(String, usize), HashMap<usize, String>> = HashMap::new();
|
||||
let mut unmatched = 0usize;
|
||||
let mut total = 0usize;
|
||||
let rank = |c: &str| match c { "A" => 0, "B" => 1, "C" => 2, _ => 3 };
|
||||
for r in db.conn.prepare("SELECT list_date, channel, category, artist FROM rotation_lists")?
|
||||
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, String>(3)?)))? {
|
||||
let (date, ch, cat, artist) = r?;
|
||||
let Ok(d) = NaiveDate::parse_from_str(&date, "%Y-%m-%d") else { continue };
|
||||
// A Friday list applies to the week starting the following Monday.
|
||||
let Some(&w) = week_of.get(&(d + Duration::days(3))) else { continue };
|
||||
total += 1;
|
||||
let ids: HashSet<usize> = names(&artist).iter().filter_map(|k| by_key.get(k)).flatten().copied().collect();
|
||||
if ids.is_empty() { unmatched += 1; continue; }
|
||||
let m = listed.entry((ch, w)).or_default();
|
||||
for a in ids {
|
||||
let e = m.entry(a).or_insert_with(|| cat.clone());
|
||||
if rank(&cat) < rank(e) { *e = cat.clone(); }
|
||||
}
|
||||
}
|
||||
let mut md = String::new();
|
||||
writeln!(md, "# Rotation thresholds from DR's lists\n")?;
|
||||
let weeks: HashSet<usize> = listed.keys().map(|k| k.1).collect();
|
||||
let mut wl: Vec<String> = weeks.iter().map(|w| p.weeks[*w].to_string()).collect();
|
||||
wl.sort();
|
||||
writeln!(md, "Lists inside the DR panel: weeks starting {}. List lines in those weeks: {total}; artist not among the DR artists read: {unmatched}.\n", wl.join(", "))?;
|
||||
writeln!(md, "| Channel | τ (best F1) | F1 | Listed artist-weeks | Median plays A | B | C | Median plays, not listed but played | Old τ |\n|---|---|---|---|---|---|---|---|---|")?;
|
||||
let old = super::common::thresholds_from_plays(p, p.w());
|
||||
let made = chrono::Utc::now().to_rfc3339();
|
||||
let mut out_tau = [0u32; C];
|
||||
for c in 0..C {
|
||||
let list_ch = if CHANNELS[c].starts_with("p4") { "p4" } else { CHANNELS[c] };
|
||||
let mut by_cat: HashMap<String, Vec<f64>> = HashMap::new();
|
||||
let mut not_listed: Vec<f64> = Vec::new();
|
||||
let mut pairs: Vec<(f64, bool)> = Vec::new();
|
||||
let mut n_weeks = 0;
|
||||
for &w in &weeks {
|
||||
let Some(m) = listed.get(&(list_ch.to_string(), w)) else { continue };
|
||||
n_weeks += 1;
|
||||
for a in 0..p.a() {
|
||||
let y = p.y(a, c, w);
|
||||
match m.get(&a) {
|
||||
Some(cat) => { by_cat.entry(cat.clone()).or_default().push(y); pairs.push((y, true)); }
|
||||
None => { if y > 0.0 { not_listed.push(y); } pairs.push((y, false)); }
|
||||
}
|
||||
}
|
||||
}
|
||||
if n_weeks == 0 { out_tau[c] = old[c]; continue; }
|
||||
let med = |v: Option<&Vec<f64>>| v.filter(|v| !v.is_empty()).map(|v| { let mut v = v.clone(); v.sort_by(f64::total_cmp); v[v.len() / 2] }).map(|m| format!("{m:.0}")).unwrap_or("—".into());
|
||||
// Best threshold by F1 of "plays >= tau" against "on the list".
|
||||
let (mut best, mut best_f1) = (old[c], 0.0);
|
||||
for tau in 1..=40u32 {
|
||||
let (mut tp, mut fp, mut fn_) = (0.0, 0.0, 0.0);
|
||||
for (y, l) in &pairs {
|
||||
let pred = *y >= tau as f64;
|
||||
match (pred, *l) { (true, true) => tp += 1.0, (true, false) => fp += 1.0, (false, true) => fn_ += 1.0, _ => {} }
|
||||
}
|
||||
let f1 = if tp > 0.0 { 2.0 * tp / (2.0 * tp + fp + fn_) } else { 0.0 };
|
||||
if f1 > best_f1 { best_f1 = f1; best = tau; }
|
||||
}
|
||||
out_tau[c] = best;
|
||||
let n_listed = pairs.iter().filter(|x| x.1).count();
|
||||
let mut nl = not_listed.clone();
|
||||
nl.sort_by(f64::total_cmp);
|
||||
writeln!(md, "| {} | {best} | {best_f1:.2} | {n_listed} | {} | {} | {} | {} | {} |", CHANNELS[c], med(by_cat.get("A")), med(by_cat.get("B")), med(by_cat.get("C")),
|
||||
med(Some(&nl)), old[c])?;
|
||||
db.conn.execute("INSERT OR REPLACE INTO rotation_thresholds VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
|
||||
rusqlite::params![CHANNELS[c], best as i64, best_f1, n_listed as i64, n_weeks as i64, made])?;
|
||||
}
|
||||
writeln!(md, "\nThe models now read these thresholds (table `rotation_thresholds`). Listed artist-weeks count an artist once per week even with several songs listed; plays are the artist's total that week, including off-list plays.\n")?;
|
||||
Ok(md)
|
||||
}
|
||||
Reference in New Issue
Block a user