Files
Airplay-Predictor/src/model/synth.rs
T

259 lines
18 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! Synthetic rotation lists, for weeks without a published list (`airplay model synth` -> docs/SYNTH.md).
//!
//! 1. Learn, per channel and category, how many plays a day a listed song gets: exact plays
//! (DR's playlists) against DR's own lists. The list for the week of the plays may be missing;
//! then songs with the same category on the lists before and after stand in for it.
//! 2. Apply the thresholds to archived playlists (Wayback Machine, 2012–2021): per channel and
//! ISO week, pool the archived plays, scale to a full day by the hours covered, and give each
//! song a category. Stored in `synthetic_rotation` with the coverage behind it.
//! 3. The national series: Airplay Top-20 positions (2001–) against DR plays in the year the two
//! overlap, so each position reads as an estimated number of DR plays a week.
use super::panel::{key, Panel, C, CHANNELS};
use crate::db::Db;
use anyhow::Result;
use chrono::{Datelike, Duration, NaiveDate, Weekday};
use std::collections::{BTreeMap, HashMap};
use std::fmt::Write as _;
const CATS: [&str; 3] = ["A", "B", "C"];
/// Song key: primary artist + title without version tags.
fn song(artist: &str, title: &str) -> String {
let a = artist.to_lowercase();
let mut cut = a.len();
for sep in [" [+] ", " feat. ", " feat ", " ft. ", " ft ", " featuring ", " & ", ", ", " x ", " og ", " med "] {
if let Some(i) = a.find(sep) { cut = cut.min(i); }
}
let t = title.split(" (").next().unwrap_or(title).split(" - ").next().unwrap_or(title);
format!("{}|{}", key(&a[..cut]), key(t))
}
/// List channel for a play station (the ten regional P4s share the national P4 list).
fn list_channel(station: &str) -> Option<&'static str> {
match station {
"p3" => Some("p3"), "p5" => Some("p5"), "p6beat" => Some("p6beat"), "p8jazz" => Some("p8jazz"),
s if s.starts_with("p4") => Some("p4"),
_ => None,
}
}
pub fn create_tables(db: &Db) -> Result<()> {
db.conn.execute_batch(
"CREATE TABLE IF NOT EXISTS synthetic_rotation (
channel TEXT NOT NULL, iso_year INTEGER NOT NULL, iso_week INTEGER NOT NULL,
song TEXT NOT NULL, artist TEXT, title TEXT,
plays_per_day REAL NOT NULL, category TEXT NOT NULL, -- 'listed' (on the list or not; formats changed over the years)
days INTEGER NOT NULL, hours_covered INTEGER NOT NULL, source TEXT NOT NULL,
PRIMARY KEY (channel, iso_year, iso_week, song)
);
CREATE TABLE IF NOT EXISTS synthetic_thresholds (
channel TEXT NOT NULL, category TEXT NOT NULL, plays_per_day REAL NOT NULL, songs INTEGER, made_at TEXT NOT NULL,
PRIMARY KEY (channel, category)
);",
)?;
Ok(())
}
pub fn run(db: &Db, p: &Panel) -> Result<String> {
create_tables(db)?;
let mut md = String::new();
writeln!(md, "# Synthetic rotation lists\n")?;
// ---- 1. thresholds from exact plays vs lists
// Lists by week (Monday) and channel: song -> category.
let mut lists: BTreeMap<NaiveDate, HashMap<String, HashMap<String, String>>> = BTreeMap::new();
for r in db.conn.prepare("SELECT list_date, channel, category, artist, title FROM rotation_lists WHERE category IN ('A', 'B', 'C')")?
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, String>(3)?, r.get::<_, String>(4)?)))? {
let (d, ch, cat, a, t) = r?;
let Ok(d) = NaiveDate::parse_from_str(&d, "%Y-%m-%d") else { continue };
lists.entry(d + Duration::days(3)).or_default().entry(ch).or_default().insert(song(&a, &t), cat);
}
// Exact plays per week (Monday, Copenhagen time is close enough at week level), station, song.
let mut plays: BTreeMap<NaiveDate, HashMap<String, HashMap<String, f64>>> = BTreeMap::new();
let mut days_seen: BTreeMap<NaiveDate, std::collections::BTreeSet<String>> = BTreeMap::new();
for r in db.conn.prepare("SELECT station, played_at, artist, title FROM plays WHERE source = 'dr'")?
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, String>(3)?)))? {
let (st, at, a, t) = r?;
let Ok(d) = NaiveDate::parse_from_str(&at[..10], "%Y-%m-%d") else { continue };
let monday = d - Duration::days(d.weekday().num_days_from_monday() as i64);
*plays.entry(monday).or_default().entry(st).or_default().entry(song(&a, &t)).or_insert(0.0) += 1.0;
days_seen.entry(monday).or_default().insert(at[..10].to_string());
}
// The list in force in a week: the published one, or songs with the same category before and after.
let list_for = |week: NaiveDate, ch: &str| -> Option<(HashMap<String, String>, &'static str)> {
if let Some(l) = lists.get(&week).and_then(|m| m.get(ch)) { return Some((l.clone(), "published")); }
let before = lists.range(..week).rev().find_map(|(d, m)| m.get(ch).map(|l| (*d, l)))?;
let after = lists.range(week + Duration::days(1)..).find_map(|(d, m)| m.get(ch).map(|l| (*d, l)))?;
if (after.0 - before.0).num_days() > 21 { return None; }
Some((before.1.iter().filter(|(s, c)| after.1.get(*s) == Some(*c)).map(|(s, c)| (s.clone(), c.clone())).collect(), "bridged"))
};
let mut per_cat: HashMap<(String, String), Vec<f64>> = HashMap::new(); // (list channel, cat) -> plays/day
let mut unlisted: HashMap<String, Vec<f64>> = HashMap::new();
let mut used = Vec::new();
for (week, by_station) in &plays {
let days = days_seen.get(week).map(|s| s.len()).unwrap_or(0) as f64;
if days < 5.0 { continue; } // a week mostly covered
for (station, songs) in by_station {
let Some(lc) = list_channel(station) else { continue };
let Some((list, how)) = list_for(*week, lc) else { continue };
used.push(format!("{week} {station} ({how})"));
for (s, n) in songs {
let ppd = n / days;
match list.get(s) { Some(cat) => per_cat.entry((lc.to_string(), cat.clone())).or_default().push(ppd), None => unlisted.entry(lc.to_string()).or_default().push(ppd) }
}
}
}
writeln!(md, "## 1. Plays per day by category (exact plays vs DR's lists)\n")?;
writeln!(md, "Weeks used: {}.\n", if used.is_empty() { "none yet".to_string() } else { used.iter().take(6).cloned().collect::<Vec<_>>().join(", ") + if used.len() > 6 { " …" } else { "" } })?;
writeln!(md, "Synthetic lists only say \"on the list or not\": DR's list formats have changed over the years, and A/B can't be told apart from plays on every channel. The category medians are for reference.\n")?;
writeln!(md, "| Channel | A median | B median | C median | Not listed, 90th pct | On-list threshold (plays/day) |\n|---|---|---|---|---|---|")?;
let med = |v: &mut Vec<f64>| { if v.is_empty() { return f64::NAN; } v.sort_by(f64::total_cmp); v[v.len() / 2] };
let pct = |v: &mut Vec<f64>, q: f64| { if v.is_empty() { return f64::NAN; } v.sort_by(f64::total_cmp); v[((v.len() - 1) as f64 * q) as usize] };
let mut thresholds: HashMap<String, [f64; 3]> = HashMap::new();
let made = chrono::Utc::now().to_rfc3339();
for lc in ["p3", "p4", "p5", "p6beat", "p8jazz"] {
let mut m = [0.0; 3];
for (k, c) in CATS.iter().enumerate() { m[k] = med(per_cat.entry((lc.into(), c.to_string())).or_default()); }
let nl = pct(unlisted.entry(lc.into()).or_default(), 0.9);
// Category thresholds: halfway (geometric) between neighbouring medians; C's floor halfway to the unlisted 90th pct.
let gm = |a: f64, b: f64| (a * b).sqrt();
// The list floor (C): the plays/day that best separates listed from unlisted songs (F1).
let listed: Vec<f64> = CATS.iter().flat_map(|c| per_cat.get(&(lc.to_string(), c.to_string())).cloned().unwrap_or_default()).collect();
let others = unlisted.get(lc).cloned().unwrap_or_default();
let mut floor = (m[2] / 2.0, 0.0);
for k in 2..=40 {
let t = k as f64 / 10.0;
let tp = listed.iter().filter(|x| **x >= t).count() as f64;
let fp = others.iter().filter(|x| **x >= t).count() as f64;
let f1 = 2.0 * tp / (2.0 * tp + fp + (listed.len() as f64 - tp));
if f1 > floor.1 { floor = (t, f1); }
}
let th = [gm(m[0], m[1]), gm(m[1], m[2]).max(floor.0), floor.0];
if m.iter().all(|x| x.is_finite()) {
thresholds.insert(lc.to_string(), th);
for (k, c) in CATS.iter().enumerate() {
db.conn.execute("INSERT OR REPLACE INTO synthetic_thresholds VALUES (?1, ?2, ?3, ?4, ?5)",
rusqlite::params![lc, c, th[k], per_cat.get(&(lc.to_string(), c.to_string())).map(|v| v.len() as i64), made])?;
}
}
writeln!(md, "| {lc} | {:.1} | {:.1} | {:.1} | {:.1} | {} |", m[0], m[1], m[2], nl,
thresholds.get(lc).map(|t| format!("{:.1}", t[2])).unwrap_or("—".into()))?;
}
// In-sample check: how well the thresholds reproduce the list in the weeks used.
let mut hit = [[0usize; 4]; 4];
let cat_of = |th: &[f64; 3], ppd: f64| if ppd >= th[0] { 0 } else if ppd >= th[1] { 1 } else if ppd >= th[2] { 2 } else { 3 };
for ((lc, c), v) in &per_cat { if let Some(th) = thresholds.get(lc) { let t = CATS.iter().position(|x| x == c).unwrap(); for ppd in v { hit[t][cat_of(th, *ppd)] += 1; } } }
for (lc, v) in &unlisted { if let Some(th) = thresholds.get(lc) { for ppd in v { hit[3][cat_of(th, *ppd)] += 1; } } }
let listed_total: usize = (0..3).map(|t| hit[t].iter().sum::<usize>()).sum();
let listed_found: usize = (0..3).map(|t| hit[t][..3].iter().sum::<usize>()).sum();
let false_pos: usize = hit[3][..3].iter().sum();
writeln!(md, "\nIn-sample check (song-weeks): listed songs found {listed_found} of {listed_total}; songs wrongly put on the list {false_pos}. Thresholds are refitted as exact plays and lists for the same weeks accumulate.\n")?;
// Typical list size per channel (DR's real lists: A/B/C or one list), the era-robust rule:
// on the list = among the channel's N most played songs that week.
let mut list_size: HashMap<String, usize> = HashMap::new();
for lc in ["p3", "p4", "p5", "p6beat", "p8jazz"] {
let mut v: Vec<i64> = db.conn.prepare("SELECT count(*) FROM rotation_lists WHERE channel = ?1 AND category IN ('A', 'B', 'C', 'LIST') GROUP BY list_date")?
.query_map([lc], |r| r.get(0))?.collect::<Result<_, _>>()?;
v.sort();
if !v.is_empty() { list_size.insert(lc.to_string(), v[v.len() / 2] as usize); }
}
// Check the rule where the real list is known: top N by plays vs the list.
let (mut tp, mut fp, mut fnn) = (0usize, 0usize, 0usize);
for (week, by_station) in &plays {
let days = days_seen.get(week).map(|s| s.len()).unwrap_or(0);
if days < 5 { continue; }
for (station, songs) in by_station {
let Some(lc) = list_channel(station) else { continue };
let (Some((list, _)), Some(&n)) = (list_for(*week, lc), list_size.get(lc)) else { continue };
let mut ranked: Vec<(&String, &f64)> = songs.iter().collect();
ranked.sort_by(|a, b| b.1.total_cmp(a.1));
let top: std::collections::HashSet<&String> = ranked.iter().take(n).map(|x| x.0).collect();
for s in &top { if list.contains_key(*s) { tp += 1 } else { fp += 1 } }
fnn += list.keys().filter(|s| !top.contains(s)).count();
}
}
writeln!(md, "Era-robust rule used for the archive: on the list = among the channel's N most played songs that week, N = the median size of DR's lists ({}). Where the real list is known it finds {tp} of {} listed songs, with {fp} songs wrongly on (precision {:.0}%, recall {:.0}%).\n",
list_size.iter().map(|(k, v)| format!("{k} {v}")).collect::<Vec<_>>().join(", "), tp + fnn, 100.0 * tp as f64 / (tp + fp).max(1) as f64, 100.0 * tp as f64 / (tp + fnn).max(1) as f64)?;
// ---- 2. apply to the archive
let has_archive: i64 = db.conn.query_row("SELECT count(*) FROM sqlite_master WHERE name = 'archive_plays'", [], |r| r.get(0))?;
writeln!(md, "## 2. Archived playlists → synthetic lists\n")?;
if has_archive == 0 || thresholds.is_empty() {
writeln!(md, "Nothing to apply yet (archive or thresholds missing).\n")?;
} else {
// per (station, iso week): hours covered per day, plays per song
struct Wk { hours: HashMap<String, std::collections::BTreeSet<String>>, songs: HashMap<String, (f64, String, String)> }
let mut weeks: BTreeMap<(String, i32, u32), Wk> = BTreeMap::new();
for r in db.conn.prepare("SELECT station, played_at, artist, title FROM archive_plays")?
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, String>(3)?)))? {
let (st, at, a, t) = r?;
let Ok(d) = NaiveDate::parse_from_str(&at[..10], "%Y-%m-%d") else { continue };
// 4-week blocks (from Monday 3 Jan 2000): a few sampled days a week are too noisy for a
// weekly list (two 36-hour samples of one week's P3 differ by ~9 of the top 32 songs),
// pooled over 4 weeks the list holds; songs stay on the list ~7+ weeks anyway.
let epoch = NaiveDate::from_ymd_opt(2000, 1, 3).unwrap();
let block = epoch + Duration::weeks(((d - epoch).num_days() / 7 / 4) * 4);
let w = weeks.entry((st, block.iso_week().year(), block.iso_week().week())).or_insert_with(|| Wk { hours: HashMap::new(), songs: HashMap::new() });
w.hours.entry(at[..10].to_string()).or_default().insert(at[11..13].to_string());
let e = w.songs.entry(song(&a, &t)).or_insert((0.0, a.clone(), t.clone()));
e.0 += 1.0;
}
let tx = db.conn.unchecked_transaction()?;
tx.execute("DELETE FROM synthetic_rotation WHERE source = 'wayback'", [])?;
let mut summary: BTreeMap<(String, i32), (usize, usize)> = BTreeMap::new(); // (station, year) -> (weeks, songs listed)
for ((st, y, w), wk) in &weeks {
let Some(lc) = list_channel(st) else { continue };
if !thresholds.contains_key(lc) { continue; }
// Only daytime-complete days count fully; scale plays by covered hours / 24.
let hours: usize = wk.hours.values().map(|h| h.len()).sum();
let days = wk.hours.values().filter(|h| h.len() >= 12).count();
if hours < 96 { continue; } // under 4 days' worth in the block: too thin (two 72-hour samples of one week still differ by ~5 of 32)
let day_equiv = hours as f64 / 24.0;
let Some(&size) = list_size.get(lc) else { continue };
let mut ranked: Vec<(&String, &(f64, String, String))> = wk.songs.iter().filter(|(_, v)| v.0 >= 3.0).collect();
ranked.sort_by(|a, b| b.1 .0.total_cmp(&a.1 .0));
let mut n = 0;
for (s, (cnt, a, t)) in ranked.into_iter().take(size) {
let ppd = cnt / day_equiv;
tx.execute("INSERT OR REPLACE INTO synthetic_rotation VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, 'wayback')",
rusqlite::params![lc, y, w, s, a, t, ppd, "listed", days as i64, hours as i64])?;
n += 1;
}
let e = summary.entry((st.clone(), *y)).or_insert((0, 0));
e.0 += 1; e.1 += n;
}
tx.commit()?;
writeln!(md, "| Station | Year | 4-week blocks with a synthetic list | Avg songs listed |\n|---|---|---|---|")?;
for ((st, y), (wks, n)) in &summary { writeln!(md, "| {st} | {y} | {wks} | {:.0} |", *n as f64 / *wks as f64)?; }
writeln!(md, "\nLists are per 4-week block (iso_year/iso_week = its first week): a block needs at least 96 archived hours; a song needs 3+ plays and is then ranked against the block's other songs (top N = on the list in that block). Weekly lists from a few sampled days are mostly sampling noise.\n")?;
}
// ---- 3. IFPI Airplay Top-20 position vs DR plays
writeln!(md, "## 3. Airplay Top-20 position → DR plays a week\n")?;
let id_by_name: HashMap<String, Vec<usize>> = p.artists.iter().enumerate().fold(HashMap::new(), |mut m, (i, a)| { m.entry(key(&a.name)).or_default().push(i); m });
let week_of: HashMap<NaiveDate, usize> = p.weeks.iter().enumerate().map(|(i, d)| (*d, i)).collect();
let mut by_pos: BTreeMap<i64, Vec<f64>> = BTreeMap::new();
for r in db.conn.prepare("SELECT year, week, position, artist FROM hitlisten_charts WHERE chart = 'airplay'")?
.query_map([], |r| Ok((r.get::<_, i32>(0)?, r.get::<_, u32>(1)?, r.get::<_, i64>(2)?, r.get::<_, String>(3)?)))? {
let (y, w, pos, a) = r?;
let Some(monday) = NaiveDate::from_isoywd_opt(y, w, Weekday::Mon) else { continue };
let Some(&wi) = week_of.get(&monday) else { continue };
let primary = song(&a, "x");
let primary = primary.split('|').next().unwrap_or("");
let Some(ids) = id_by_name.get(primary) else { continue };
let plays: f64 = ids.iter().map(|&i| (0..C).map(|c| p.y(i, c, wi)).sum::<f64>()).fold(0.0, f64::max);
by_pos.entry((pos - 1) / 5).or_default().push(plays);
}
writeln!(md, "Artist's DR plays (all {} channels) in the chart week, for Airplay Top-20 entries {}–{}:\n", C, p.weeks[0], p.weeks[p.w() - 1])?;
writeln!(md, "| Positions | Entries | Median DR plays | 25th–75th pct |\n|---|---|---|---|")?;
for (b, mut v) in by_pos {
let n = v.len();
writeln!(md, "| {}–{} | {n} | {:.0} | {:.0}–{:.0} |", b * 5 + 1, b * 5 + 5, med(&mut v.clone()), pct(&mut v, 0.25), pct(&mut v.clone(), 0.75))?;
}
writeln!(md, "\nSo the 25-year Airplay Top-20 can stand in for \"in heavy rotation nationally\" (on the order of the plays above), but it only covers the top 20 songs each week; it can't produce B/C lists or anything below.\n")?;
let _ = CHANNELS;
Ok(md)
}