//! Synthetic rotation lists, for weeks without a published list (`airplay model synth` -> docs/SYNTH.md). //! //! 1. Learn, per channel and category, how many plays a day a listed song gets: exact plays //! (DR's playlists) against DR's own lists. The list for the week of the plays may be missing; //! then songs with the same category on the lists before and after stand in for it. //! 2. Apply the thresholds to archived playlists (Wayback Machine, 2012–2021): per channel and //! ISO week, pool the archived plays, scale to a full day by the hours covered, and give each //! song a category. Stored in `synthetic_rotation` with the coverage behind it. //! 3. The national series: Airplay Top-20 positions (2001–) against DR plays in the year the two //! overlap, so each position reads as an estimated number of DR plays a week. use super::panel::{key, Panel, C, CHANNELS}; use crate::db::Db; use anyhow::Result; use chrono::{Datelike, Duration, NaiveDate, Weekday}; use std::collections::{BTreeMap, HashMap}; use std::fmt::Write as _; const CATS: [&str; 3] = ["A", "B", "C"]; /// Song key: primary artist + title without version tags. fn song(artist: &str, title: &str) -> String { let a = artist.to_lowercase(); let mut cut = a.len(); for sep in [" [+] ", " feat. ", " feat ", " ft. ", " ft ", " featuring ", " & ", ", ", " x ", " og ", " med "] { if let Some(i) = a.find(sep) { cut = cut.min(i); } } let t = title.split(" (").next().unwrap_or(title).split(" - ").next().unwrap_or(title); format!("{}|{}", key(&a[..cut]), key(t)) } /// List channel for a play station (the ten regional P4s share the national P4 list). fn list_channel(station: &str) -> Option<&'static str> { match station { "p3" => Some("p3"), "p5" => Some("p5"), "p6beat" => Some("p6beat"), "p8jazz" => Some("p8jazz"), s if s.starts_with("p4") => Some("p4"), _ => None, } } pub fn create_tables(db: &Db) -> Result<()> { db.conn.execute_batch( "CREATE TABLE IF NOT EXISTS synthetic_rotation ( channel TEXT NOT NULL, iso_year INTEGER NOT NULL, iso_week INTEGER NOT NULL, song TEXT NOT NULL, artist TEXT, title TEXT, plays_per_day REAL NOT NULL, category TEXT NOT NULL, -- 'listed' (on the list or not; formats changed over the years) days INTEGER NOT NULL, hours_covered INTEGER NOT NULL, source TEXT NOT NULL, PRIMARY KEY (channel, iso_year, iso_week, song) ); CREATE TABLE IF NOT EXISTS synthetic_thresholds ( channel TEXT NOT NULL, category TEXT NOT NULL, plays_per_day REAL NOT NULL, songs INTEGER, made_at TEXT NOT NULL, PRIMARY KEY (channel, category) );", )?; Ok(()) } pub fn run(db: &Db, p: &Panel) -> Result { create_tables(db)?; let mut md = String::new(); writeln!(md, "# Synthetic rotation lists\n")?; // ---- 1. thresholds from exact plays vs lists // Lists by week (Monday) and channel: song -> category. let mut lists: BTreeMap>> = BTreeMap::new(); for r in db.conn.prepare("SELECT list_date, channel, category, artist, title FROM rotation_lists WHERE category IN ('A', 'B', 'C')")? .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, String>(3)?, r.get::<_, String>(4)?)))? { let (d, ch, cat, a, t) = r?; let Ok(d) = NaiveDate::parse_from_str(&d, "%Y-%m-%d") else { continue }; lists.entry(d + Duration::days(3)).or_default().entry(ch).or_default().insert(song(&a, &t), cat); } // Exact plays per week (Monday, Copenhagen time is close enough at week level), station, song. let mut plays: BTreeMap>> = BTreeMap::new(); let mut days_seen: BTreeMap> = BTreeMap::new(); for r in db.conn.prepare("SELECT station, played_at, artist, title FROM plays WHERE source = 'dr'")? .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, String>(3)?)))? { let (st, at, a, t) = r?; let Ok(d) = NaiveDate::parse_from_str(&at[..10], "%Y-%m-%d") else { continue }; let monday = d - Duration::days(d.weekday().num_days_from_monday() as i64); *plays.entry(monday).or_default().entry(st).or_default().entry(song(&a, &t)).or_insert(0.0) += 1.0; days_seen.entry(monday).or_default().insert(at[..10].to_string()); } // The list in force in a week: the published one, or songs with the same category before and after. let list_for = |week: NaiveDate, ch: &str| -> Option<(HashMap, &'static str)> { if let Some(l) = lists.get(&week).and_then(|m| m.get(ch)) { return Some((l.clone(), "published")); } let before = lists.range(..week).rev().find_map(|(d, m)| m.get(ch).map(|l| (*d, l)))?; let after = lists.range(week + Duration::days(1)..).find_map(|(d, m)| m.get(ch).map(|l| (*d, l)))?; if (after.0 - before.0).num_days() > 21 { return None; } Some((before.1.iter().filter(|(s, c)| after.1.get(*s) == Some(*c)).map(|(s, c)| (s.clone(), c.clone())).collect(), "bridged")) }; let mut per_cat: HashMap<(String, String), Vec> = HashMap::new(); // (list channel, cat) -> plays/day let mut unlisted: HashMap> = HashMap::new(); let mut used = Vec::new(); for (week, by_station) in &plays { let days = days_seen.get(week).map(|s| s.len()).unwrap_or(0) as f64; if days < 5.0 { continue; } // a week mostly covered for (station, songs) in by_station { let Some(lc) = list_channel(station) else { continue }; let Some((list, how)) = list_for(*week, lc) else { continue }; used.push(format!("{week} {station} ({how})")); for (s, n) in songs { let ppd = n / days; match list.get(s) { Some(cat) => per_cat.entry((lc.to_string(), cat.clone())).or_default().push(ppd), None => unlisted.entry(lc.to_string()).or_default().push(ppd) } } } } writeln!(md, "## 1. Plays per day by category (exact plays vs DR's lists)\n")?; writeln!(md, "Weeks used: {}.\n", if used.is_empty() { "none yet".to_string() } else { used.iter().take(6).cloned().collect::>().join(", ") + if used.len() > 6 { " …" } else { "" } })?; writeln!(md, "Synthetic lists only say \"on the list or not\": DR's list formats have changed over the years, and A/B can't be told apart from plays on every channel. The category medians are for reference.\n")?; writeln!(md, "| Channel | A median | B median | C median | Not listed, 90th pct | On-list threshold (plays/day) |\n|---|---|---|---|---|---|")?; let med = |v: &mut Vec| { if v.is_empty() { return f64::NAN; } v.sort_by(f64::total_cmp); v[v.len() / 2] }; let pct = |v: &mut Vec, q: f64| { if v.is_empty() { return f64::NAN; } v.sort_by(f64::total_cmp); v[((v.len() - 1) as f64 * q) as usize] }; let mut thresholds: HashMap = HashMap::new(); let made = chrono::Utc::now().to_rfc3339(); for lc in ["p3", "p4", "p5", "p6beat", "p8jazz"] { let mut m = [0.0; 3]; for (k, c) in CATS.iter().enumerate() { m[k] = med(per_cat.entry((lc.into(), c.to_string())).or_default()); } let nl = pct(unlisted.entry(lc.into()).or_default(), 0.9); // Category thresholds: halfway (geometric) between neighbouring medians; C's floor halfway to the unlisted 90th pct. let gm = |a: f64, b: f64| (a * b).sqrt(); // The list floor (C): the plays/day that best separates listed from unlisted songs (F1). let listed: Vec = CATS.iter().flat_map(|c| per_cat.get(&(lc.to_string(), c.to_string())).cloned().unwrap_or_default()).collect(); let others = unlisted.get(lc).cloned().unwrap_or_default(); let mut floor = (m[2] / 2.0, 0.0); for k in 2..=40 { let t = k as f64 / 10.0; let tp = listed.iter().filter(|x| **x >= t).count() as f64; let fp = others.iter().filter(|x| **x >= t).count() as f64; let f1 = 2.0 * tp / (2.0 * tp + fp + (listed.len() as f64 - tp)); if f1 > floor.1 { floor = (t, f1); } } let th = [gm(m[0], m[1]), gm(m[1], m[2]).max(floor.0), floor.0]; if m.iter().all(|x| x.is_finite()) { thresholds.insert(lc.to_string(), th); for (k, c) in CATS.iter().enumerate() { db.conn.execute("INSERT OR REPLACE INTO synthetic_thresholds VALUES (?1, ?2, ?3, ?4, ?5)", rusqlite::params![lc, c, th[k], per_cat.get(&(lc.to_string(), c.to_string())).map(|v| v.len() as i64), made])?; } } writeln!(md, "| {lc} | {:.1} | {:.1} | {:.1} | {:.1} | {} |", m[0], m[1], m[2], nl, thresholds.get(lc).map(|t| format!("{:.1}", t[2])).unwrap_or("—".into()))?; } // In-sample check: how well the thresholds reproduce the list in the weeks used. let mut hit = [[0usize; 4]; 4]; let cat_of = |th: &[f64; 3], ppd: f64| if ppd >= th[0] { 0 } else if ppd >= th[1] { 1 } else if ppd >= th[2] { 2 } else { 3 }; for ((lc, c), v) in &per_cat { if let Some(th) = thresholds.get(lc) { let t = CATS.iter().position(|x| x == c).unwrap(); for ppd in v { hit[t][cat_of(th, *ppd)] += 1; } } } for (lc, v) in &unlisted { if let Some(th) = thresholds.get(lc) { for ppd in v { hit[3][cat_of(th, *ppd)] += 1; } } } let listed_total: usize = (0..3).map(|t| hit[t].iter().sum::()).sum(); let listed_found: usize = (0..3).map(|t| hit[t][..3].iter().sum::()).sum(); let false_pos: usize = hit[3][..3].iter().sum(); writeln!(md, "\nIn-sample check (song-weeks): listed songs found {listed_found} of {listed_total}; songs wrongly put on the list {false_pos}. Thresholds are refitted as exact plays and lists for the same weeks accumulate.\n")?; // Typical list size per channel (DR's real lists: A/B/C or one list), the era-robust rule: // on the list = among the channel's N most played songs that week. let mut list_size: HashMap = HashMap::new(); for lc in ["p3", "p4", "p5", "p6beat", "p8jazz"] { let mut v: Vec = db.conn.prepare("SELECT count(*) FROM rotation_lists WHERE channel = ?1 AND category IN ('A', 'B', 'C', 'LIST') GROUP BY list_date")? .query_map([lc], |r| r.get(0))?.collect::>()?; v.sort(); if !v.is_empty() { list_size.insert(lc.to_string(), v[v.len() / 2] as usize); } } // Check the rule where the real list is known: top N by plays vs the list. let (mut tp, mut fp, mut fnn) = (0usize, 0usize, 0usize); for (week, by_station) in &plays { let days = days_seen.get(week).map(|s| s.len()).unwrap_or(0); if days < 5 { continue; } for (station, songs) in by_station { let Some(lc) = list_channel(station) else { continue }; let (Some((list, _)), Some(&n)) = (list_for(*week, lc), list_size.get(lc)) else { continue }; let mut ranked: Vec<(&String, &f64)> = songs.iter().collect(); ranked.sort_by(|a, b| b.1.total_cmp(a.1)); let top: std::collections::HashSet<&String> = ranked.iter().take(n).map(|x| x.0).collect(); for s in &top { if list.contains_key(*s) { tp += 1 } else { fp += 1 } } fnn += list.keys().filter(|s| !top.contains(s)).count(); } } writeln!(md, "Era-robust rule used for the archive: on the list = among the channel's N most played songs that week, N = the median size of DR's lists ({}). Where the real list is known it finds {tp} of {} listed songs, with {fp} songs wrongly on (precision {:.0}%, recall {:.0}%).\n", list_size.iter().map(|(k, v)| format!("{k} {v}")).collect::>().join(", "), tp + fnn, 100.0 * tp as f64 / (tp + fp).max(1) as f64, 100.0 * tp as f64 / (tp + fnn).max(1) as f64)?; // ---- 2. apply to the archive let has_archive: i64 = db.conn.query_row("SELECT count(*) FROM sqlite_master WHERE name = 'archive_plays'", [], |r| r.get(0))?; writeln!(md, "## 2. Archived playlists → synthetic lists\n")?; if has_archive == 0 || thresholds.is_empty() { writeln!(md, "Nothing to apply yet (archive or thresholds missing).\n")?; } else { // per (station, iso week): hours covered per day, plays per song struct Wk { hours: HashMap>, songs: HashMap } let mut weeks: BTreeMap<(String, i32, u32), Wk> = BTreeMap::new(); for r in db.conn.prepare("SELECT station, played_at, artist, title FROM archive_plays")? .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, String>(3)?)))? { let (st, at, a, t) = r?; let Ok(d) = NaiveDate::parse_from_str(&at[..10], "%Y-%m-%d") else { continue }; // 4-week blocks (from Monday 3 Jan 2000): a few sampled days a week are too noisy for a // weekly list (two 36-hour samples of one week's P3 differ by ~9 of the top 32 songs), // pooled over 4 weeks the list holds; songs stay on the list ~7+ weeks anyway. let epoch = NaiveDate::from_ymd_opt(2000, 1, 3).unwrap(); let block = epoch + Duration::weeks(((d - epoch).num_days() / 7 / 4) * 4); let w = weeks.entry((st, block.iso_week().year(), block.iso_week().week())).or_insert_with(|| Wk { hours: HashMap::new(), songs: HashMap::new() }); w.hours.entry(at[..10].to_string()).or_default().insert(at[11..13].to_string()); let e = w.songs.entry(song(&a, &t)).or_insert((0.0, a.clone(), t.clone())); e.0 += 1.0; } let tx = db.conn.unchecked_transaction()?; tx.execute("DELETE FROM synthetic_rotation WHERE source = 'wayback'", [])?; let mut summary: BTreeMap<(String, i32), (usize, usize)> = BTreeMap::new(); // (station, year) -> (weeks, songs listed) for ((st, y, w), wk) in &weeks { let Some(lc) = list_channel(st) else { continue }; if !thresholds.contains_key(lc) { continue; } // Only daytime-complete days count fully; scale plays by covered hours / 24. let hours: usize = wk.hours.values().map(|h| h.len()).sum(); let days = wk.hours.values().filter(|h| h.len() >= 12).count(); if hours < 96 { continue; } // under 4 days' worth in the block: too thin (two 72-hour samples of one week still differ by ~5 of 32) let day_equiv = hours as f64 / 24.0; let Some(&size) = list_size.get(lc) else { continue }; let mut ranked: Vec<(&String, &(f64, String, String))> = wk.songs.iter().filter(|(_, v)| v.0 >= 3.0).collect(); ranked.sort_by(|a, b| b.1 .0.total_cmp(&a.1 .0)); let mut n = 0; for (s, (cnt, a, t)) in ranked.into_iter().take(size) { let ppd = cnt / day_equiv; tx.execute("INSERT OR REPLACE INTO synthetic_rotation VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, 'wayback')", rusqlite::params![lc, y, w, s, a, t, ppd, "listed", days as i64, hours as i64])?; n += 1; } let e = summary.entry((st.clone(), *y)).or_insert((0, 0)); e.0 += 1; e.1 += n; } tx.commit()?; writeln!(md, "| Station | Year | 4-week blocks with a synthetic list | Avg songs listed |\n|---|---|---|---|")?; for ((st, y), (wks, n)) in &summary { writeln!(md, "| {st} | {y} | {wks} | {:.0} |", *n as f64 / *wks as f64)?; } writeln!(md, "\nLists are per 4-week block (iso_year/iso_week = its first week): a block needs at least 96 archived hours; a song needs 3+ plays and is then ranked against the block's other songs (top N = on the list in that block). Weekly lists from a few sampled days are mostly sampling noise.\n")?; } // ---- 3. IFPI Airplay Top-20 position vs DR plays writeln!(md, "## 3. Airplay Top-20 position → DR plays a week\n")?; let id_by_name: HashMap> = p.artists.iter().enumerate().fold(HashMap::new(), |mut m, (i, a)| { m.entry(key(&a.name)).or_default().push(i); m }); let week_of: HashMap = p.weeks.iter().enumerate().map(|(i, d)| (*d, i)).collect(); let mut by_pos: BTreeMap> = BTreeMap::new(); for r in db.conn.prepare("SELECT year, week, position, artist FROM hitlisten_charts WHERE chart = 'airplay'")? .query_map([], |r| Ok((r.get::<_, i32>(0)?, r.get::<_, u32>(1)?, r.get::<_, i64>(2)?, r.get::<_, String>(3)?)))? { let (y, w, pos, a) = r?; let Some(monday) = NaiveDate::from_isoywd_opt(y, w, Weekday::Mon) else { continue }; let Some(&wi) = week_of.get(&monday) else { continue }; let primary = song(&a, "x"); let primary = primary.split('|').next().unwrap_or(""); let Some(ids) = id_by_name.get(primary) else { continue }; let plays: f64 = ids.iter().map(|&i| (0..C).map(|c| p.y(i, c, wi)).sum::()).fold(0.0, f64::max); by_pos.entry((pos - 1) / 5).or_default().push(plays); } writeln!(md, "Artist's DR plays (all {} channels) in the chart week, for Airplay Top-20 entries {}–{}:\n", C, p.weeks[0], p.weeks[p.w() - 1])?; writeln!(md, "| Positions | Entries | Median DR plays | 25th–75th pct |\n|---|---|---|---|")?; for (b, mut v) in by_pos { let n = v.len(); writeln!(md, "| {}–{} | {n} | {:.0} | {:.0}–{:.0} |", b * 5 + 1, b * 5 + 5, med(&mut v.clone()), pct(&mut v, 0.25), pct(&mut v.clone(), 0.75))?; } writeln!(md, "\nSo the 25-year Airplay Top-20 can stand in for \"in heavy rotation nationally\" (on the order of the plays above), but it only covers the top 20 songs each week; it can't produce B/C lists or anything below.\n")?; let _ = CHANNELS; Ok(md) }