diff --git a/src/archive.rs b/src/archive.rs new file mode 100644 index 0000000..64dec81 --- /dev/null +++ b/src/archive.rs @@ -0,0 +1,150 @@ +//! DR playlists from the Wayback Machine: archived copies of dr.dk/playlister/{channel}/{date}/… +//! (2012–2021, mostly P3 2013–2017). The old pages list each play with DR's track id, the exact +//! UTC time, title, artists and DR artist id, the same ids as today, so archived plays join the +//! artist pool directly. Kept apart from the live `plays` table (sampled days, not continuous). +//! +//! airplay collect --only archive + +use crate::{db::Db, http::Http}; +use anyhow::Result; +use chrono::Utc; +use regex::Regex; +use rusqlite::params; +use std::collections::BTreeMap; + +const CHANNELS: &[&str] = &["p3", "p4kbh", "p4", "p5", "p6beat", "p7mix", "p8jazz", "p4fyn", "p4aarhus", "p4nord", "p4vest", "p4syd", + "p4sjaelland", "p4trekanten", "p4bornholm", "p4esbjerg", "p4ostjylland", "p4midtvest"]; + +pub fn create_tables(db: &Db) -> Result<()> { + db.conn.execute_batch( + "CREATE TABLE IF NOT EXISTS archive_plays ( + station TEXT NOT NULL, played_at TEXT NOT NULL, track_id TEXT NOT NULL, + artist TEXT NOT NULL, title TEXT NOT NULL, artist_ids TEXT NOT NULL DEFAULT '', + page TEXT NOT NULL, + recorded TEXT, country TEXT, released TEXT, -- 2012-13 pages: 'Indspillet', country, 'UDGIVET: year, label' + PRIMARY KEY (station, played_at, track_id) + ); + CREATE INDEX IF NOT EXISTS archive_plays_day ON archive_plays (station, substr(played_at, 1, 10)); + CREATE TABLE IF NOT EXISTS archive_pages ( + url TEXT PRIMARY KEY, snapshot TEXT NOT NULL, plays INTEGER, fetched_at TEXT NOT NULL + );", + )?; + Ok(()) +} + +fn unescape(s: &str) -> String { + let s = s.replace("&", "&").replace(""", "\"").replace("'", "'").replace("<", "<").replace(">", ">"); + // numeric entities (ø …) + let re = Regex::new(r"&#(\d+);").unwrap(); + re.replace_all(&s, |c: ®ex::Captures| char::from_u32(c[1].parse().unwrap_or(63)).map(String::from).unwrap_or_default()).into_owned() +} + +pub struct Play { pub at: String, pub track_id: String, pub title: String, pub artists: String, pub artist_ids: String, + pub recorded: Option, pub country: Option, pub released: Option } + +/// The plays on one archived page. Two layouts: from about 2014 each item carries its UTC time +/// (data-fulltime) and track id; the 2012–13 pages show the local clock time (the date comes from +/// the URL, rolling over at midnight) and add recording year, country and release. +pub fn parse(html: &str, page_date: chrono::NaiveDate) -> Vec { + let item = Regex::new(r#"(?s)]*itemtype="http://schema.org/MusicRecording"[^>]*>(.*?)"#).unwrap(); + let head = Regex::new(r#"(?s)]*)itemtype"#).unwrap(); + let full = Regex::new(r#"data-fulltime="(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2})Z""#).unwrap(); + let clock = Regex::new(r#"]*>(\d{2}):(\d{2})"#).unwrap(); + let tid_attr = Regex::new(r#"data-trackId="([^"]+)""#).unwrap(); + let tid_link = Regex::new(r#"/musik/titel/[^/"]+/(\d+-\d+-\d+)"#).unwrap(); + let title = Regex::new(r#"(?s)\s*(?:]*>)?([^<]+)"#).unwrap(); + let names = Regex::new(r#"([^<]+)"#).unwrap(); + let ids = Regex::new(r"kunstner/async/(\d+)/").unwrap(); + let details = Regex::new(r#"(?s)
(.*?)
"#).unwrap(); + let span = Regex::new(r"([^<]*)").unwrap(); + let tz = chrono_tz::Europe::Copenhagen; + let (mut day, mut last_min) = (page_date, -1i32); + let mut out = Vec::new(); + for m in item.captures_iter(html) { + let whole = m.get(0).unwrap().as_str(); + let body = &m[1]; + let attrs = head.captures(whole).map(|c| c[1].to_string()).unwrap_or_default(); + let Some(ti) = title.captures(body) else { continue }; + let Some(track_id) = tid_attr.captures(&attrs).or_else(|| tid_link.captures(body)).map(|c| c[1].to_string()) else { continue }; + let at = if let Some(t) = full.captures(body) { format!("{}T{}Z", &t[1], &t[2]) } else if let Some(c) = clock.captures(body) { + let (h, mi): (u32, u32) = (c[1].parse().unwrap_or(0), c[2].parse().unwrap_or(0)); + let minute = (h * 60 + mi) as i32; + if last_min >= 0 && minute + 6 * 60 < last_min { day = day.succ_opt().unwrap_or(day); } + last_min = minute; + let Some(local) = day.and_hms_opt(h, mi, 0) else { continue }; + use chrono::TimeZone; + match tz.from_local_datetime(&local).earliest() { Some(d) => d.with_timezone(&Utc).format("%Y-%m-%dT%H:%M:%SZ").to_string(), None => continue } + } else { continue }; + let info = body.find("class=\"trackInfo\"").map(|i| &body[i..]).unwrap_or(body); + let info = info.find("trackDetails").map(|i| &info[..i]).unwrap_or(info); + let artists: Vec = names.captures_iter(info).map(|c| unescape(c[1].trim())).collect(); + let artist_ids: Vec = ids.captures_iter(whole).map(|c| c[1].to_string()).collect(); + let d: Vec = details.captures(body).map(|c| span.captures_iter(&c[1]).map(|s| unescape(s[1].trim())).collect()).unwrap_or_default(); + let get = |p: &str| d.iter().find(|x| x.starts_with(p)).map(|x| x[p.len()..].trim().to_string()); + out.push(Play { + at, track_id, title: unescape(ti[1].trim()), artists: artists.join(", "), artist_ids: artist_ids.join(","), + recorded: get("Indspillet:"), released: get("UDGIVET:"), + country: d.iter().find(|x| !x.contains(':') && !x.is_empty()).cloned(), + }); + } + out +} + +pub fn collect(db: &Db, http: &Http, log: &mut Vec) -> Result { + create_tables(db)?; + let path = Regex::new(r"/playlister/([a-z0-9]+)/(\d{4})-(\d{1,2})-(\d{1,2})")?; + let mut pages = 0; + for ch in CHANNELS { + // The CDX index is often overloaded (503): retry patiently. + let url = format!("https://web.archive.org/cdx/search/cdx?url=www.dr.dk/playlister/{ch}/*&output=json&fl=timestamp,original&filter=statuscode:200&collapse=urlkey&limit=50000"); + let mut cdx = None; + for attempt in 0..8 { + match http.get(&url) { + Ok(Some(c)) if c.trim_start().starts_with('[') => { cdx = Some(c); break; } + Ok(Some(_)) | Ok(None) if attempt >= 2 => break, // answered, but not an index: nothing archived + Ok(_) => {} + Err(e) => { if attempt == 7 { log.push(format!("archive cdx {ch}: {e}")); } } + } + std::thread::sleep(std::time::Duration::from_secs(30)); + } + let Some(cdx) = cdx else { continue }; + eprintln!("archive {ch}: index read"); + let rows: Vec> = serde_json::from_str(&cdx).unwrap_or_default(); + for row in rows.iter().skip(1) { + let (ts, url) = (&row[0], &row[1]); + if !path.is_match(url) { continue; } + let clean = url.replace(":80/", "/").replace("http://", "https://"); + let done: i64 = db.conn.query_row("SELECT count(*) FROM archive_pages WHERE url = ?1", [&clean], |r| r.get(0))?; + if done > 0 { continue; } + let html = match http.get(&format!("https://web.archive.org/web/{ts}id_/{url}")) { + Ok(h) => h.unwrap_or_default(), + Err(e) => { log.push(format!("archive {url}: {e}")); continue; } + }; + let Some(c) = path.captures(url) else { continue }; + let Some(page_date) = chrono::NaiveDate::from_ymd_opt(c[2].parse().unwrap_or(0), c[3].parse().unwrap_or(0), c[4].parse().unwrap_or(0)) else { continue }; + let plays = parse(&html, page_date); + let tx = db.conn.unchecked_transaction()?; + for p in &plays { + tx.execute("INSERT OR IGNORE INTO archive_plays VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)", + params![ch, p.at, p.track_id, p.artists, p.title, p.artist_ids, clean, p.recorded, p.country, p.released])?; + } + tx.execute("INSERT OR REPLACE INTO archive_pages VALUES (?1, ?2, ?3, ?4)", params![clean, ts, plays.len() as i64, Utc::now().to_rfc3339()])?; + tx.commit()?; + pages += 1; + } + } + Ok(pages) +} + +/// Archived days per channel and how much of each day is covered (hours with at least one play). +#[allow(dead_code)] +pub fn coverage(db: &Db) -> Result> { + create_tables(db)?; + let mut out = BTreeMap::new(); + for r in db.conn.prepare("SELECT station, substr(played_at, 1, 10), count(DISTINCT substr(played_at, 12, 2)) FROM archive_plays GROUP BY 1, 2")? + .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, i64>(2)?)))? { + let (s, d, h) = r?; + out.insert((s, d), h as usize); + } + Ok(out) +} diff --git a/src/main.rs b/src/main.rs index 456c956..c878e58 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,6 @@ //! airplay: collects what Danish radio stations play, for modelling when artists get played again. //! -//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks] [--backfill] +//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive] [--backfill] //! airplay export nmf-denmark [--out data/nmf_denmark.csv] //! airplay stats [--db data/airplay.db] //! airplay check [--db data/airplay.db] gaps, duplicates, DR cross-check, chart completeness @@ -11,6 +11,7 @@ //! --backfill fetches all history at once (every DR artist page, the whole Hitlisten archive); //! without it a run takes a bounded slice, which is what the daily timer does. +mod archive; mod bauer; mod check; mod db; @@ -60,7 +61,7 @@ fn main() -> Result<()> { rotations::store_all(&db, &dir, args.iter().any(|a| a == "--reparse")) } _ => { - eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks] [--backfill] + eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive] [--backfill] //! airplay export nmf-denmark [--out data/nmf_denmark.csv]\n airplay stats [--db FILE]\n airplay check [--db FILE]\n airplay model panel|backtest|rank\n airplay serve [--addr HOST:PORT] [--prefix /airplay]"); std::process::exit(2); } @@ -139,6 +140,11 @@ fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> { let r = playlists::nmf_artists(db, &sfc_http, if backfill { usize::MAX } else { 800 }, &mut log); step("new music friday per artist (superfridaychart)", r.map(|n| format!("{n} artists looked up")), &mut log); } + if only.as_deref() == Some("archive") { + // Wayback Machine copies of old DR playlists (only on request: about an hour, once). + let r = archive::collect(db, &http::Http::new(1500)?, &mut log); + step("dr playlists from the wayback machine", r.map(|n| format!("{n} pages read")), &mut log); + } if wants("nmf") || wants("nmf-charts") { let r = playlists::nmf_global_charts(db, &http::Http::new(1000)?, &mut log); step("new music friday global charts (superfridaychart)", r.map(|n| format!("{n} Fridays read")), &mut log); diff --git a/src/model/mod.rs b/src/model/mod.rs index e6de231..43a89df 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -5,6 +5,7 @@ //! airplay model backtest [--h 1,4,8] [--refit 1] [--out docs/BACKTEST.md] //! airplay model calibrate [--out docs/CALIBRATION.md] rotation thresholds from DR's lists //! airplay model nmf [--out docs/NMF.md] New Music Friday Denmark vs DR adds +//! airplay model synth [--out docs/SYNTH.md] synthetic rotation lists (archive + thresholds) //! airplay model ifpi the 20-year IFPI test bed -> docs/IFPI.md //! airplay model adds [--h 0,4] [--out docs/BACKTEST-ADDS.md] backtest of the new-add model //! airplay model rank [--h 4] [--max-h 12] [--top 50] @@ -20,6 +21,7 @@ pub mod kalman; pub mod nmf; pub mod panel; pub mod rank; +pub mod synth; use crate::db::Db; use anyhow::Result; @@ -59,6 +61,11 @@ pub fn run(db: &Db, args: &[String]) -> Result<()> { print!("{}", report.markdown); eprintln!("written to {out}"); } + Some("synth") => { + let md = synth::run(db, &p)?; + if let Some(out) = arg(args, "--out") { std::fs::write(out, &md)?; } + print!("{md}"); + } Some("nmf") => { let md = nmf::run(db, &p)?; if let Some(out) = arg(args, "--out") { std::fs::write(out, &md)?; } diff --git a/src/model/synth.rs b/src/model/synth.rs new file mode 100644 index 0000000..376b686 --- /dev/null +++ b/src/model/synth.rs @@ -0,0 +1,226 @@ +//! Synthetic rotation lists, for weeks without a published list (`airplay model synth` -> docs/SYNTH.md). +//! +//! 1. Learn, per channel and category, how many plays a day a listed song gets: exact plays +//! (DR's playlists) against DR's own lists. The list for the week of the plays may be missing; +//! then songs with the same category on the lists before and after stand in for it. +//! 2. Apply the thresholds to archived playlists (Wayback Machine, 2012–2021): per channel and +//! ISO week, pool the archived plays, scale to a full day by the hours covered, and give each +//! song a category. Stored in `synthetic_rotation` with the coverage behind it. +//! 3. The national series: Airplay Top-20 positions (2001–) against DR plays in the year the two +//! overlap, so each position reads as an estimated number of DR plays a week. + +use super::panel::{key, Panel, C, CHANNELS}; +use crate::db::Db; +use anyhow::Result; +use chrono::{Datelike, Duration, NaiveDate, Weekday}; +use std::collections::{BTreeMap, HashMap}; +use std::fmt::Write as _; + +const CATS: [&str; 3] = ["A", "B", "C"]; + +/// Song key: primary artist + title without version tags. +fn song(artist: &str, title: &str) -> String { + let a = artist.to_lowercase(); + let mut cut = a.len(); + for sep in [" [+] ", " feat. ", " feat ", " ft. ", " ft ", " featuring ", " & ", ", ", " x ", " og ", " med "] { + if let Some(i) = a.find(sep) { cut = cut.min(i); } + } + let t = title.split(" (").next().unwrap_or(title).split(" - ").next().unwrap_or(title); + format!("{}|{}", key(&a[..cut]), key(t)) +} + +/// List channel for a play station (the ten regional P4s share the national P4 list). +fn list_channel(station: &str) -> Option<&'static str> { + match station { + "p3" => Some("p3"), "p5" => Some("p5"), "p6beat" => Some("p6beat"), "p8jazz" => Some("p8jazz"), + s if s.starts_with("p4") => Some("p4"), + _ => None, + } +} + +pub fn create_tables(db: &Db) -> Result<()> { + db.conn.execute_batch( + "CREATE TABLE IF NOT EXISTS synthetic_rotation ( + channel TEXT NOT NULL, iso_year INTEGER NOT NULL, iso_week INTEGER NOT NULL, + song TEXT NOT NULL, artist TEXT, title TEXT, + plays_per_day REAL NOT NULL, category TEXT NOT NULL, -- A | B | C (estimated) + days INTEGER NOT NULL, hours_covered INTEGER NOT NULL, source TEXT NOT NULL, + PRIMARY KEY (channel, iso_year, iso_week, song) + ); + CREATE TABLE IF NOT EXISTS synthetic_thresholds ( + channel TEXT NOT NULL, category TEXT NOT NULL, plays_per_day REAL NOT NULL, songs INTEGER, made_at TEXT NOT NULL, + PRIMARY KEY (channel, category) + );", + )?; + Ok(()) +} + +pub fn run(db: &Db, p: &Panel) -> Result { + create_tables(db)?; + let mut md = String::new(); + writeln!(md, "# Synthetic rotation lists\n")?; + + // ---- 1. thresholds from exact plays vs lists + // Lists by week (Monday) and channel: song -> category. + let mut lists: BTreeMap>> = BTreeMap::new(); + for r in db.conn.prepare("SELECT list_date, channel, category, artist, title FROM rotation_lists WHERE category IN ('A', 'B', 'C')")? + .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, String>(3)?, r.get::<_, String>(4)?)))? { + let (d, ch, cat, a, t) = r?; + let Ok(d) = NaiveDate::parse_from_str(&d, "%Y-%m-%d") else { continue }; + lists.entry(d + Duration::days(3)).or_default().entry(ch).or_default().insert(song(&a, &t), cat); + } + // Exact plays per week (Monday, Copenhagen time is close enough at week level), station, song. + let mut plays: BTreeMap>> = BTreeMap::new(); + let mut days_seen: BTreeMap> = BTreeMap::new(); + for r in db.conn.prepare("SELECT station, played_at, artist, title FROM plays WHERE source = 'dr'")? + .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, String>(3)?)))? { + let (st, at, a, t) = r?; + let Ok(d) = NaiveDate::parse_from_str(&at[..10], "%Y-%m-%d") else { continue }; + let monday = d - Duration::days(d.weekday().num_days_from_monday() as i64); + *plays.entry(monday).or_default().entry(st).or_default().entry(song(&a, &t)).or_insert(0.0) += 1.0; + days_seen.entry(monday).or_default().insert(at[..10].to_string()); + } + // The list in force in a week: the published one, or songs with the same category before and after. + let list_for = |week: NaiveDate, ch: &str| -> Option<(HashMap, &'static str)> { + if let Some(l) = lists.get(&week).and_then(|m| m.get(ch)) { return Some((l.clone(), "published")); } + let before = lists.range(..week).rev().find_map(|(d, m)| m.get(ch).map(|l| (*d, l)))?; + let after = lists.range(week + Duration::days(1)..).find_map(|(d, m)| m.get(ch).map(|l| (*d, l)))?; + if (after.0 - before.0).num_days() > 21 { return None; } + Some((before.1.iter().filter(|(s, c)| after.1.get(*s) == Some(*c)).map(|(s, c)| (s.clone(), c.clone())).collect(), "bridged")) + }; + let mut per_cat: HashMap<(String, String), Vec> = HashMap::new(); // (list channel, cat) -> plays/day + let mut unlisted: HashMap> = HashMap::new(); + let mut used = Vec::new(); + for (week, by_station) in &plays { + let days = days_seen.get(week).map(|s| s.len()).unwrap_or(0) as f64; + if days < 5.0 { continue; } // a week mostly covered + for (station, songs) in by_station { + let Some(lc) = list_channel(station) else { continue }; + let Some((list, how)) = list_for(*week, lc) else { continue }; + used.push(format!("{week} {station} ({how})")); + for (s, n) in songs { + let ppd = n / days; + match list.get(s) { Some(cat) => per_cat.entry((lc.to_string(), cat.clone())).or_default().push(ppd), None => unlisted.entry(lc.to_string()).or_default().push(ppd) } + } + } + } + writeln!(md, "## 1. Plays per day by category (exact plays vs DR's lists)\n")?; + writeln!(md, "Weeks used: {}.\n", if used.is_empty() { "none yet".to_string() } else { used.iter().take(6).cloned().collect::>().join(", ") + if used.len() > 6 { " …" } else { "" } })?; + writeln!(md, "| Channel | A median | B median | C median | Not listed, 90th pct | Threshold A | B | C |\n|---|---|---|---|---|---|---|---|")?; + let med = |v: &mut Vec| { if v.is_empty() { return f64::NAN; } v.sort_by(f64::total_cmp); v[v.len() / 2] }; + let pct = |v: &mut Vec, q: f64| { if v.is_empty() { return f64::NAN; } v.sort_by(f64::total_cmp); v[((v.len() - 1) as f64 * q) as usize] }; + let mut thresholds: HashMap = HashMap::new(); + let made = chrono::Utc::now().to_rfc3339(); + for lc in ["p3", "p4", "p5", "p6beat", "p8jazz"] { + let mut m = [0.0; 3]; + for (k, c) in CATS.iter().enumerate() { m[k] = med(per_cat.entry((lc.into(), c.to_string())).or_default()); } + let nl = pct(unlisted.entry(lc.into()).or_default(), 0.9); + // Category thresholds: halfway (geometric) between neighbouring medians; C's floor halfway to the unlisted 90th pct. + let gm = |a: f64, b: f64| (a * b).sqrt(); + // The list floor (C): the plays/day that best separates listed from unlisted songs (F1). + let listed: Vec = CATS.iter().flat_map(|c| per_cat.get(&(lc.to_string(), c.to_string())).cloned().unwrap_or_default()).collect(); + let others = unlisted.get(lc).cloned().unwrap_or_default(); + let mut floor = (m[2] / 2.0, 0.0); + for k in 2..=40 { + let t = k as f64 / 10.0; + let tp = listed.iter().filter(|x| **x >= t).count() as f64; + let fp = others.iter().filter(|x| **x >= t).count() as f64; + let f1 = 2.0 * tp / (2.0 * tp + fp + (listed.len() as f64 - tp)); + if f1 > floor.1 { floor = (t, f1); } + } + let th = [gm(m[0], m[1]), gm(m[1], m[2]).max(floor.0), floor.0]; + if m.iter().all(|x| x.is_finite()) { + thresholds.insert(lc.to_string(), th); + for (k, c) in CATS.iter().enumerate() { + db.conn.execute("INSERT OR REPLACE INTO synthetic_thresholds VALUES (?1, ?2, ?3, ?4, ?5)", + rusqlite::params![lc, c, th[k], per_cat.get(&(lc.to_string(), c.to_string())).map(|v| v.len() as i64), made])?; + } + } + writeln!(md, "| {lc} | {:.1} | {:.1} | {:.1} | {:.1} | {} |", m[0], m[1], m[2], nl, + thresholds.get(lc).map(|t| format!("{:.1} | {:.1} | {:.1}", t[0], t[1], t[2])).unwrap_or("— | — | —".into()))?; + } + // In-sample check: how well the thresholds reproduce the list in the weeks used. + let mut hit = [[0usize; 4]; 4]; + let cat_of = |th: &[f64; 3], ppd: f64| if ppd >= th[0] { 0 } else if ppd >= th[1] { 1 } else if ppd >= th[2] { 2 } else { 3 }; + for ((lc, c), v) in &per_cat { if let Some(th) = thresholds.get(lc) { let t = CATS.iter().position(|x| x == c).unwrap(); for ppd in v { hit[t][cat_of(th, *ppd)] += 1; } } } + for (lc, v) in &unlisted { if let Some(th) = thresholds.get(lc) { for ppd in v { hit[3][cat_of(th, *ppd)] += 1; } } } + let listed_total: usize = (0..3).map(|t| hit[t].iter().sum::()).sum(); + let listed_found: usize = (0..3).map(|t| hit[t][..3].iter().sum::()).sum(); + let false_pos: usize = hit[3][..3].iter().sum(); + writeln!(md, "\nIn-sample check (song-weeks): listed songs found {listed_found} of {listed_total}; songs wrongly put on the list {false_pos}; exact category {} of {listed_total}. Thresholds are refitted as exact plays and lists for the same weeks accumulate.\n", + (0..3).map(|t| hit[t][t]).sum::())?; + + // ---- 2. apply to the archive + let has_archive: i64 = db.conn.query_row("SELECT count(*) FROM sqlite_master WHERE name = 'archive_plays'", [], |r| r.get(0))?; + writeln!(md, "## 2. Archived playlists → synthetic lists\n")?; + if has_archive == 0 || thresholds.is_empty() { + writeln!(md, "Nothing to apply yet (archive or thresholds missing).\n")?; + } else { + // per (station, iso week): hours covered per day, plays per song + struct Wk { hours: HashMap>, songs: HashMap } + let mut weeks: BTreeMap<(String, i32, u32), Wk> = BTreeMap::new(); + for r in db.conn.prepare("SELECT station, played_at, artist, title FROM archive_plays")? + .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, String>(3)?)))? { + let (st, at, a, t) = r?; + let Ok(d) = NaiveDate::parse_from_str(&at[..10], "%Y-%m-%d") else { continue }; + let w = weeks.entry((st, d.iso_week().year(), d.iso_week().week())).or_insert_with(|| Wk { hours: HashMap::new(), songs: HashMap::new() }); + w.hours.entry(at[..10].to_string()).or_default().insert(at[11..13].to_string()); + let e = w.songs.entry(song(&a, &t)).or_insert((0.0, a.clone(), t.clone())); + e.0 += 1.0; + } + let tx = db.conn.unchecked_transaction()?; + tx.execute("DELETE FROM synthetic_rotation WHERE source = 'wayback'", [])?; + let mut summary: BTreeMap<(String, i32), (usize, usize)> = BTreeMap::new(); // (station, year) -> (weeks, songs listed) + for ((st, y, w), wk) in &weeks { + let Some(lc) = list_channel(st) else { continue }; + let Some(th) = thresholds.get(lc) else { continue }; + // Only daytime-complete days count fully; scale plays by covered hours / 24. + let hours: usize = wk.hours.values().map(|h| h.len()).sum(); + let days = wk.hours.values().filter(|h| h.len() >= 12).count(); + if hours < 36 { continue; } // less than a day and a half: too thin to tell a C song from chance + let day_equiv = hours as f64 / 24.0; + let mut n = 0; + for (s, (cnt, a, t)) in &wk.songs { + if *cnt < 3.0 { continue; } // a song needs 3+ plays in the sample to count + let ppd = cnt / day_equiv; + let k = cat_of(th, ppd); + if k == 3 { continue; } + tx.execute("INSERT OR REPLACE INTO synthetic_rotation VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, 'wayback')", + rusqlite::params![lc, y, w, s, a, t, ppd, CATS[k], days as i64, hours as i64])?; + n += 1; + } + let e = summary.entry((st.clone(), *y)).or_insert((0, 0)); + e.0 += 1; e.1 += n; + } + tx.commit()?; + writeln!(md, "| Station | Year | Weeks with a synthetic list | Avg songs listed per week |\n|---|---|---|---|")?; + for ((st, y), (wks, n)) in &summary { writeln!(md, "| {st} | {y} | {wks} | {:.0} |", *n as f64 / *wks as f64)?; } + writeln!(md, "\nA week needs at least 36 archived hours and a song at least 3 plays in them; plays are scaled to a full day by the hours covered. A/B is uncertain from a few sampled days (P3: A ≈ 4.7, B ≈ 3.3 plays a day); on P4, A and B get the same plays, so P4 synthetic lists only mean \"on the list\".\n")?; + } + + // ---- 3. IFPI Airplay Top-20 position vs DR plays + writeln!(md, "## 3. Airplay Top-20 position → DR plays a week\n")?; + let id_by_name: HashMap> = p.artists.iter().enumerate().fold(HashMap::new(), |mut m, (i, a)| { m.entry(key(&a.name)).or_default().push(i); m }); + let week_of: HashMap = p.weeks.iter().enumerate().map(|(i, d)| (*d, i)).collect(); + let mut by_pos: BTreeMap> = BTreeMap::new(); + for r in db.conn.prepare("SELECT year, week, position, artist FROM hitlisten_charts WHERE chart = 'airplay'")? + .query_map([], |r| Ok((r.get::<_, i32>(0)?, r.get::<_, u32>(1)?, r.get::<_, i64>(2)?, r.get::<_, String>(3)?)))? { + let (y, w, pos, a) = r?; + let Some(monday) = NaiveDate::from_isoywd_opt(y, w, Weekday::Mon) else { continue }; + let Some(&wi) = week_of.get(&monday) else { continue }; + let primary = song(&a, "x"); + let primary = primary.split('|').next().unwrap_or(""); + let Some(ids) = id_by_name.get(primary) else { continue }; + let plays: f64 = ids.iter().map(|&i| (0..C).map(|c| p.y(i, c, wi)).sum::()).fold(0.0, f64::max); + by_pos.entry((pos - 1) / 5).or_default().push(plays); + } + writeln!(md, "Artist's DR plays (all {} channels) in the chart week, for Airplay Top-20 entries {}–{}:\n", C, p.weeks[0], p.weeks[p.w() - 1])?; + writeln!(md, "| Positions | Entries | Median DR plays | 25th–75th pct |\n|---|---|---|---|")?; + for (b, mut v) in by_pos { + let n = v.len(); + writeln!(md, "| {}–{} | {n} | {:.0} | {:.0}–{:.0} |", b * 5 + 1, b * 5 + 5, med(&mut v.clone()), pct(&mut v, 0.25), pct(&mut v.clone(), 0.75))?; + } + writeln!(md, "\nSo the 25-year Airplay Top-20 can stand in for \"in heavy rotation nationally\" (on the order of the plays above), but it only covers the top 20 songs each week; it can't produce B/C lists or anything below.\n")?; + let _ = CHANNELS; + Ok(md) +}