//! DR's weekly rotation lists (PDF): which songs are in rotation, in which category, per channel. //! //! The PDF merges one page per channel. Each page has category headers (A, B, C, and on the //! P3 page also "Nye i power", "Ugens crush", "P3's uundgåelige") with artist + title lines //! under them, in one or several columns. Glyph positions are read from the PDF; a line belongs //! to the nearest header above it in its column. Which channel a page is gets decided from the //! data: the channel where the page's artists are played most (DR artist pages), so a change in //! page order doesn't mislabel anything. The P4 list is national and stored as channel "p4". //! //! airplay rotations [--reparse] parse every saved PDF into rotation_lists //! airplay rotations --wayback also fetch older lists archived by the Wayback Machine use crate::{db::Db, http::Http}; use anyhow::{anyhow, Result}; use chrono::{Datelike, NaiveDate, Utc}; use pdf_extract::{MediaBox, OutputDev, OutputError, Transform}; use regex::Regex; use rusqlite::params; use std::collections::HashMap; pub fn create_tables(db: &Db) -> Result<()> { db.conn.execute_batch( "CREATE TABLE IF NOT EXISTS rotation_lists ( list_date TEXT NOT NULL, -- the date in the PDF's file name (published, a Friday) iso_year INTEGER, iso_week INTEGER, -- 'UGE nn' printed on the list: the week it applies to channel TEXT NOT NULL, -- p3 | p4 (national, all regions) | p5 | p6beat | p8jazz category TEXT NOT NULL, -- A | B | C | or the list's own label (NYE I POWER, UGENS CRUSH, UUNDGÅELIGE) position INTEGER NOT NULL, -- order within the category artist TEXT NOT NULL, title TEXT NOT NULL, file TEXT NOT NULL, page INTEGER NOT NULL, channel_score REAL, -- how clearly the page matched its channel (share of plays) PRIMARY KEY (list_date, channel, category, position) ); CREATE TABLE IF NOT EXISTS rotation_wayback ( snapshot TEXT PRIMARY KEY, pdf_url TEXT, fetched_at TEXT NOT NULL, status TEXT );", )?; Ok(()) } // ------------------------------------------------------------------ glyphs → runs #[derive(Debug, Clone)] pub struct Run { pub page: u32, pub x: f64, pub y: f64, pub x_end: f64, pub size: f64, pub text: String } #[derive(Default)] struct Glyphs { page: u32, width: f64, height: f64, glyphs: Vec<(u32, f64, f64, f64, f64, String)> } impl OutputDev for Glyphs { fn begin_page(&mut self, page_num: u32, media_box: &MediaBox, _: Option<(f64, f64, f64, f64)>) -> Result<(), OutputError> { self.page = page_num; self.width = media_box.urx - media_box.llx; self.height = media_box.ury - media_box.lly; Ok(()) } fn end_page(&mut self) -> Result<(), OutputError> { Ok(()) } fn output_character(&mut self, trm: &Transform, width: f64, _spacing: f64, font_size: f64, ch: &str) -> Result<(), OutputError> { let scale = (trm.m11 * trm.m11 + trm.m12 * trm.m12).sqrt(); let size = font_size * scale; let (x, y) = (trm.m31, self.height - trm.m32); // Decoration: rotated text (the vertical "ROTATIONSLISTEN" frame, the tilted "P3 ROTATIONER" // background of older lists) and big letters along the page edges. // (m12 is the rotation; m21 alone is the skew of fake italics, which is kept) let rotated = trm.m12.abs() > 0.2 * scale.max(1e-9); let margin = size > 25.0 && (x < 0.06 * self.width || x > 0.94 * self.width || y < 0.06 * self.height || y > 0.94 * self.height); if !margin && !rotated { self.glyphs.push((self.page, x, y, width * size, size, ch.to_string())); } Ok(()) } fn begin_word(&mut self) -> Result<(), OutputError> { Ok(()) } fn end_word(&mut self) -> Result<(), OutputError> { Ok(()) } fn end_line(&mut self) -> Result<(), OutputError> { Ok(()) } } /// Text runs per page. Glyphs drawn twice at one spot (fake bold) count once; a run ends at a /// gap wider than ~1.6 times the font size (a column gap). pub fn runs(bytes: &[u8]) -> Result> { let doc = pdf_extract::Document::load_mem(bytes)?; let mut g = Glyphs::default(); pdf_extract::output_doc(&doc, &mut g)?; let mut glyphs = g.glyphs; glyphs.sort_by(|a, b| a.0.cmp(&b.0).then(a.2.total_cmp(&b.2)).then(a.1.total_cmp(&b.1))); let mut lines: Vec> = Vec::new(); for gl in glyphs { match lines.last_mut() { Some(l) if l[0].0 == gl.0 && (gl.2 - l[0].2).abs() < l[0].4.min(gl.4).max(1.0) * 0.35 => l.push(gl), _ => lines.push(vec![gl]), } } let mut out = Vec::new(); for mut l in lines { l.sort_by(|a, b| a.1.total_cmp(&b.1)); l.dedup_by(|b, a| a.5 == b.5 && (a.1 - b.1).abs() < 0.5); let mut cur: Option = None; for (page, x, y, w, size, ch) in l { if let Some(r) = cur.as_mut() { let same_size = (size - r.size).abs() < 0.5; if same_size && x - r.x_end < size * 0.45 { r.text.push_str(&ch); r.x_end = r.x_end.max(x + w); continue; } if same_size && x - r.x_end < size * 1.6 { r.text.push(' '); r.text.push_str(&ch); r.x_end = x + w; continue; } out.push(cur.take().unwrap()); } cur = Some(Run { page, x, y, x_end: x + w, size, text: ch }); } if let Some(r) = cur { out.push(r); } } for r in &mut out { r.text = r.text.split_whitespace().collect::>().join(" "); } out.retain(|r| !r.text.is_empty()); Ok(out) } pub fn dump(path: &std::path::Path) -> Result<()> { for r in runs(&std::fs::read(path)?)? { println!("p{} y{:7.1} x{:6.1}-{:6.1} s{:4.1} | {}", r.page, r.y, r.x, r.x_end, r.size, r.text); } Ok(()) } // ------------------------------------------------------------------ runs → entries #[derive(Debug, Clone)] pub struct Entry { pub page: u32, pub category: String, pub position: usize, pub artist: String, pub title: String } pub struct Parsed { pub week: Option, pub entries: Vec } pub fn parse(bytes: &[u8]) -> Result { let runs = runs(bytes)?; let week_re = Regex::new(r"UGE\s*(\d{1,2})")?; let week = runs.iter().find_map(|r| week_re.captures(&r.text).and_then(|c| c[1].parse().ok())); let is_cat = |t: &str| matches!(t, "A" | "B" | "C"); let is_label = |r: &Run| r.size >= 12.0 && r.size < 40.0 && r.text.chars().count() >= 4 && !is_cat(&r.text) && !r.text.starts_with("UGE") && !r.text.starts_with("E ") && !r.text.contains("ROTATIONSLISTEN"); let mut entries = Vec::new(); let pages: Vec = { let mut p: Vec = runs.iter().map(|r| r.page).collect(); p.dedup(); p }; for page in pages { let pr: Vec<&Run> = runs.iter().filter(|r| r.page == page).collect(); // Headers: category letters and labelled sections. let headers: Vec<(&Run, String)> = pr.iter().filter(|r| (r.size >= 12.0 && is_cat(&r.text)) || is_label(r)) .map(|r| (*r, if is_cat(&r.text) { r.text.clone() } else { label(&r.text) })).collect(); // "P3's uundgåelige": the two big runs after the label are artist and title. if let Some((h, _)) = headers.iter().find(|(r, _)| r.text.to_uppercase().contains("UUNDGÅELIGE")) { let big: Vec<&&Run> = pr.iter().filter(|r| r.size >= 40.0 && r.y > h.y && !is_cat(&r.text)).take(2).collect(); if big.len() == 2 { entries.push(Entry { page, category: "UUNDGÅELIGE".into(), position: 1, artist: big[0].text.clone(), title: big[1].text.clone() }); } } let body: Vec<&Run> = pr.iter().copied().filter(|r| r.size < 12.0).collect(); // Columns start at the category letters (A, B, C side by side on the P3 summary page). let mut cols: Vec = headers.iter().filter(|(r, _)| is_cat(&r.text)).map(|(r, _)| r.x).collect(); cols.sort_by(f64::total_cmp); cols.dedup_by(|b, a| (*b - *a).abs() < 40.0); let col_of = |x: f64| cols.iter().rposition(|c| *c <= x + 20.0).unwrap_or(0); // Lines: same y and same column; the header: the lowest one above, in that column. let mut lines: Vec<(usize, Vec<&Run>)> = Vec::new(); let mut sorted = body.clone(); sorted.sort_by(|a, b| a.y.total_cmp(&b.y).then(a.x.total_cmp(&b.x))); for r in sorted { let c = col_of(r.x); match lines.iter_mut().find(|(lc, l)| *lc == c && (l[0].y - r.y).abs() < 2.0) { Some((_, l)) => l.push(r), None => lines.push((c, vec![r])), } } let mut count: HashMap = HashMap::new(); for (c, mut l) in lines { l.sort_by(|a, b| a.x.total_cmp(&b.x)); if l.len() < 2 { continue; } // a lone run: not an artist + title pair let Some((hi, (_, cat))) = headers.iter().enumerate() .filter(|(_, (h, _))| col_of(h.x) == c && h.y < l[0].y + 1.0) .max_by(|a, b| a.1 .0.y.total_cmp(&b.1 .0.y)) else { continue }; let n = count.entry(hi).or_insert(0); *n += 1; entries.push(Entry { page, category: cat.clone(), position: *n, artist: l[0].text.clone(), title: l[1..].iter().map(|r| r.text.as_str()).collect::>().join(" ") }); } } Ok(Parsed { week, entries }) } fn label(t: &str) -> String { let u = t.to_uppercase(); if u.contains("CRUSH") { "UGENS CRUSH".into() } else if u.contains("NYE I POWER") { "NYE I POWER".into() } else if u.contains("RECURRENT") { "RECURRENTS".into() } else if u.contains("ROTATIONER") || u.contains("ROTATIONSLISTE") { "LIST".into() } else { u } } // ------------------------------------------------------------------ pages → channels const LIST_CHANNELS: [&str; 5] = ["p3", "p4", "p5", "p6beat", "p8jazz"]; /// Share of each list channel in the plays of the page's artists (DR artist pages, all weeks). fn channel_scores(db: &Db, artists: &[&str]) -> Result<[f64; 5]> { let key = crate::model::panel::key; let mut by_name: HashMap> = HashMap::new(); for r in db.conn.prepare("SELECT artist_id, name FROM dr_artists")?.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? { let (id, n) = r?; by_name.entry(key(&n)).or_default().push(id); } let mut st = db.conn.prepare("SELECT channel, sum(plays) FROM dr_artist_weeks WHERE artist_id = ?1 GROUP BY channel")?; let mut score = [0.0; 5]; for a in artists { let first = a.split(&['[', ','][..]).next().unwrap_or(a).split(" feat").next().unwrap_or(a).split(" ft.").next().unwrap_or(a) .split(" & ").next().unwrap_or(a).split(" x ").next().unwrap_or(a).trim(); let Some(ids) = by_name.get(&key(first)) else { continue }; let mut per = [0.0; 5]; for id in ids { for r in st.query_map([id], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?)))? { let (ch, n) = r?; let k = if ch.starts_with("p4") && ch != "p4" { Some(1) } else { LIST_CHANNELS.iter().position(|c| *c == ch) }; // the ten regional P4s count as one channel if let Some(k) = k { per[k] += n as f64 / if k == 1 { 10.0 } else { 1.0 }; } } } let tot: f64 = per.iter().sum(); if tot > 0.0 { for k in 0..5 { score[k] += per[k] / tot; } } } Ok(score) } /// Parses one PDF and stores its rows (a list date already stored is left alone unless `replace`). pub fn store(db: &Db, file: &std::path::Path, replace: bool, log: &mut Vec) -> Result { create_tables(db)?; let name = file.file_name().and_then(|n| n.to_str()).unwrap_or_default().to_string(); let date = Regex::new(r"^(\d{2})-(\d{2})-(\d{4})")?.captures(&name) .and_then(|c| NaiveDate::from_ymd_opt(c[3].parse().ok()?, c[2].parse().ok()?, c[1].parse().ok()?)) .or_else(|| Regex::new(r"^(\d{4})-(\d{2})-(\d{2})").ok()?.captures(&name) .and_then(|c| NaiveDate::from_ymd_opt(c[1].parse().ok()?, c[2].parse().ok()?, c[3].parse().ok()?))) .ok_or_else(|| anyhow!("{name}: no date in the file name"))?; let have: i64 = db.conn.query_row("SELECT count(*) FROM rotation_lists WHERE list_date = ?1", [date.to_string()], |r| r.get(0))?; if have > 0 && !replace { return Ok(0); } let parsed = parse(&std::fs::read(file)?)?; // ISO week of the list: "UGE nn" near the list date (the list is for the coming week). let iso = parsed.week.map(|w| { let y = date.iso_week().year(); let y = if w < 10 && date.iso_week().week() > 40 { y + 1 } else { y }; (y, w) }); let mut pages: Vec = parsed.entries.iter().map(|e| e.page).collect(); pages.sort(); pages.dedup(); let mut channel_of: HashMap = HashMap::new(); let mut taken: Vec = Vec::new(); // Most distinctive pages first, each channel used once (P3 may span two pages: the summary page with "uundgåelige"). let mut scored: Vec<(u32, [f64; 5], f64)> = Vec::new(); for &pg in &pages { let artists: Vec<&str> = parsed.entries.iter().filter(|e| e.page == pg).map(|e| e.artist.as_str()).collect(); let s = channel_scores(db, &artists)?; let tot: f64 = s.iter().sum(); scored.push((pg, s, tot)); } scored.sort_by(|a, b| { let m = |x: &[f64; 5], t: f64| x.iter().cloned().fold(0.0, f64::max) / t.max(1e-9); m(&b.1, b.2).total_cmp(&m(&a.1, a.2)) }); for (pg, s, tot) in &scored { let mut order: Vec = (0..5).collect(); order.sort_by(|&a, &b| s[b].total_cmp(&s[a])); let k = order.iter().copied().find(|k| !taken.contains(k) || (*k == 0)).unwrap_or(order[0]); if k != 0 { taken.push(k); } channel_of.insert(*pg, (LIST_CHANNELS[k].to_string(), s[k] / tot.max(1e-9))); if s[k] / tot.max(1e-9) < 0.4 { log.push(format!("{name} page {pg}: weak channel match ({} {:.2})", LIST_CHANNELS[k], s[k] / tot.max(1e-9))); } } let tx = db.conn.unchecked_transaction()?; tx.execute("DELETE FROM rotation_lists WHERE list_date = ?1", [date.to_string()])?; let mut n = 0; for e in &parsed.entries { let (ch, score) = channel_of.get(&e.page).cloned().unwrap_or(("unknown".into(), 0.0)); // The P3 summary page repeats categories A/B/C of a sub-list; keep its own labels apart. let cat = if ch == "p3" && pages.len() > 5 && e.page == pages[0] && matches!(e.category.as_str(), "A" | "B" | "C") { format!("P3 SIDE {}", e.category) } else { e.category.clone() }; tx.execute("INSERT OR REPLACE INTO rotation_lists VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)", params![date.to_string(), iso.map(|x| x.0), iso.map(|x| x.1), ch, cat, e.position as i64, e.artist, e.title, name, e.page as i64, score])?; n += 1; } tx.commit()?; Ok(n) } /// Parses every PDF in `dir`. pub fn store_all(db: &Db, dir: &std::path::Path, replace: bool) -> Result<()> { let mut log = Vec::new(); let mut files: Vec<_> = std::fs::read_dir(dir)?.filter_map(|e| e.ok()).map(|e| e.path()).filter(|p| p.extension().is_some_and(|x| x == "pdf")).collect(); files.sort(); for f in files { match store(db, &f, replace, &mut log) { Ok(n) => println!("{}: {n} rows", f.display()), Err(e) => println!("{}: {e}", f.display()), } } for l in log { println!(" {l}"); } let mut st = db.conn.prepare("SELECT list_date, channel, category, count(*), round(avg(channel_score), 2) FROM rotation_lists GROUP BY 1, 2, 3 ORDER BY 1, 2, 3")?; for r in st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, i64>(3)?, r.get::<_, f64>(4)?)))? { let (d, c, k, n, s) = r?; println!(" {d} {c:8} {k:14} {n:3} (channel match {s})"); } Ok(()) } // ------------------------------------------------------------------ Wayback Machine /// Older lists: every archived copy of DR's rotation page links that week's PDF; fetch the PDF /// from DR if it's still there, else from the archive. Each snapshot is tried once. pub fn wayback(db: &Db, http: &Http, dir: &std::path::Path, log: &mut Vec) -> Result { create_tables(db)?; let page = "www.dr.dk/om-dr/nyheder/musik-i-rotation-paa-drs-radiokanaler-netop-nu"; let cdx = http.get(&format!("https://web.archive.org/cdx/search/cdx?url={page}&output=json&fl=timestamp,statuscode&filter=statuscode:200&collapse=digest"))? .ok_or_else(|| anyhow!("no CDX answer"))?; let rows: Vec> = serde_json::from_str(&cdx).map_err(|e| anyhow!("CDX: {e}"))?; // The link has moved over the years: www.dr.dk/static/documents/... (2023-24), asset.dr.dk/... (later). let pdf_re = Regex::new(r#"https?://(?:asset\.dr\.dk|www\.dr\.dk/static)/[^"'\s<>]+?rotation[^"'\s<>]*?\.pdf"#)?; let path_date = Regex::new(r"/(\d{4})/(\d{2})/(\d{2})/")?; std::fs::create_dir_all(dir)?; let mut saved = 0; for row in rows.iter().skip(1) { let ts = &row[0]; let done: i64 = db.conn.query_row("SELECT count(*) FROM rotation_wayback WHERE snapshot = ?1", [ts], |r| r.get(0))?; if done > 0 { continue; } let html = http.get(&format!("https://web.archive.org/web/{ts}id_/https://{page}"))?.unwrap_or_default(); let url = pdf_re.find(&html).map(|m| m.as_str().replace("http://", "https://")); let status = match &url { None => "no pdf link".to_string(), Some(u) => { // Name the file by its publication date (dd-mm-yyyy first, which `store` reads). let base = u.rsplit('/').next().unwrap_or("rotation.pdf"); let name = match path_date.captures(u) { Some(c) => format!("{}-{}-{}-{base}", &c[3], &c[2], &c[1]), None => base.to_string() }; let file = dir.join(name); if file.exists() { "already saved".to_string() } else { let bytes = match http.get_bytes(u) { Ok(Some(b)) if b.starts_with(b"%PDF") => Some(b), _ => http.get_bytes(&format!("https://web.archive.org/web/{ts}id_/{u}")).ok().flatten().filter(|b| b.starts_with(b"%PDF")), }; match bytes { Some(b) => { std::fs::write(&file, &b)?; db.conn.execute("INSERT OR IGNORE INTO dr_rotation_pdfs VALUES (?1, ?2, ?3)", params![u, file.to_string_lossy(), Utc::now().to_rfc3339()])?; saved += 1; "saved".to_string() } None => { log.push(format!("wayback {ts}: {u} not retrievable")); "pdf not retrievable".to_string() } } } } }; db.conn.execute("INSERT OR REPLACE INTO rotation_wayback VALUES (?1, ?2, ?3, ?4)", params![ts, url, Utc::now().to_rfc3339(), status])?; println!("{ts}: {status} {}", url.unwrap_or_default()); } Ok(saved) }