Wayback archive of old DR playlists (2012-2021, both page layouts) into archive_plays; synthetic rotation lists (thresholds from exact plays vs lists, applied to archived weeks); Airplay Top-20 position vs DR plays
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+150
@@ -0,0 +1,150 @@
|
||||
//! DR playlists from the Wayback Machine: archived copies of dr.dk/playlister/{channel}/{date}/…
|
||||
//! (2012–2021, mostly P3 2013–2017). The old pages list each play with DR's track id, the exact
|
||||
//! UTC time, title, artists and DR artist id, the same ids as today, so archived plays join the
|
||||
//! artist pool directly. Kept apart from the live `plays` table (sampled days, not continuous).
|
||||
//!
|
||||
//! airplay collect --only archive
|
||||
|
||||
use crate::{db::Db, http::Http};
|
||||
use anyhow::Result;
|
||||
use chrono::Utc;
|
||||
use regex::Regex;
|
||||
use rusqlite::params;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
const CHANNELS: &[&str] = &["p3", "p4kbh", "p4", "p5", "p6beat", "p7mix", "p8jazz", "p4fyn", "p4aarhus", "p4nord", "p4vest", "p4syd",
|
||||
"p4sjaelland", "p4trekanten", "p4bornholm", "p4esbjerg", "p4ostjylland", "p4midtvest"];
|
||||
|
||||
pub fn create_tables(db: &Db) -> Result<()> {
|
||||
db.conn.execute_batch(
|
||||
"CREATE TABLE IF NOT EXISTS archive_plays (
|
||||
station TEXT NOT NULL, played_at TEXT NOT NULL, track_id TEXT NOT NULL,
|
||||
artist TEXT NOT NULL, title TEXT NOT NULL, artist_ids TEXT NOT NULL DEFAULT '',
|
||||
page TEXT NOT NULL,
|
||||
recorded TEXT, country TEXT, released TEXT, -- 2012-13 pages: 'Indspillet', country, 'UDGIVET: year, label'
|
||||
PRIMARY KEY (station, played_at, track_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS archive_plays_day ON archive_plays (station, substr(played_at, 1, 10));
|
||||
CREATE TABLE IF NOT EXISTS archive_pages (
|
||||
url TEXT PRIMARY KEY, snapshot TEXT NOT NULL, plays INTEGER, fetched_at TEXT NOT NULL
|
||||
);",
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn unescape(s: &str) -> String {
|
||||
let s = s.replace("&", "&").replace(""", "\"").replace("'", "'").replace("<", "<").replace(">", ">");
|
||||
// numeric entities (ø …)
|
||||
let re = Regex::new(r"&#(\d+);").unwrap();
|
||||
re.replace_all(&s, |c: ®ex::Captures| char::from_u32(c[1].parse().unwrap_or(63)).map(String::from).unwrap_or_default()).into_owned()
|
||||
}
|
||||
|
||||
pub struct Play { pub at: String, pub track_id: String, pub title: String, pub artists: String, pub artist_ids: String,
|
||||
pub recorded: Option<String>, pub country: Option<String>, pub released: Option<String> }
|
||||
|
||||
/// The plays on one archived page. Two layouts: from about 2014 each item carries its UTC time
|
||||
/// (data-fulltime) and track id; the 2012–13 pages show the local clock time (the date comes from
|
||||
/// the URL, rolling over at midnight) and add recording year, country and release.
|
||||
pub fn parse(html: &str, page_date: chrono::NaiveDate) -> Vec<Play> {
|
||||
let item = Regex::new(r#"(?s)<li[^>]*itemtype="http://schema.org/MusicRecording"[^>]*>(.*?)</li>"#).unwrap();
|
||||
let head = Regex::new(r#"(?s)<li([^>]*)itemtype"#).unwrap();
|
||||
let full = Regex::new(r#"data-fulltime="(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2})Z""#).unwrap();
|
||||
let clock = Regex::new(r#"<time[^>]*>(\d{2}):(\d{2})</time>"#).unwrap();
|
||||
let tid_attr = Regex::new(r#"data-trackId="([^"]+)""#).unwrap();
|
||||
let tid_link = Regex::new(r#"/musik/titel/[^/"]+/(\d+-\d+-\d+)"#).unwrap();
|
||||
let title = Regex::new(r#"(?s)<span class="track" itemprop="name">\s*(?:<a[^>]*>)?([^<]+)"#).unwrap();
|
||||
let names = Regex::new(r#"<span class="name"(?: itemprop="byArtist")?>([^<]+)</span>"#).unwrap();
|
||||
let ids = Regex::new(r"kunstner/async/(\d+)/").unwrap();
|
||||
let details = Regex::new(r#"(?s)<div class="trackDetails">(.*?)</div>"#).unwrap();
|
||||
let span = Regex::new(r"<span>([^<]*)</span>").unwrap();
|
||||
let tz = chrono_tz::Europe::Copenhagen;
|
||||
let (mut day, mut last_min) = (page_date, -1i32);
|
||||
let mut out = Vec::new();
|
||||
for m in item.captures_iter(html) {
|
||||
let whole = m.get(0).unwrap().as_str();
|
||||
let body = &m[1];
|
||||
let attrs = head.captures(whole).map(|c| c[1].to_string()).unwrap_or_default();
|
||||
let Some(ti) = title.captures(body) else { continue };
|
||||
let Some(track_id) = tid_attr.captures(&attrs).or_else(|| tid_link.captures(body)).map(|c| c[1].to_string()) else { continue };
|
||||
let at = if let Some(t) = full.captures(body) { format!("{}T{}Z", &t[1], &t[2]) } else if let Some(c) = clock.captures(body) {
|
||||
let (h, mi): (u32, u32) = (c[1].parse().unwrap_or(0), c[2].parse().unwrap_or(0));
|
||||
let minute = (h * 60 + mi) as i32;
|
||||
if last_min >= 0 && minute + 6 * 60 < last_min { day = day.succ_opt().unwrap_or(day); }
|
||||
last_min = minute;
|
||||
let Some(local) = day.and_hms_opt(h, mi, 0) else { continue };
|
||||
use chrono::TimeZone;
|
||||
match tz.from_local_datetime(&local).earliest() { Some(d) => d.with_timezone(&Utc).format("%Y-%m-%dT%H:%M:%SZ").to_string(), None => continue }
|
||||
} else { continue };
|
||||
let info = body.find("class=\"trackInfo\"").map(|i| &body[i..]).unwrap_or(body);
|
||||
let info = info.find("trackDetails").map(|i| &info[..i]).unwrap_or(info);
|
||||
let artists: Vec<String> = names.captures_iter(info).map(|c| unescape(c[1].trim())).collect();
|
||||
let artist_ids: Vec<String> = ids.captures_iter(whole).map(|c| c[1].to_string()).collect();
|
||||
let d: Vec<String> = details.captures(body).map(|c| span.captures_iter(&c[1]).map(|s| unescape(s[1].trim())).collect()).unwrap_or_default();
|
||||
let get = |p: &str| d.iter().find(|x| x.starts_with(p)).map(|x| x[p.len()..].trim().to_string());
|
||||
out.push(Play {
|
||||
at, track_id, title: unescape(ti[1].trim()), artists: artists.join(", "), artist_ids: artist_ids.join(","),
|
||||
recorded: get("Indspillet:"), released: get("UDGIVET:"),
|
||||
country: d.iter().find(|x| !x.contains(':') && !x.is_empty()).cloned(),
|
||||
});
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
pub fn collect(db: &Db, http: &Http, log: &mut Vec<String>) -> Result<usize> {
|
||||
create_tables(db)?;
|
||||
let path = Regex::new(r"/playlister/([a-z0-9]+)/(\d{4})-(\d{1,2})-(\d{1,2})")?;
|
||||
let mut pages = 0;
|
||||
for ch in CHANNELS {
|
||||
// The CDX index is often overloaded (503): retry patiently.
|
||||
let url = format!("https://web.archive.org/cdx/search/cdx?url=www.dr.dk/playlister/{ch}/*&output=json&fl=timestamp,original&filter=statuscode:200&collapse=urlkey&limit=50000");
|
||||
let mut cdx = None;
|
||||
for attempt in 0..8 {
|
||||
match http.get(&url) {
|
||||
Ok(Some(c)) if c.trim_start().starts_with('[') => { cdx = Some(c); break; }
|
||||
Ok(Some(_)) | Ok(None) if attempt >= 2 => break, // answered, but not an index: nothing archived
|
||||
Ok(_) => {}
|
||||
Err(e) => { if attempt == 7 { log.push(format!("archive cdx {ch}: {e}")); } }
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_secs(30));
|
||||
}
|
||||
let Some(cdx) = cdx else { continue };
|
||||
eprintln!("archive {ch}: index read");
|
||||
let rows: Vec<Vec<String>> = serde_json::from_str(&cdx).unwrap_or_default();
|
||||
for row in rows.iter().skip(1) {
|
||||
let (ts, url) = (&row[0], &row[1]);
|
||||
if !path.is_match(url) { continue; }
|
||||
let clean = url.replace(":80/", "/").replace("http://", "https://");
|
||||
let done: i64 = db.conn.query_row("SELECT count(*) FROM archive_pages WHERE url = ?1", [&clean], |r| r.get(0))?;
|
||||
if done > 0 { continue; }
|
||||
let html = match http.get(&format!("https://web.archive.org/web/{ts}id_/{url}")) {
|
||||
Ok(h) => h.unwrap_or_default(),
|
||||
Err(e) => { log.push(format!("archive {url}: {e}")); continue; }
|
||||
};
|
||||
let Some(c) = path.captures(url) else { continue };
|
||||
let Some(page_date) = chrono::NaiveDate::from_ymd_opt(c[2].parse().unwrap_or(0), c[3].parse().unwrap_or(0), c[4].parse().unwrap_or(0)) else { continue };
|
||||
let plays = parse(&html, page_date);
|
||||
let tx = db.conn.unchecked_transaction()?;
|
||||
for p in &plays {
|
||||
tx.execute("INSERT OR IGNORE INTO archive_plays VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)",
|
||||
params![ch, p.at, p.track_id, p.artists, p.title, p.artist_ids, clean, p.recorded, p.country, p.released])?;
|
||||
}
|
||||
tx.execute("INSERT OR REPLACE INTO archive_pages VALUES (?1, ?2, ?3, ?4)", params![clean, ts, plays.len() as i64, Utc::now().to_rfc3339()])?;
|
||||
tx.commit()?;
|
||||
pages += 1;
|
||||
}
|
||||
}
|
||||
Ok(pages)
|
||||
}
|
||||
|
||||
/// Archived days per channel and how much of each day is covered (hours with at least one play).
|
||||
#[allow(dead_code)]
|
||||
pub fn coverage(db: &Db) -> Result<BTreeMap<(String, String), usize>> {
|
||||
create_tables(db)?;
|
||||
let mut out = BTreeMap::new();
|
||||
for r in db.conn.prepare("SELECT station, substr(played_at, 1, 10), count(DISTINCT substr(played_at, 12, 2)) FROM archive_plays GROUP BY 1, 2")?
|
||||
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, i64>(2)?)))? {
|
||||
let (s, d, h) = r?;
|
||||
out.insert((s, d), h as usize);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
Reference in New Issue
Block a user