Files
Airplay-Predictor/src/archive.rs
T
2026-11-15 04:21:57 +00:00

157 lines
9.1 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! DR playlists from the Wayback Machine: archived copies of dr.dk/playlister/{channel}/{date}/…
//! (2012–2021, mostly P3 2013–2017). The old pages list each play with DR's track id, the exact
//! UTC time, title, artists and DR artist id, the same ids as today, so archived plays join the
//! artist pool directly. Kept apart from the live `plays` table (sampled days, not continuous).
//!
//! airplay collect --only archive
use crate::{db::Db, http::Http};
use anyhow::Result;
use chrono::Utc;
use regex::Regex;
use rusqlite::params;
use std::collections::BTreeMap;
const CHANNELS: &[&str] = &["p3", "p4kbh", "p4", "p5", "p6beat", "p7mix", "p8jazz", "p4fyn", "p4aarhus", "p4nord", "p4vest", "p4syd",
"p4sjaelland", "p4trekanten", "p4bornholm", "p4esbjerg", "p4ostjylland", "p4midtvest"];
pub fn create_tables(db: &Db) -> Result<()> {
db.conn.execute_batch(
"CREATE TABLE IF NOT EXISTS archive_plays (
station TEXT NOT NULL, played_at TEXT NOT NULL, track_id TEXT NOT NULL,
artist TEXT NOT NULL, title TEXT NOT NULL, artist_ids TEXT NOT NULL DEFAULT '',
page TEXT NOT NULL,
recorded TEXT, country TEXT, released TEXT, -- 2012-13 pages: 'Indspillet', country, 'UDGIVET: year, label'
PRIMARY KEY (station, played_at, track_id)
);
CREATE INDEX IF NOT EXISTS archive_plays_day ON archive_plays (station, substr(played_at, 1, 10));
CREATE TABLE IF NOT EXISTS archive_pages (
url TEXT PRIMARY KEY, snapshot TEXT NOT NULL, plays INTEGER, fetched_at TEXT NOT NULL
);",
)?;
Ok(())
}
fn unescape(s: &str) -> String {
let s = s.replace("&amp;", "&").replace("&quot;", "\"").replace("&#39;", "'").replace("&lt;", "<").replace("&gt;", ">");
// numeric entities (&#248; …)
let re = Regex::new(r"&#(\d+);").unwrap();
re.replace_all(&s, |c: &regex::Captures| char::from_u32(c[1].parse().unwrap_or(63)).map(String::from).unwrap_or_default()).into_owned()
}
pub struct Play { pub at: String, pub track_id: String, pub title: String, pub artists: String, pub artist_ids: String,
pub recorded: Option<String>, pub country: Option<String>, pub released: Option<String> }
/// The plays on one archived page. Two layouts: from about 2014 each item carries its UTC time
/// (data-fulltime) and track id; the 2012–13 pages show the local clock time (the date comes from
/// the URL, rolling over at midnight) and add recording year, country and release.
pub fn parse(html: &str, page_date: chrono::NaiveDate) -> Vec<Play> {
let item = Regex::new(r#"(?s)<li[^>]*itemtype="http://schema.org/MusicRecording"[^>]*>(.*?)</li>"#).unwrap();
let head = Regex::new(r#"(?s)<li([^>]*)itemtype"#).unwrap();
let full = Regex::new(r#"data-fulltime="(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2})Z""#).unwrap();
let clock = Regex::new(r#"<time[^>]*>(\d{2}):(\d{2})</time>"#).unwrap();
let tid_attr = Regex::new(r#"data-trackId="([^"]+)""#).unwrap();
let tid_link = Regex::new(r#"/musik/titel/[^/"]+/(\d+-\d+-\d+)"#).unwrap();
let title = Regex::new(r#"(?s)<span class="track" itemprop="name">\s*(?:<a[^>]*>)?([^<]+)"#).unwrap();
let names = Regex::new(r#"<span class="name"(?: itemprop="byArtist")?>([^<]+)</span>"#).unwrap();
let ids = Regex::new(r"kunstner/async/(\d+)/").unwrap();
let details = Regex::new(r#"(?s)<div class="trackDetails">(.*?)</div>"#).unwrap();
let span = Regex::new(r"<span>([^<]*)</span>").unwrap();
let tz = chrono_tz::Europe::Copenhagen;
let (mut day, mut last_min) = (page_date, -1i32);
let mut out = Vec::new();
for m in item.captures_iter(html) {
let whole = m.get(0).unwrap().as_str();
let body = &m[1];
let attrs = head.captures(whole).map(|c| c[1].to_string()).unwrap_or_default();
let Some(ti) = title.captures(body) else { continue };
let Some(track_id) = tid_attr.captures(&attrs).or_else(|| tid_link.captures(body)).map(|c| c[1].to_string()) else { continue };
let at = if let Some(t) = full.captures(body) { format!("{}T{}Z", &t[1], &t[2]) } else if let Some(c) = clock.captures(body) {
let (h, mi): (u32, u32) = (c[1].parse().unwrap_or(0), c[2].parse().unwrap_or(0));
let minute = (h * 60 + mi) as i32;
if last_min >= 0 && minute + 6 * 60 < last_min { day = day.succ_opt().unwrap_or(day); }
last_min = minute;
let Some(local) = day.and_hms_opt(h, mi, 0) else { continue };
use chrono::TimeZone;
match tz.from_local_datetime(&local).earliest() { Some(d) => d.with_timezone(&Utc).format("%Y-%m-%dT%H:%M:%SZ").to_string(), None => continue }
} else { continue };
let info = body.find("class=\"trackInfo\"").map(|i| &body[i..]).unwrap_or(body);
let info = info.find("trackDetails").map(|i| &info[..i]).unwrap_or(info);
let artists: Vec<String> = names.captures_iter(info).map(|c| unescape(c[1].trim())).collect();
let artist_ids: Vec<String> = ids.captures_iter(whole).map(|c| c[1].to_string()).collect();
let d: Vec<String> = details.captures(body).map(|c| span.captures_iter(&c[1]).map(|s| unescape(s[1].trim())).collect()).unwrap_or_default();
let get = |p: &str| d.iter().find(|x| x.starts_with(p)).map(|x| x[p.len()..].trim().to_string());
out.push(Play {
at, track_id, title: unescape(ti[1].trim()), artists: artists.join(", "), artist_ids: artist_ids.join(","),
recorded: get("Indspillet:"), released: get("UDGIVET:"),
country: d.iter().find(|x| !x.contains(':') && !x.is_empty()).cloned(),
});
}
out
}
pub fn collect(db: &Db, http: &Http, log: &mut Vec<String>) -> Result<usize> {
create_tables(db)?;
let path = Regex::new(r"/playlister/([a-z0-9]+)/(\d{4})-(\d{1,2})-(\d{1,2})")?;
// Pages that gave no plays get another try each run (an archive outage looks like an empty page).
db.conn.execute("DELETE FROM archive_pages WHERE plays = 0", [])?;
let mut pages = 0;
for ch in CHANNELS {
// The CDX index is often overloaded (503): retry patiently.
let url = format!("https://web.archive.org/cdx/search/cdx?url=www.dr.dk/playlister/{ch}/*&output=json&fl=timestamp,original&filter=statuscode:200&collapse=urlkey&limit=50000");
let mut cdx = None;
for attempt in 0..8 {
match http.get(&url) {
Ok(Some(c)) if c.trim_start().starts_with('[') => { cdx = Some(c); break; }
Ok(Some(_)) | Ok(None) if attempt >= 2 => break, // answered, but not an index: nothing archived
Ok(_) => {}
Err(e) => { if attempt == 7 { log.push(format!("archive cdx {ch}: {e}")); } }
}
std::thread::sleep(std::time::Duration::from_secs(30));
}
let Some(cdx) = cdx else { continue };
eprintln!("archive {ch}: index read");
let rows: Vec<Vec<String>> = serde_json::from_str(&cdx).unwrap_or_default();
for row in rows.iter().skip(1) {
let (ts, url) = (&row[0], &row[1]);
if !path.is_match(url) { continue; }
let clean = url.replace(":80/", "/").replace("http://", "https://");
let done: i64 = db.conn.query_row("SELECT count(*) FROM archive_pages WHERE url = ?1", [&clean], |r| r.get(0))?;
if done > 0 { continue; }
let html = match http.get(&format!("https://web.archive.org/web/{ts}id_/{url}")) {
Ok(h) => h.unwrap_or_default(),
Err(e) => { log.push(format!("archive {url}: {e}")); continue; }
};
if html.is_empty() || html.contains("Internet Archive: Temporarily Offline") {
std::thread::sleep(std::time::Duration::from_secs(60));
continue;
}
let Some(c) = path.captures(url) else { continue };
let Some(page_date) = chrono::NaiveDate::from_ymd_opt(c[2].parse().unwrap_or(0), c[3].parse().unwrap_or(0), c[4].parse().unwrap_or(0)) else { continue };
let plays = parse(&html, page_date);
let tx = db.conn.unchecked_transaction()?;
for p in &plays {
tx.execute("INSERT OR IGNORE INTO archive_plays VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)",
params![ch, p.at, p.track_id, p.artists, p.title, p.artist_ids, clean, p.recorded, p.country, p.released])?;
}
tx.execute("INSERT OR REPLACE INTO archive_pages VALUES (?1, ?2, ?3, ?4)", params![clean, ts, plays.len() as i64, Utc::now().to_rfc3339()])?;
tx.commit()?;
pages += 1;
}
}
Ok(pages)
}
/// Archived days per channel and how much of each day is covered (hours with at least one play).
#[allow(dead_code)]
pub fn coverage(db: &Db) -> Result<BTreeMap<(String, String), usize>> {
create_tables(db)?;
let mut out = BTreeMap::new();
for r in db.conn.prepare("SELECT station, substr(played_at, 1, 10), count(DISTINCT substr(played_at, 12, 2)) FROM archive_plays GROUP BY 1, 2")?
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, i64>(2)?)))? {
let (s, d, h) = r?;
out.insert((s, d), h as usize);
}
Ok(out)
}