Collector for Wayback copies of DR's current artist pages: per-channel weeks 2023-

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-19 16:29:56 +00:00
co-authored by Claude Opus 5.5
parent 1ff6b4bdd6
commit 7e340a074f
2 changed files with 167 additions and 2 deletions
+159
View File
@@ -0,0 +1,159 @@
//! Wayback copies of DR's current artist pages (dr.dk/lyd/musik/kunstner/{slug}-{id}, archived
//! since mid-2024). Each copy holds the artist's last 53 weeks of plays per channel, so they give
//! per-channel history from about autumn 2023 up to the live panel, including the 2024-25 gap
//! between the old pages (2019-24, totals only) and the panel.
//!
//! airplay collect --only archpages
//!
//! Tables: arch_artist_captures (one row per archived copy; chosen ones are read, and first_week /
//! last_week record which weeks the copy covers, so missing weeks can be told from zero plays),
//! dr_artist_weeks_arch (artist, week, channel, plays; later copies win), arch_artist_meta.
use crate::{db::Db, http::Http};
use anyhow::{anyhow, Result};
use chrono::{Duration, NaiveDate, Utc};
use regex::Regex;
use rusqlite::params;
use serde_json::Value;
use std::collections::BTreeMap;
pub fn create_tables(db: &Db) -> Result<()> {
db.conn.execute_batch(
"CREATE TABLE IF NOT EXISTS arch_artist_captures (
artist_id TEXT NOT NULL, snapshot TEXT NOT NULL, url TEXT NOT NULL,
chosen INTEGER NOT NULL DEFAULT 0, fetched_at TEXT, weeks INTEGER,
first_week TEXT, last_week TEXT,
PRIMARY KEY (artist_id, snapshot)
);
CREATE TABLE IF NOT EXISTS arch_cdx_progress (key TEXT PRIMARY KEY, value TEXT);
CREATE TABLE IF NOT EXISTS dr_artist_weeks_arch (
artist_id TEXT NOT NULL, week_start TEXT NOT NULL, channel TEXT NOT NULL, plays INTEGER NOT NULL,
capture TEXT NOT NULL,
PRIMARY KEY (artist_id, week_start, channel)
);
CREATE TABLE IF NOT EXISTS arch_artist_meta (artist_id TEXT PRIMARY KEY, name TEXT, capture TEXT NOT NULL, raw TEXT);",
)?;
Ok(())
}
fn urlencode(s: &str) -> String {
s.bytes().map(|b| match b { b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => (b as char).to_string(), _ => format!("%{b:02X}") }).collect()
}
/// Lists every archived copy into arch_artist_captures (resumable).
fn list_captures(db: &Db, http: &Http, log: &mut Vec<String>) -> Result<()> {
let done: i64 = db.conn.query_row("SELECT count(*) FROM arch_cdx_progress WHERE key = 'complete'", [], |r| r.get(0))?;
if done > 0 { return Ok(()); }
let id_re = Regex::new(r"/kunstner/(?:[^/?#]*-)?(\d+)/?(?:[?#].*)?$")?;
let mut resume: Option<String> = db.conn.query_row("SELECT value FROM arch_cdx_progress WHERE key = 'resume'", [], |r| r.get(0)).ok();
loop {
let mut url = "https://web.archive.org/cdx/search/cdx?url=www.dr.dk/lyd/musik/kunstner/*&output=json&fl=timestamp,original&filter=statuscode:200&limit=20000&showResumeKey=true".to_string();
if let Some(k) = &resume { url.push_str(&format!("&resumeKey={}", urlencode(k))); }
let mut body = None;
for attempt in 0..10 {
match http.get(&url) {
Ok(Some(b)) if b.trim_start().starts_with('[') => { body = Some(b); break; }
_ => { if attempt == 9 { log.push("archived artist pages: CDX listing failed".into()); } }
}
std::thread::sleep(std::time::Duration::from_secs(30));
}
let Some(body) = body else { return Err(anyhow!("CDX listing failed")) };
let rows: Vec<Vec<String>> = serde_json::from_str(&body).unwrap_or_default();
let mut next: Option<String> = None;
let tx = db.conn.unchecked_transaction()?;
for (k, row) in rows.iter().enumerate().skip(1) {
if row.is_empty() { next = rows.get(k + 1).and_then(|r| r.first()).cloned(); break; }
if row.len() < 2 { continue; }
if let Some(c) = id_re.captures(&row[1]) {
tx.execute("INSERT OR IGNORE INTO arch_artist_captures (artist_id, snapshot, url) VALUES (?1, ?2, ?3)", params![&c[1], &row[0], &row[1]])?;
}
}
match &next {
Some(k) => { tx.execute("INSERT OR REPLACE INTO arch_cdx_progress VALUES ('resume', ?1)", [k])?; }
None => { tx.execute("INSERT OR REPLACE INTO arch_cdx_progress VALUES ('complete', '1')", [])?; }
}
tx.commit()?;
match next { Some(k) => resume = Some(k), None => break }
}
Ok(())
}
/// Per artist: the oldest copy, then each next one at least 44 weeks later (each covers 53 weeks),
/// and the last one before the live panel began, so the 2024-25 gap is covered up to the panel.
fn choose(db: &Db) -> Result<()> {
let panel_start: Option<String> = db.conn.query_row("SELECT min(week_start) FROM dr_artist_weeks", [], |r| r.get(0)).ok().flatten();
let cutoff = panel_start.as_deref().and_then(|s| NaiveDate::parse_from_str(s, "%Y-%m-%d").ok()).map(|d| d + Duration::weeks(2));
let mut by: BTreeMap<String, Vec<String>> = BTreeMap::new();
for r in db.conn.prepare("SELECT artist_id, snapshot FROM arch_artist_captures ORDER BY artist_id, snapshot")?.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? {
let (a, s) = r?;
by.entry(a).or_default().push(s);
}
let date = |s: &str| NaiveDate::parse_from_str(&s[..8], "%Y%m%d").ok();
let tx = db.conn.unchecked_transaction()?;
for (a, snaps) in by {
let mut pick: Vec<&String> = Vec::new();
let mut last: Option<NaiveDate> = None;
for s in &snaps {
let Some(d) = date(s) else { continue };
if last.is_none_or(|l| (d - l).num_weeks() >= 44) { pick.push(s); last = Some(d); }
}
if let Some(c) = cutoff {
if let Some(s) = snaps.iter().rev().find(|s| date(s).is_some_and(|d| d <= c)) {
if !pick.iter().any(|p| date(p).zip(date(s)).is_some_and(|(x, y)| (y - x).num_weeks().abs() < 8)) { pick.push(s); }
}
}
for s in pick { tx.execute("UPDATE arch_artist_captures SET chosen = 1 WHERE artist_id = ?1 AND snapshot = ?2", params![a, s])?; }
}
tx.commit()?;
Ok(())
}
pub fn collect(db: &Db, http: &Http, log: &mut Vec<String>) -> Result<usize> {
create_tables(db)?;
list_captures(db, http, log)?;
choose(db)?;
// Artists in today's pool first, then the rest; oldest copies first.
let todo: Vec<(String, String, String)> = db.conn.prepare(
"SELECT c.artist_id, c.snapshot, c.url FROM arch_artist_captures c LEFT JOIN dr_artists a ON a.artist_id = c.artist_id
WHERE c.chosen = 1 AND c.fetched_at IS NULL ORDER BY a.artist_id IS NULL, c.artist_id, c.snapshot")?
.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?.collect::<Result<_, _>>()?;
eprintln!("archived artist pages: {} copies to read", todo.len());
let next_re = Regex::new(r#"(?s)<script id="__NEXT_DATA__"[^>]*>(.*?)</script>"#)?;
let mut pages = 0;
for (a, snap, url) in todo {
let html = match http.get(&format!("https://web.archive.org/web/{snap}id_/{url}")) {
Ok(h) => h.unwrap_or_default(),
Err(e) => { log.push(format!("archived page {url}: {e}")); continue; }
};
if html.contains("Internet Archive: Temporarily Offline") || html.contains("Too Many Requests") { std::thread::sleep(std::time::Duration::from_secs(120)); continue; }
let props: Value = next_re.captures(&html).and_then(|c| serde_json::from_str::<Value>(&c[1]).ok()).map(|v| v["props"]["pageProps"].clone()).unwrap_or(Value::Null);
let Some(day) = NaiveDate::parse_from_str(&snap[..8], "%Y%m%d").ok() else { continue };
let (mut n, mut first, mut lastw): (i64, Option<String>, Option<String>) = (0, None, None);
let tx = db.conn.unchecked_transaction()?;
for w in props["weekPlaySums"].as_array().cloned().unwrap_or_default() {
let Some(ws) = w["date"].as_str().and_then(|d| NaiveDate::parse_from_str(&d[..d.len().min(10)], "%Y-%m-%d").ok()) else { continue };
// The week the copy was taken in is incomplete.
if ws + Duration::days(7) > day { continue; }
let wk = ws.to_string();
for c in w["statisticsByChannels"].as_array().cloned().unwrap_or_default() {
tx.execute("INSERT INTO dr_artist_weeks_arch VALUES (?1, ?2, ?3, ?4, ?5)
ON CONFLICT (artist_id, week_start, channel) DO UPDATE SET plays = excluded.plays, capture = excluded.capture WHERE excluded.capture > capture",
params![a, wk, c["channel"]["slug"].as_str().unwrap_or_default(), c["count"].as_i64().unwrap_or(0), snap])?;
}
if first.as_ref().is_none_or(|f| wk < *f) { first = Some(wk.clone()); }
if lastw.as_ref().is_none_or(|l| wk > *l) { lastw = Some(wk.clone()); }
n += 1;
}
if n > 0 {
tx.execute("INSERT INTO arch_artist_meta VALUES (?1, ?2, ?3, ?4)
ON CONFLICT (artist_id) DO UPDATE SET name = excluded.name, capture = excluded.capture, raw = excluded.raw WHERE excluded.capture > capture",
params![a, props["artist"]["displayName"].as_str(), snap, props["artist"].to_string()])?;
}
tx.execute("UPDATE arch_artist_captures SET fetched_at = ?1, weeks = ?2, first_week = ?3, last_week = ?4 WHERE artist_id = ?5 AND snapshot = ?6",
params![Utc::now().to_rfc3339(), n, first, lastw, a, snap])?;
tx.commit()?;
pages += 1;
if pages % 200 == 0 { eprintln!("archived artist pages: {pages} read"); }
}
Ok(pages)
}
+8 -2
View File
@@ -1,6 +1,6 @@
//! airplay: collects what Danish radio stations play, for modelling when artists get played again.
//!
//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|spotify-api|itunes|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive|oldpages] [--backfill]
//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|spotify-api|itunes|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive|oldpages|archpages] [--backfill]
//! airplay export nmf-denmark [--out data/nmf_denmark.csv]
//! airplay stats [--db data/airplay.db]
//! airplay check [--db data/airplay.db] gaps, duplicates, DR cross-check, chart completeness
@@ -23,6 +23,7 @@ mod http;
mod model;
mod musicbrainz;
mod oldpages;
mod archpages;
mod playlists;
mod rotations;
mod spotify;
@@ -64,7 +65,7 @@ fn main() -> Result<()> {
rotations::store_all(&db, &dir, args.iter().any(|a| a == "--reparse"))
}
_ => {
eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|spotify-api|itunes|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive|oldpages] [--backfill]
eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|spotify-api|itunes|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive|oldpages|archpages] [--backfill]
//! airplay export nmf-denmark [--out data/nmf_denmark.csv]\n airplay stats [--db FILE]\n airplay check [--db FILE]\n airplay model panel|backtest|rank\n airplay serve [--addr HOST:PORT] [--prefix /airplay]");
std::process::exit(2);
}
@@ -156,6 +157,11 @@ fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> {
let r = playlists::nmf_artists(db, &sfc_http, if backfill { usize::MAX } else { 800 }, &mut log);
step("new music friday per artist (superfridaychart)", r.map(|n| format!("{n} artists looked up")), &mut log);
}
if only.as_deref() == Some("archpages") {
// Wayback copies of DR's current artist pages (2024-): per-channel weeks (only on request; once).
let r = archpages::collect(db, &http::Http::new(1500)?, &mut log);
step("archived dr artist pages, per channel (wayback)", r.map(|n| format!("{n} pages read")), &mut log);
}
if only.as_deref() == Some("oldpages") {
// Wayback copies of DR's old artist pages: 53 weeks of plays each (only on request; long, once).
let r = oldpages::collect(db, &http::Http::new(1500)?, &mut log);