From 6e8b258dcf8b6f81a4e609d52c5748eb8ed7146e Mon Sep 17 00:00:00 2001 From: moritsbw-cpu Date: Sun, 15 Nov 2026 01:04:36 +0000 Subject: [PATCH] Hitlisten's Airplay Top-20 since 2001; --backfill for all history at once New entries and re-entries (NY/RE) are kept; a week without exactly 20 rows is logged. Co-Authored-By: Claude Opus 5.5 --- src/hitlisten.rs | 101 +++++++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 25 +++++++++--- 2 files changed, 121 insertions(+), 5 deletions(-) create mode 100644 src/hitlisten.rs diff --git a/src/hitlisten.rs b/src/hitlisten.rs new file mode 100644 index 0000000..d45bc4b --- /dev/null +++ b/src/hitlisten.rs @@ -0,0 +1,101 @@ +//! Hitlisten (IFPI Danmark's official charts): the weekly Airplay Top-20, the most-played songs +//! on the biggest Danish radio stations (monitored by M&I Service), every week since week 38 of 2001. +//! +//! https://www.hitlisten.nu/default.asp?w={week}&y={year}&list=airplay +//! +//! The page's week selector lists every week that exists; weeks already stored aren't fetched again. + +use crate::{db::Db, http::Http}; +use anyhow::Result; +use regex::Regex; +use rusqlite::params; + +const BASE: &str = "https://www.hitlisten.nu/default.asp"; + +pub fn create_tables(db: &Db) -> Result<()> { + db.conn.execute_batch( + "CREATE TABLE IF NOT EXISTS hitlisten_airplay ( + year INTEGER NOT NULL, week INTEGER NOT NULL, position INTEGER NOT NULL, + last_week TEXT, -- last week's position, 'NY' (new), 'RE' (re-entry) or empty + weeks_on_chart INTEGER, peak INTEGER, + artist TEXT NOT NULL, title TEXT NOT NULL, label TEXT, + PRIMARY KEY (year, week, position) + );", + )?; + Ok(()) +} + +fn text(s: &str) -> String { + let s = s.replace("&", "&").replace("´", "ยด").replace(" ", " "); + s.split_whitespace().collect::>().join(" ") +} + +pub struct Row { + pub position: i64, + pub last_week: String, // last week's position, "NY" (new), "RE" (re-entry) or empty + pub weeks_on_chart: Option, + pub peak: Option, + pub artist: String, + pub title: String, + pub label: String, +} + +/// The chart rows of one week's page. Each row is a
block; the fields +/// are read one by one so markup variants (re-entries, bold NY/RE) don't drop rows. +pub fn parse(html: &str) -> Vec { + let tags = Regex::new(r"<[^>]+>").unwrap(); + let field = |block: &str, re: &str| -> String { + Regex::new(re).unwrap().captures(block) + .map(|c| text(&tags.replace_all(&c[1], " "))).unwrap_or_default() + }; + html.split(r#"
"#) + .skip(1) + .filter_map(|b| { + let position = field(b, r#"(?s)
]*>(.*?)
"#).parse().ok()?; + let stat = |id: &str| field(b, &format!(r#"(?s)
]*>\s*
\d
(.*?)
"#)); + Some(Row { + position, + last_week: stat("sidsteuge"), + weeks_on_chart: stat("ugerialt").parse().ok(), + peak: stat("peak").parse().ok(), + artist: field(b, r#"(?s)
(.*?)
"#), + title: field(b, r#"(?s)
(.*?)
"#), + label: field(b, r#"(?s)
(.*?)
"#), + }) + }) + .filter(|r| !r.artist.is_empty() || !r.title.is_empty()) + .collect() +} + +/// Every week in the archive that isn't stored yet. `limit` caps the pages per run. +pub fn collect(db: &Db, http: &Http, limit: usize, log: &mut Vec) -> Result { + create_tables(db)?; + let first = http.get(&format!("{BASE}?list=airplay"))?.unwrap_or_default(); + let weeks: Vec<(i64, i64)> = Regex::new(r"w=(\d+)&y=(\d+)&list=airplay")? + .captures_iter(&first) + .filter_map(|c| Some((c[2].parse().ok()?, c[1].parse().ok()?))) + .collect(); + let mut stored = db.conn.prepare("SELECT count(*) FROM hitlisten_airplay WHERE year = ?1 AND week = ?2")?; + let mut fetched = 0; + for (year, week) in weeks { + if fetched >= limit { break; } + if stored.query_row(params![year, week], |r| r.get::<_, i64>(0))? > 0 { continue; } + let html = match http.get_bytes(&format!("{BASE}?w={week}&y={year}&list=airplay")) { + // The site is Latin-1. + Ok(Some(b)) => b.iter().map(|&c| c as char).collect::(), + Ok(None) => continue, + Err(e) => { log.push(format!("hitlisten {year}-{week}: {e}")); continue; } + }; + let rows = parse(&html); + if rows.is_empty() { log.push(format!("hitlisten {year}-{week}: no rows")); continue; } + if rows.len() != 20 { log.push(format!("hitlisten {year}-{week}: {} rows, not 20", rows.len())); } + let tx = db.conn.unchecked_transaction()?; + for r in rows { + tx.execute("INSERT OR REPLACE INTO hitlisten_airplay VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", + params![year, week, r.position, r.last_week, r.weeks_on_chart, r.peak, r.artist, r.title, r.label])?; + } + tx.commit()?; + fetched += 1; + } + Ok(fetched) +} diff --git a/src/main.rs b/src/main.rs index 0477d96..3f0c17c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,11 +1,15 @@ //! airplay: collects what Danish radio stations play, for modelling when artists get played again. //! -//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer] +//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten] [--backfill] //! airplay stats [--db data/airplay.db] +//! +//! --backfill fetches all history at once (every DR artist page, the whole Hitlisten archive); +//! without it a run takes a bounded slice, which is what the daily timer does. mod bauer; mod db; mod dr; +mod hitlisten; mod http; use anyhow::Result; @@ -27,7 +31,7 @@ fn main() -> Result<()> { Some("collect") => collect(&db, &args, &db_path), Some("stats") => stats(&db), _ => { - eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer]\n airplay stats [--db FILE]"); + eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten] [--backfill]\n airplay stats [--db FILE]"); std::process::exit(2); } } @@ -36,6 +40,8 @@ fn main() -> Result<()> { fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> { let days: i64 = arg(args, "--days").and_then(|d| d.parse().ok()).unwrap_or(7); let only = arg(args, "--only"); + let wants = |source: &str| only.as_deref().map_or(true, |o| o == source); + let backfill = args.iter().any(|a| a == "--backfill"); let http = http::Http::new(400)?; let started = Utc::now().to_rfc3339(); db.conn.execute("INSERT INTO runs (started_at) VALUES (?1)", params![started])?; @@ -47,17 +53,17 @@ fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> { Err(e) => { eprintln!("{name}: {e}"); log.push(format!("{name}: {e}")); } }; - if only.as_deref() != Some("bauer") { + if wants("dr") { let r = dr::collect_plays(db, &http, days, &mut log); if let Ok(n) = &r { new += n; } step("dr plays", r.map(|n| format!("{n} new")), &mut log); let dir = std::path::Path::new(db_path).parent().unwrap_or(std::path::Path::new(".")).join("rotations"); let r = dr::collect_rotation_pdf(db, &http, &dir); step("dr rotation list", r.map(|f| f.map(|f| format!("saved {f}")).unwrap_or_else(|| "unchanged".into())), &mut log); - let r = dr::collect_artists(db, &http, 6, 400, &mut log); + let r = dr::collect_artists(db, &http, 6, if backfill { usize::MAX } else { 400 }, &mut log); step("dr artist pages", r.map(|n| format!("{n} fetched")), &mut log); } - if only.as_deref() != Some("dr") { + if wants("bauer") { match bauer::stations(db, &http, args.iter().any(|a| a == "--all-bauer")) { Ok(stations) => { for (code, name) in stations { @@ -69,6 +75,10 @@ fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> { Err(e) => step("bauer stations", Err(e), &mut log), } } + if wants("hitlisten") { + let r = hitlisten::collect(db, &http, if backfill { usize::MAX } else { 60 }, &mut log); + step("hitlisten airplay chart", r.map(|n| format!("{n} weeks fetched")), &mut log); + } db.conn.execute( "UPDATE runs SET finished_at = ?1, new_plays = ?2, errors = ?3, log = ?4 WHERE id = ?5", params![Utc::now().to_rfc3339(), new as i64, log.len() as i64, log.join("\n"), run], @@ -93,5 +103,10 @@ fn stats(db: &db::Db) -> Result<()> { let artists: i64 = db.conn.query_row("SELECT count(*) FROM dr_artists", [], |r| r.get(0))?; let pdfs: i64 = db.conn.query_row("SELECT count(*) FROM dr_rotation_pdfs", [], |r| r.get(0))?; println!("DR artist history: {artists} artists, {weeks} artist-week-channel rows; rotation PDFs: {pdfs}"); + hitlisten::create_tables(db)?; + let (n, first, last): (i64, Option, Option) = db.conn.query_row( + "SELECT count(DISTINCT year * 100 + week), min(printf('%d-W%02d', year, week)), max(printf('%d-W%02d', year, week)) FROM hitlisten_airplay", + [], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?; + println!("Hitlisten airplay chart: {n} weeks ({} to {})", first.unwrap_or_default(), last.unwrap_or_default()); Ok(()) }