Hitlisten's Airplay Top-20 since 2001; --backfill for all history at once

New entries and re-entries (NY/RE) are kept; a week without exactly 20 rows is logged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-15 01:04:36 +00:00
co-authored by Claude Opus 5.5
parent 411b55fb9f
commit 6e8b258dcf
2 changed files with 121 additions and 5 deletions
+101
View File
@@ -0,0 +1,101 @@
//! Hitlisten (IFPI Danmark's official charts): the weekly Airplay Top-20, the most-played songs
//! on the biggest Danish radio stations (monitored by M&I Service), every week since week 38 of 2001.
//!
//! https://www.hitlisten.nu/default.asp?w={week}&y={year}&list=airplay
//!
//! The page's week selector lists every week that exists; weeks already stored aren't fetched again.
use crate::{db::Db, http::Http};
use anyhow::Result;
use regex::Regex;
use rusqlite::params;
const BASE: &str = "https://www.hitlisten.nu/default.asp";
pub fn create_tables(db: &Db) -> Result<()> {
db.conn.execute_batch(
"CREATE TABLE IF NOT EXISTS hitlisten_airplay (
year INTEGER NOT NULL, week INTEGER NOT NULL, position INTEGER NOT NULL,
last_week TEXT, -- last week's position, 'NY' (new), 'RE' (re-entry) or empty
weeks_on_chart INTEGER, peak INTEGER,
artist TEXT NOT NULL, title TEXT NOT NULL, label TEXT,
PRIMARY KEY (year, week, position)
);",
)?;
Ok(())
}
fn text(s: &str) -> String {
let s = s.replace("&amp;", "&").replace("&#180;", "´").replace("&nbsp;", " ");
s.split_whitespace().collect::<Vec<_>>().join(" ")
}
pub struct Row {
pub position: i64,
pub last_week: String, // last week's position, "NY" (new), "RE" (re-entry) or empty
pub weeks_on_chart: Option<i64>,
pub peak: Option<i64>,
pub artist: String,
pub title: String,
pub label: String,
}
/// The chart rows of one week's page. Each row is a <div id="liniesmall"> block; the fields
/// are read one by one so markup variants (re-entries, bold NY/RE) don't drop rows.
pub fn parse(html: &str) -> Vec<Row> {
let tags = Regex::new(r"<[^>]+>").unwrap();
let field = |block: &str, re: &str| -> String {
Regex::new(re).unwrap().captures(block)
.map(|c| text(&tags.replace_all(&c[1], " "))).unwrap_or_default()
};
html.split(r#"<div id="liniesmall">"#)
.skip(1)
.filter_map(|b| {
let position = field(b, r#"(?s)<div id="denneuge[a-z]*"[^>]*>(.*?)</div>"#).parse().ok()?;
let stat = |id: &str| field(b, &format!(r#"(?s)<div id="{id}"[^>]*>\s*<div id="listeikon">\d</div>(.*?)</div>"#));
Some(Row {
position,
last_week: stat("sidsteuge"),
weeks_on_chart: stat("ugerialt").parse().ok(),
peak: stat("peak").parse().ok(),
artist: field(b, r#"(?s)<div id="artistnavnsmall">(.*?)</div>"#),
title: field(b, r#"(?s)<div id="titelbred">(.*?)</div>"#),
label: field(b, r#"(?s)<div id="selskab">(.*?)</div>"#),
})
})
.filter(|r| !r.artist.is_empty() || !r.title.is_empty())
.collect()
}
/// Every week in the archive that isn't stored yet. `limit` caps the pages per run.
pub fn collect(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) -> Result<usize> {
create_tables(db)?;
let first = http.get(&format!("{BASE}?list=airplay"))?.unwrap_or_default();
let weeks: Vec<(i64, i64)> = Regex::new(r"w=(\d+)&y=(\d+)&list=airplay")?
.captures_iter(&first)
.filter_map(|c| Some((c[2].parse().ok()?, c[1].parse().ok()?)))
.collect();
let mut stored = db.conn.prepare("SELECT count(*) FROM hitlisten_airplay WHERE year = ?1 AND week = ?2")?;
let mut fetched = 0;
for (year, week) in weeks {
if fetched >= limit { break; }
if stored.query_row(params![year, week], |r| r.get::<_, i64>(0))? > 0 { continue; }
let html = match http.get_bytes(&format!("{BASE}?w={week}&y={year}&list=airplay")) {
// The site is Latin-1.
Ok(Some(b)) => b.iter().map(|&c| c as char).collect::<String>(),
Ok(None) => continue,
Err(e) => { log.push(format!("hitlisten {year}-{week}: {e}")); continue; }
};
let rows = parse(&html);
if rows.is_empty() { log.push(format!("hitlisten {year}-{week}: no rows")); continue; }
if rows.len() != 20 { log.push(format!("hitlisten {year}-{week}: {} rows, not 20", rows.len())); }
let tx = db.conn.unchecked_transaction()?;
for r in rows {
tx.execute("INSERT OR REPLACE INTO hitlisten_airplay VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)",
params![year, week, r.position, r.last_week, r.weeks_on_chart, r.peak, r.artist, r.title, r.label])?;
}
tx.commit()?;
fetched += 1;
}
Ok(fetched)
}
+20 -5
View File
@@ -1,11 +1,15 @@
//! airplay: collects what Danish radio stations play, for modelling when artists get played again.
//!
//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer]
//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten] [--backfill]
//! airplay stats [--db data/airplay.db]
//!
//! --backfill fetches all history at once (every DR artist page, the whole Hitlisten archive);
//! without it a run takes a bounded slice, which is what the daily timer does.
mod bauer;
mod db;
mod dr;
mod hitlisten;
mod http;
use anyhow::Result;
@@ -27,7 +31,7 @@ fn main() -> Result<()> {
Some("collect") => collect(&db, &args, &db_path),
Some("stats") => stats(&db),
_ => {
eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer]\n airplay stats [--db FILE]");
eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten] [--backfill]\n airplay stats [--db FILE]");
std::process::exit(2);
}
}
@@ -36,6 +40,8 @@ fn main() -> Result<()> {
fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> {
let days: i64 = arg(args, "--days").and_then(|d| d.parse().ok()).unwrap_or(7);
let only = arg(args, "--only");
let wants = |source: &str| only.as_deref().map_or(true, |o| o == source);
let backfill = args.iter().any(|a| a == "--backfill");
let http = http::Http::new(400)?;
let started = Utc::now().to_rfc3339();
db.conn.execute("INSERT INTO runs (started_at) VALUES (?1)", params![started])?;
@@ -47,17 +53,17 @@ fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> {
Err(e) => { eprintln!("{name}: {e}"); log.push(format!("{name}: {e}")); }
};
if only.as_deref() != Some("bauer") {
if wants("dr") {
let r = dr::collect_plays(db, &http, days, &mut log);
if let Ok(n) = &r { new += n; }
step("dr plays", r.map(|n| format!("{n} new")), &mut log);
let dir = std::path::Path::new(db_path).parent().unwrap_or(std::path::Path::new(".")).join("rotations");
let r = dr::collect_rotation_pdf(db, &http, &dir);
step("dr rotation list", r.map(|f| f.map(|f| format!("saved {f}")).unwrap_or_else(|| "unchanged".into())), &mut log);
let r = dr::collect_artists(db, &http, 6, 400, &mut log);
let r = dr::collect_artists(db, &http, 6, if backfill { usize::MAX } else { 400 }, &mut log);
step("dr artist pages", r.map(|n| format!("{n} fetched")), &mut log);
}
if only.as_deref() != Some("dr") {
if wants("bauer") {
match bauer::stations(db, &http, args.iter().any(|a| a == "--all-bauer")) {
Ok(stations) => {
for (code, name) in stations {
@@ -69,6 +75,10 @@ fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> {
Err(e) => step("bauer stations", Err(e), &mut log),
}
}
if wants("hitlisten") {
let r = hitlisten::collect(db, &http, if backfill { usize::MAX } else { 60 }, &mut log);
step("hitlisten airplay chart", r.map(|n| format!("{n} weeks fetched")), &mut log);
}
db.conn.execute(
"UPDATE runs SET finished_at = ?1, new_plays = ?2, errors = ?3, log = ?4 WHERE id = ?5",
params![Utc::now().to_rfc3339(), new as i64, log.len() as i64, log.join("\n"), run],
@@ -93,5 +103,10 @@ fn stats(db: &db::Db) -> Result<()> {
let artists: i64 = db.conn.query_row("SELECT count(*) FROM dr_artists", [], |r| r.get(0))?;
let pdfs: i64 = db.conn.query_row("SELECT count(*) FROM dr_rotation_pdfs", [], |r| r.get(0))?;
println!("DR artist history: {artists} artists, {weeks} artist-week-channel rows; rotation PDFs: {pdfs}");
hitlisten::create_tables(db)?;
let (n, first, last): (i64, Option<String>, Option<String>) = db.conn.query_row(
"SELECT count(DISTINCT year * 100 + week), min(printf('%d-W%02d', year, week)), max(printf('%d-W%02d', year, week)) FROM hitlisten_airplay",
[], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?;
println!("Hitlisten airplay chart: {n} weeks ({} to {})", first.unwrap_or_default(), last.unwrap_or_default());
Ok(())
}