Collector for DR and Bauer airplay; data sources and modelling plan

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-15 00:42:05 +00:00
co-authored by Claude Opus 5.5
commit 6e4d2f1585
13 changed files with 2329 additions and 0 deletions
+76
View File
@@ -0,0 +1,76 @@
//! Bauer Media Denmark (NOVA, Radio Soft, The Voice, Pop FM, Radio 100 and their web stations),
//! through the API behind rayo.dk (listenapi.planetradio.co.uk, shared with Bauer's other countries).
//!
//! GET /api9.2/stations/DK every Danish station: stationCode, stationName
//! GET /api9.2/events/{code}/{local time}/{n} the n (<= 100) plays before that time, newest first
//!
//! Times are Danish local time. The API answers ["invalid date"] for anything older than about
//! 7 days, so the history has to be collected continuously.
use crate::{db::{Db, Play}, http::Http};
use anyhow::Result;
use chrono::{Duration, NaiveDateTime, TimeZone, Utc};
use chrono_tz::Europe::Copenhagen;
use serde_json::Value;
const BASE: &str = "https://listenapi.planetradio.co.uk/api9.2";
/// The broadcast (FM/DAB) stations; with `all`, Bauer's web-only stations too.
pub const MAIN: &[&str] = &["nov", "dso", "the", "pop", "rhu"];
pub fn stations(db: &Db, http: &Http, all: bool) -> Result<Vec<(String, String)>> {
let body = http.get(&format!("{BASE}/stations/DK"))?.unwrap_or_default();
let list: Value = serde_json::from_str(&body)?;
let mut out = Vec::new();
for s in list.as_array().cloned().unwrap_or_default() {
let (Some(code), Some(name)) = (s["stationCode"].as_str(), s["stationName"].as_str()) else { continue };
if all || MAIN.contains(&code) {
db.add_station("bauer", code, name)?;
out.push((code.to_string(), name.to_string()));
}
}
Ok(out)
}
fn local_to_utc(t: &str) -> Option<String> {
let naive = NaiveDateTime::parse_from_str(t, "%Y-%m-%d %H:%M:%S").ok()?;
let local = Copenhagen.from_local_datetime(&naive).earliest()?;
Some(local.with_timezone(&Utc).format("%Y-%m-%dT%H:%M:%SZ").to_string())
}
/// Walks back from now in pages of 100 until it reaches plays it already has, or the API's
/// window ends (`days` at most).
pub fn collect(db: &Db, http: &Http, code: &str, days: i64, log: &mut Vec<String>) -> Result<usize> {
let floor = db.latest_play(code)?
.unwrap_or_else(|| (Utc::now() - Duration::days(days)).format("%Y-%m-%dT%H:%M:%SZ").to_string());
let mut before = Utc::now().with_timezone(&Copenhagen).naive_local();
let mut new = 0;
for _page in 0..200 {
let url = format!("{BASE}/events/{code}/{}/100", before.format("%Y-%m-%d%%20%H:%M:%S"));
let body = match http.get(&url) {
Ok(Some(b)) => b,
Ok(None) => break,
Err(e) => { log.push(format!("bauer {code}: {e}")); break; }
};
let page: Value = serde_json::from_str(&body).unwrap_or(Value::Null);
let Some(events) = page.as_array().filter(|a| a.first().map(|e| e.is_object()).unwrap_or(false)) else { break };
let mut oldest = before;
let mut reached_known = false;
for e in events {
let Some(local) = e["nowPlayingTime"].as_str() else { continue };
let Some(played_at) = local_to_utc(local) else { continue };
if let Ok(t) = NaiveDateTime::parse_from_str(local, "%Y-%m-%d %H:%M:%S") { oldest = oldest.min(t); }
if played_at <= floor { reached_known = true; continue; }
if db.add_play(&Play {
source: "bauer", station: code, played_at: &played_at,
duration_ms: e["nowPlayingDuration"].as_i64().map(|s| s * 1000),
artist: e["nowPlayingArtist"].as_str().unwrap_or_default(),
title: e["nowPlayingTrack"].as_str().unwrap_or_default(),
track_id: &e["nowPlayingTrackId"].to_string(), artist_ids: "",
})? { new += 1; }
}
if reached_known || oldest >= before { break; }
before = oldest;
}
Ok(new)
}
+96
View File
@@ -0,0 +1,96 @@
//! The airplay database (SQLite). Every collector writes here; plays are deduplicated, so runs
//! can overlap and repeat safely.
use anyhow::Result;
use rusqlite::{params, Connection};
pub struct Db {
pub conn: Connection,
}
pub struct Play<'a> {
pub source: &'a str, // "dr" | "bauer"
pub station: &'a str, // "p3", "p4kbh", "nov", "dso", ...
pub played_at: &'a str, // UTC, RFC 3339 ("2026-11-14T21:00:09Z")
pub duration_ms: Option<i64>,
pub artist: &'a str,
pub title: &'a str,
pub track_id: &'a str, // the station's own id for the recording
pub artist_ids: &'a str, // DR artist ids, comma separated ("" for Bauer)
}
impl Db {
pub fn open(path: &str) -> Result<Db> {
let conn = Connection::open(path)?;
conn.execute_batch(
"PRAGMA journal_mode=WAL;
CREATE TABLE IF NOT EXISTS plays (
id INTEGER PRIMARY KEY,
source TEXT NOT NULL,
station TEXT NOT NULL,
played_at TEXT NOT NULL,
duration_ms INTEGER,
artist TEXT NOT NULL,
title TEXT NOT NULL,
track_id TEXT NOT NULL,
artist_ids TEXT NOT NULL DEFAULT '',
collected_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ','now')),
UNIQUE (station, played_at, track_id)
);
CREATE INDEX IF NOT EXISTS plays_station_time ON plays (station, played_at);
CREATE INDEX IF NOT EXISTS plays_artist ON plays (artist);
CREATE TABLE IF NOT EXISTS stations (
source TEXT NOT NULL, code TEXT NOT NULL, name TEXT NOT NULL,
PRIMARY KEY (source, code)
);
-- DR programmes whose playlist was read after they ended: no need to read them again.
CREATE TABLE IF NOT EXISTS dr_episodes_done (
channel TEXT NOT NULL, slug TEXT NOT NULL, fetched_at TEXT NOT NULL,
PRIMARY KEY (channel, slug)
);
-- DR artist pages: 52 weeks of weekly plays per channel (history before we started).
CREATE TABLE IF NOT EXISTS dr_artists (
artist_id TEXT PRIMARY KEY, name TEXT NOT NULL, fetched_at TEXT NOT NULL, raw TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS dr_artist_weeks (
artist_id TEXT NOT NULL, year INTEGER NOT NULL, week INTEGER NOT NULL, week_start TEXT NOT NULL,
channel TEXT NOT NULL, plays INTEGER NOT NULL,
PRIMARY KEY (artist_id, year, week, channel)
);
-- DR's weekly rotation lists (PDF), kept as files under data/rotations.
CREATE TABLE IF NOT EXISTS dr_rotation_pdfs (
url TEXT PRIMARY KEY, file TEXT NOT NULL, fetched_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS runs (
id INTEGER PRIMARY KEY, started_at TEXT NOT NULL, finished_at TEXT,
new_plays INTEGER, errors INTEGER, log TEXT
);",
)?;
Ok(Db { conn })
}
/// True if the play was new.
pub fn add_play(&self, p: &Play) -> Result<bool> {
let n = self.conn.execute(
"INSERT OR IGNORE INTO plays (source, station, played_at, duration_ms, artist, title, track_id, artist_ids)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
params![p.source, p.station, p.played_at, p.duration_ms, p.artist, p.title, p.track_id, p.artist_ids],
)?;
Ok(n > 0)
}
pub fn add_station(&self, source: &str, code: &str, name: &str) -> Result<()> {
self.conn.execute(
"INSERT INTO stations (source, code, name) VALUES (?1, ?2, ?3)
ON CONFLICT (source, code) DO UPDATE SET name = excluded.name",
params![source, code, name],
)?;
Ok(())
}
pub fn latest_play(&self, station: &str) -> Result<Option<String>> {
Ok(self
.conn
.query_row("SELECT max(played_at) FROM plays WHERE station = ?1", params![station], |r| r.get(0))?)
}
}
+146
View File
@@ -0,0 +1,146 @@
//! DR (Danmarks Radio): every play from the public playlist pages, the 52-week history on each
//! artist's page, and the weekly rotation-list PDF. See docs/DATA_SOURCES.md.
//!
//! The pages are Next.js; their data is the JSON in <script id="__NEXT_DATA__">.
use crate::{db::{Db, Play}, http::Http};
use anyhow::{anyhow, Result};
use chrono::{DateTime, Duration, NaiveDate, Utc};
use regex::Regex;
use rusqlite::params;
use serde_json::Value;
use std::sync::OnceLock;
/// The music channels: P3, every regional P4, P5, P6 Beat, P8 Jazz.
pub const CHANNELS: &[&str] = &[
"p3", "p4kbh", "p4sjaelland", "p4fyn", "p4syd", "p4trekanten", "p4vest", "p4aarhus", "p4nord",
"p4bornholm", "p4esbjerg", "p5", "p6beat", "p8jazz",
];
const BASE: &str = "https://www.dr.dk";
fn next_data(html: &str) -> Result<Value> {
static RE: OnceLock<Regex> = OnceLock::new();
let re = RE.get_or_init(|| Regex::new(r#"(?s)<script id="__NEXT_DATA__"[^>]*>(.*?)</script>"#).unwrap());
let m = re.captures(html).ok_or_else(|| anyhow!("no __NEXT_DATA__ in the page"))?;
Ok(serde_json::from_str(&m[1])?)
}
fn page_props(http: &Http, path: &str) -> Result<Option<Value>> {
match http.get(&format!("{BASE}{path}"))? {
None => Ok(None),
Some(html) => Ok(Some(next_data(&html)?["props"]["pageProps"].clone())),
}
}
fn to_utc(t: &str) -> Option<String> {
DateTime::parse_from_rfc3339(t).ok().map(|d| d.with_timezone(&Utc).format("%Y-%m-%dT%H:%M:%SZ").to_string())
}
/// Plays of the last `days` days on every channel. Programmes read after they ended are
/// remembered and skipped next time.
pub fn collect_plays(db: &Db, http: &Http, days: i64, log: &mut Vec<String>) -> Result<usize> {
let now = Utc::now();
let mut new = 0;
for &channel in CHANNELS {
db.add_station("dr", channel, channel)?;
for back in 0..=days {
let date: NaiveDate = (now - Duration::days(back)).date_naive();
let Some(day) = page_props(http, &format!("/playlister/{channel}/{date}"))? else { continue };
let items = day["schedule"]["items"].as_array().cloned().unwrap_or_default();
for item in items {
let slug = item["slug"].as_str().unwrap_or_default();
if slug.is_empty() { continue; }
let done: bool = db.conn.query_row(
"SELECT count(*) FROM dr_episodes_done WHERE channel = ?1 AND slug = ?2",
params![channel, slug], |r| r.get::<_, i64>(0))? > 0;
if done { continue; }
let ended = item["endTime"].as_str().and_then(|t| DateTime::parse_from_rfc3339(t).ok())
.map(|t| t.with_timezone(&Utc) < now).unwrap_or(false);
let episode = match page_props(http, &format!("/playlister/{channel}/{date}/{slug}")) {
Ok(Some(e)) => e,
Ok(None) => continue,
Err(e) => { log.push(format!("dr {channel} {slug}: {e}")); continue; }
};
for p in episode["playlistIndexPoints"].as_array().cloned().unwrap_or_default() {
if p["type"].as_str() != Some("Track") { continue; }
let Some(played_at) = p["playedTime"].as_str().and_then(to_utc) else { continue };
let roles = p["roles"].as_array().cloned().unwrap_or_default();
let main: Vec<&Value> = roles.iter().filter(|r| r["role"].as_str() == Some("Hovedkunstner")).collect();
let artist = if main.is_empty() { p["description"].as_str().unwrap_or_default().to_string() }
else { main.iter().filter_map(|r| r["name"].as_str()).collect::<Vec<_>>().join(", ") };
let artist_ids = roles.iter()
.filter_map(|r| r["artistUrn"].as_str().and_then(|u| u.rsplit(':').next()))
.collect::<Vec<_>>().join(",");
if db.add_play(&Play {
source: "dr", station: channel, played_at: &played_at,
duration_ms: p["durationMilliseconds"].as_i64(),
artist: &artist, title: p["title"].as_str().unwrap_or_default(),
track_id: p["trackUrn"].as_str().unwrap_or_default(), artist_ids: &artist_ids,
})? { new += 1; }
}
if ended {
db.conn.execute("INSERT OR REPLACE INTO dr_episodes_done VALUES (?1, ?2, ?3)",
params![channel, slug, now.to_rfc3339()])?;
}
}
}
}
Ok(new)
}
/// Artist pages: 52 weeks of weekly plays per channel. Fetched for every artist seen in the
/// plays, again once the page is `max_age_days` old. `limit` caps the number per run.
pub fn collect_artists(db: &Db, http: &Http, max_age_days: i64, limit: usize, log: &mut Vec<String>) -> Result<usize> {
let cutoff = (Utc::now() - Duration::days(max_age_days)).to_rfc3339();
let mut ids: Vec<String> = Vec::new();
{
let mut st = db.conn.prepare("SELECT DISTINCT artist_ids FROM plays WHERE source = 'dr' AND artist_ids != ''")?;
let rows = st.query_map([], |r| r.get::<_, String>(0))?;
let mut seen = std::collections::BTreeSet::new();
for r in rows { for id in r?.split(',') { if !id.is_empty() { seen.insert(id.to_string()); } } }
let mut fresh = db.conn.prepare("SELECT count(*) FROM dr_artists WHERE artist_id = ?1 AND fetched_at > ?2")?;
for id in seen {
if fresh.query_row(params![id, cutoff], |r| r.get::<_, i64>(0))? == 0 { ids.push(id); }
}
}
let mut done = 0;
for id in ids.into_iter().take(limit) {
let props = match page_props(http, &format!("/lyd/musik/kunstner/{id}")) {
Ok(Some(p)) => p,
Ok(None) => continue,
Err(e) => { log.push(format!("dr artist {id}: {e}")); continue; }
};
let name = props["artist"]["displayName"].as_str().unwrap_or_default().to_string();
let tx = db.conn.unchecked_transaction()?;
tx.execute("INSERT OR REPLACE INTO dr_artists VALUES (?1, ?2, ?3, ?4)",
params![id, name, Utc::now().to_rfc3339(), props.to_string()])?;
for w in props["weekPlaySums"].as_array().cloned().unwrap_or_default() {
for c in w["statisticsByChannels"].as_array().cloned().unwrap_or_default() {
tx.execute("INSERT OR REPLACE INTO dr_artist_weeks VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
params![id, w["year"].as_i64(), w["week"].as_i64(), w["date"].as_str().unwrap_or_default(),
c["channel"]["slug"].as_str().unwrap_or_default(), c["count"].as_i64()])?;
}
}
tx.commit()?;
done += 1;
}
Ok(done)
}
/// The weekly rotation list: the PDF linked from dr.dk/rotationer, kept under `dir`.
pub fn collect_rotation_pdf(db: &Db, http: &Http, dir: &std::path::Path) -> Result<Option<String>> {
let html = http.get(&format!("{BASE}/om-dr/nyheder/musik-i-rotation-paa-drs-radiokanaler-netop-nu"))?
.ok_or_else(|| anyhow!("the rotation page is gone"))?;
let re = Regex::new(r#"https://asset\.dr\.dk/[^"]+rotationer[^"]*\.pdf"#)?;
let url = re.find(&html).ok_or_else(|| anyhow!("no rotation PDF linked"))?.as_str().to_string();
let known: i64 = db.conn.query_row("SELECT count(*) FROM dr_rotation_pdfs WHERE url = ?1", params![url], |r| r.get(0))?;
if known > 0 { return Ok(None); }
let bytes = http.get_bytes(&url)?.ok_or_else(|| anyhow!("{url} is gone"))?;
std::fs::create_dir_all(dir)?;
let file = dir.join(url.rsplit('/').next().unwrap_or("rotation.pdf"));
std::fs::write(&file, bytes)?;
db.conn.execute("INSERT INTO dr_rotation_pdfs VALUES (?1, ?2, ?3)",
params![url, file.to_string_lossy(), Utc::now().to_rfc3339()])?;
Ok(Some(file.to_string_lossy().into_owned()))
}
+38
View File
@@ -0,0 +1,38 @@
//! One polite HTTP client: identifies itself, waits between requests, retries briefly.
use anyhow::{anyhow, Result};
use std::{thread, time::Duration};
pub struct Http {
client: reqwest::blocking::Client,
delay: Duration,
}
impl Http {
pub fn new(delay_ms: u64) -> Result<Http> {
let client = reqwest::blocking::Client::builder()
.user_agent("AirplayPredictor/0.1 (research; +https://git.idiotmaxxing.com/morits/Airplay-Predictor)")
.timeout(Duration::from_secs(30))
.build()?;
Ok(Http { client, delay: Duration::from_millis(delay_ms) })
}
/// The body, or None for 404 (a page that doesn't exist, e.g. a date outside the window).
pub fn get(&self, url: &str) -> Result<Option<String>> {
Ok(self.get_bytes(url)?.map(|b| String::from_utf8_lossy(&b).into_owned()))
}
pub fn get_bytes(&self, url: &str) -> Result<Option<Vec<u8>>> {
let mut last = anyhow!("no attempt");
for attempt in 0..3 {
thread::sleep(self.delay * (1 + attempt * 3));
match self.client.get(url).send() {
Ok(r) if r.status().as_u16() == 404 => return Ok(None),
Ok(r) if r.status().is_success() => return Ok(Some(r.bytes()?.to_vec())),
Ok(r) => last = anyhow!("{} from {}", r.status(), url),
Err(e) => last = anyhow!("{}: {}", url, e),
}
}
Err(last)
}
}
+97
View File
@@ -0,0 +1,97 @@
//! airplay: collects what Danish radio stations play, for modelling when artists get played again.
//!
//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer]
//! airplay stats [--db data/airplay.db]
mod bauer;
mod db;
mod dr;
mod http;
use anyhow::Result;
use chrono::Utc;
use rusqlite::params;
fn arg(args: &[String], name: &str) -> Option<String> {
args.iter().position(|a| a == name).and_then(|i| args.get(i + 1)).cloned()
}
fn main() -> Result<()> {
let args: Vec<String> = std::env::args().collect();
let db_path = arg(&args, "--db").unwrap_or_else(|| "data/airplay.db".into());
if let Some(dir) = std::path::Path::new(&db_path).parent() {
std::fs::create_dir_all(dir)?;
}
let db = db::Db::open(&db_path)?;
match args.get(1).map(String::as_str) {
Some("collect") => collect(&db, &args, &db_path),
Some("stats") => stats(&db),
_ => {
eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer]\n airplay stats [--db FILE]");
std::process::exit(2);
}
}
}
fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> {
let days: i64 = arg(args, "--days").and_then(|d| d.parse().ok()).unwrap_or(7);
let only = arg(args, "--only");
let http = http::Http::new(400)?;
let started = Utc::now().to_rfc3339();
db.conn.execute("INSERT INTO runs (started_at) VALUES (?1)", params![started])?;
let run = db.conn.last_insert_rowid();
let mut log: Vec<String> = Vec::new();
let mut new = 0;
let step = |name: &str, r: Result<String>, log: &mut Vec<String>| match r {
Ok(msg) => { println!("{name}: {msg}"); }
Err(e) => { eprintln!("{name}: {e}"); log.push(format!("{name}: {e}")); }
};
if only.as_deref() != Some("bauer") {
let r = dr::collect_plays(db, &http, days, &mut log);
if let Ok(n) = &r { new += n; }
step("dr plays", r.map(|n| format!("{n} new")), &mut log);
let dir = std::path::Path::new(db_path).parent().unwrap_or(std::path::Path::new(".")).join("rotations");
let r = dr::collect_rotation_pdf(db, &http, &dir);
step("dr rotation list", r.map(|f| f.map(|f| format!("saved {f}")).unwrap_or_else(|| "unchanged".into())), &mut log);
let r = dr::collect_artists(db, &http, 6, 400, &mut log);
step("dr artist pages", r.map(|n| format!("{n} fetched")), &mut log);
}
if only.as_deref() != Some("dr") {
match bauer::stations(db, &http, args.iter().any(|a| a == "--all-bauer")) {
Ok(stations) => {
for (code, name) in stations {
let r = bauer::collect(db, &http, &code, days, &mut log);
if let Ok(n) = &r { new += n; }
step(&format!("bauer {code} ({name})"), r.map(|n| format!("{n} new")), &mut log);
}
}
Err(e) => step("bauer stations", Err(e), &mut log),
}
}
db.conn.execute(
"UPDATE runs SET finished_at = ?1, new_plays = ?2, errors = ?3, log = ?4 WHERE id = ?5",
params![Utc::now().to_rfc3339(), new as i64, log.len() as i64, log.join("\n"), run],
)?;
println!("{new} new plays, {} problem(s)", log.len());
for l in log.iter().take(20) { eprintln!(" {l}"); }
Ok(())
}
fn stats(db: &db::Db) -> Result<()> {
let mut st = db.conn.prepare(
"SELECT p.station, coalesce(s.name, p.station), count(*), min(played_at), max(played_at), count(DISTINCT artist)
FROM plays p LEFT JOIN stations s ON s.code = p.station GROUP BY p.station ORDER BY count(*) DESC")?;
println!("{:<12} {:<22} {:>7} {:>8} {:<20} {}", "station", "name", "plays", "artists", "from", "to");
let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, i64>(2)?,
r.get::<_, String>(3)?, r.get::<_, String>(4)?, r.get::<_, i64>(5)?)))?;
for r in rows {
let (code, name, n, from, to, artists) = r?;
println!("{code:<12} {name:<22} {n:>7} {artists:>8} {from:<20} {to}");
}
let weeks: i64 = db.conn.query_row("SELECT count(*) FROM dr_artist_weeks", [], |r| r.get(0))?;
let artists: i64 = db.conn.query_row("SELECT count(*) FROM dr_artists", [], |r| r.get(0))?;
let pdfs: i64 = db.conn.query_row("SELECT count(*) FROM dr_rotation_pdfs", [], |r| r.get(0))?;
println!("DR artist history: {artists} artists, {weeks} artist-week-channel rows; rotation PDFs: {pdfs}");
Ok(())
}