From ff5c8e939c0e868ba8d4ffc586876a342cd4d499 Mon Sep 17 00:00:00 2001 From: moritsbw-cpu Date: Sun, 15 Nov 2026 01:19:40 +0000 Subject: [PATCH] Spotify Denmark chart history from kworb: every song that charted, weekly history since it charted, daily from now on Co-Authored-By: Claude Opus 5.5 --- src/main.rs | 18 ++++++- src/spotify.rs | 134 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 src/spotify.rs diff --git a/src/main.rs b/src/main.rs index 97f7e9b..577be6d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,6 @@ //! airplay: collects what Danish radio stations play, for modelling when artists get played again. //! -//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|musicbrainz] [--backfill] +//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|musicbrainz|spotify] [--backfill] //! airplay stats [--db data/airplay.db] //! airplay check [--db data/airplay.db] gaps, duplicates, DR cross-check, chart completeness //! @@ -15,6 +15,7 @@ mod dr; mod hitlisten; mod http; mod musicbrainz; +mod spotify; use anyhow::Result; use chrono::Utc; @@ -36,7 +37,7 @@ fn main() -> Result<()> { Some("stats") => stats(&db), Some("check") => check::run(&db), _ => { - eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|musicbrainz] [--backfill]\n airplay stats [--db FILE]\n airplay check [--db FILE]"); + eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|musicbrainz|spotify] [--backfill]\n airplay stats [--db FILE]\n airplay check [--db FILE]"); std::process::exit(2); } } @@ -92,6 +93,12 @@ fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> { let r = musicbrainz::collect(db, &mb_http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 2000 }), &mut log); step("musicbrainz release dates", r.map(|n| format!("{n} ISRCs looked up")), &mut log); } + if wants("spotify") { + // kworb.net is a one-person site: one request a second. + let kworb_http = http::Http::new(1000)?; + let r = spotify::collect(db, &kworb_http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 400 }), &mut log); + step("spotify chart history (kworb)", r.map(|(l, p)| format!("{l} songs listed, {p} histories read")), &mut log); + } if wants("hitlisten") { let r = hitlisten::collect(db, &http, if backfill { usize::MAX } else { 60 }, &mut log); step("hitlisten airplay chart", r.map(|n| format!("{n} weeks fetched")), &mut log); @@ -131,6 +138,13 @@ fn stats(db: &db::Db) -> Result<()> { (SELECT count(*) FROM deezer_tracks), (SELECT count(*) FROM deezer_albums), (SELECT count(*) FROM deezer_albums WHERE coalesce(label, '') != '')", [], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)))?; + spotify::create_tables(db)?; + let (sp_tracks, sp_read, sp_rows, sp_from): (i64, i64, i64, Option) = db.conn.query_row( + "SELECT (SELECT count(*) FROM spotify_dk_tracks), (SELECT count(*) FROM spotify_dk_tracks WHERE history_fetched_at IS NOT NULL), + (SELECT count(*) FROM spotify_chart_history WHERE country = 'DK'), + (SELECT min(date) FROM spotify_chart_history WHERE country = 'DK' AND chart = 'daily')", [], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)))?; + println!("Spotify Denmark: {sp_tracks} songs that charted, {sp_read} histories read, {sp_rows} DK chart rows (daily from {})", sp_from.unwrap_or_default()); musicbrainz::create_tables(db)?; let (mb, mb_known): (i64, i64) = db.conn.query_row( "SELECT count(*), count(*) FILTER (WHERE recordings > 0) FROM mb_isrc", [], |r| Ok((r.get(0)?, r.get(1)?)))?; diff --git a/src/spotify.rs b/src/spotify.rs new file mode 100644 index 0000000..5f1b622 --- /dev/null +++ b/src/spotify.rs @@ -0,0 +1,134 @@ +//! Spotify chart history for Denmark, from kworb.net (which republishes Spotify's charts): +//! +//! /spotify/country/dk_daily_totals.html every song ever on the Danish daily chart, with its +//! Spotify track id, days on chart, top-10 days, peak, streams +//! /spotify/country/dk_daily.html today's Danish chart (which songs to refresh) +//! /spotify/track/{id}.html a song's chart history: weekly and daily, position and +//! streams per country (Global, DK, SE, NO, ...) +//! +//! A backfill reads every track page once (about 12,700 songs); a daily run refreshes the songs +//! on today's chart and any new ones. No login needed. + +use crate::{db::Db, http::Http}; +use anyhow::Result; +use chrono::Utc; +use regex::Regex; +use rusqlite::params; + +const BASE: &str = "https://kworb.net/spotify"; + +pub fn create_tables(db: &Db) -> Result<()> { + db.conn.execute_batch( + "CREATE TABLE IF NOT EXISTS spotify_dk_tracks ( + track_id TEXT PRIMARY KEY, artist TEXT NOT NULL, title TEXT NOT NULL, artist_ids TEXT, + days INTEGER, top10_days INTEGER, peak INTEGER, peak_times INTEGER, + peak_streams INTEGER, total_streams INTEGER, listed_at TEXT NOT NULL, + history_fetched_at TEXT + ); + CREATE TABLE IF NOT EXISTS spotify_chart_history ( + track_id TEXT NOT NULL, chart TEXT NOT NULL, -- 'daily' | 'weekly' + date TEXT NOT NULL, country TEXT NOT NULL, -- 'DK', 'Global', 'SE', ... + position INTEGER, streams INTEGER, + PRIMARY KEY (track_id, chart, date, country) + ); + CREATE INDEX IF NOT EXISTS spotify_history_dk ON spotify_chart_history (country, chart, date);", + )?; + Ok(()) +} + +fn num(s: &str) -> Option { + let d: String = s.chars().filter(|c| c.is_ascii_digit()).collect(); + d.parse().ok() +} + +fn strip(s: &str) -> String { + let tags = Regex::new(r"<[^>]+>").unwrap(); + tags.replace_all(s, "").replace("&", "&").replace("'", "'").replace(""", "\"").trim().to_string() +} + +/// The list of every song that has charted in Denmark. +fn read_totals(db: &Db, http: &Http) -> Result { + let html = http.get(&format!("{BASE}/country/dk_daily_totals.html"))?.unwrap_or_default(); + let row = Regex::new(r#"(?s)]*>\s*
(.*?)
(.*?)"#)?; + let link = Regex::new(r#"(.*?)"#)?; + let cell = Regex::new(r"(?s)]*>(.*?)")?; + let now = Utc::now().to_rfc3339(); + let mut n = 0; + for r in row.captures_iter(&html) { + let (mut artists, mut artist_ids, mut track) = (Vec::new(), Vec::new(), None); + for l in link.captures_iter(&r[1]) { + if &l[1] == "artist" { artists.push(strip(&l[3])); artist_ids.push(l[2].to_string()); } + else { track = Some((l[2].to_string(), strip(&l[3]))); } + } + let Some((id, title)) = track else { continue }; + // Days, T10, Pk, (x?), PkStreams, Total + let cells: Vec = cell.captures_iter(&r[2]).map(|c| strip(&c[1])).collect(); + let get = |i: usize| cells.get(i).and_then(|c| num(c)); + db.conn.execute( + "INSERT INTO spotify_dk_tracks (track_id, artist, title, artist_ids, days, top10_days, peak, peak_times, peak_streams, total_streams, listed_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11) + ON CONFLICT (track_id) DO UPDATE SET days = excluded.days, top10_days = excluded.top10_days, peak = excluded.peak, + peak_times = excluded.peak_times, peak_streams = excluded.peak_streams, total_streams = excluded.total_streams, + listed_at = excluded.listed_at", + params![id, artists.join(", "), title, artist_ids.join(","), get(0), get(1), get(2), get(3), get(4), get(5), now])?; + n += 1; + } + Ok(n) +} + +/// A song's chart history (both tables: weekly first, then daily), every country. +fn read_history(db: &Db, http: &Http, id: &str) -> Result { + let Some(html) = http.get(&format!("{BASE}/track/{id}.html"))? else { return Ok(0) }; + let header = Regex::new(r"(?s)Date(.*?)")?; + let th = Regex::new(r"([^<]*)")?; + let row = Regex::new(r#"(?s)]*>(\d{4})/(\d{2})/(\d{2})(.*?)"#)?; + let cell = Regex::new(r"(?s)(.*?)")?; + let pos = Regex::new(r#"(\d+)"#)?; + let streams = Regex::new(r#"([\d,]+)"#)?; + let starts: Vec<(usize, Vec)> = header.captures_iter(&html) + .map(|c| (c.get(0).unwrap().start(), th.captures_iter(&c[1]).map(|t| t[1].trim().to_string()).collect())) + .collect(); + let tx = db.conn.unchecked_transaction()?; + let mut n = 0; + for (k, (start, countries)) in starts.iter().enumerate() { + let end = html[*start..].find("").map(|e| start + e).unwrap_or(html.len()); + // kworb shows the weekly table first, then the daily one. + let chart = if k == 0 && starts.len() > 1 { "weekly" } else { "daily" }; + for r in row.captures_iter(&html[*start..end]) { + let date = format!("{}-{}-{}", &r[1], &r[2], &r[3]); + for (i, c) in cell.captures_iter(&r[4]).enumerate() { + let Some(country) = countries.get(i) else { break }; + let p = pos.captures(&c[1]).and_then(|m| m[1].parse::().ok()); + let s = streams.captures(&c[1]).and_then(|m| num(&m[1])); + if p.is_none() && s.is_none() { continue; } + tx.execute("INSERT OR REPLACE INTO spotify_chart_history VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + params![id, chart, date, country, p, s])?; + n += 1; + } + } + } + tx.execute("UPDATE spotify_dk_tracks SET history_fetched_at = ?1 WHERE track_id = ?2", params![Utc::now().to_rfc3339(), id])?; + tx.commit()?; + Ok(n) +} + +/// Refreshes the list, then reads the history of songs never read, plus (daily runs) the songs +/// on today's chart. `limit` caps the track pages per run. +pub fn collect(db: &Db, http: &Http, limit: usize, log: &mut Vec) -> Result<(usize, usize)> { + create_tables(db)?; + let listed = read_totals(db, http)?; + let today = http.get(&format!("{BASE}/country/dk_daily.html"))?.unwrap_or_default(); + let on_chart: Vec = Regex::new(r"\.\./track/([A-Za-z0-9]{22})\.html")? + .captures_iter(&today).map(|c| c[1].to_string()).collect(); + let mut todo: Vec = db.conn.prepare("SELECT track_id FROM spotify_dk_tracks WHERE history_fetched_at IS NULL ORDER BY total_streams DESC")? + .query_map([], |r| r.get(0))?.collect::>()?; + for id in on_chart { if !todo.contains(&id) { todo.insert(0, id); } } + let mut pages = 0; + for id in todo.into_iter().take(limit) { + match read_history(db, http, &id) { + Ok(_) => pages += 1, + Err(e) => log.push(format!("kworb track {id}: {e}")), + } + } + Ok((listed, pages)) +}