Spotify Denmark chart history from kworb: every song that charted, weekly history since it charted, daily from now on
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+16
-2
@@ -1,6 +1,6 @@
|
||||
//! airplay: collects what Danish radio stations play, for modelling when artists get played again.
|
||||
//!
|
||||
//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|musicbrainz] [--backfill]
|
||||
//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|musicbrainz|spotify] [--backfill]
|
||||
//! airplay stats [--db data/airplay.db]
|
||||
//! airplay check [--db data/airplay.db] gaps, duplicates, DR cross-check, chart completeness
|
||||
//!
|
||||
@@ -15,6 +15,7 @@ mod dr;
|
||||
mod hitlisten;
|
||||
mod http;
|
||||
mod musicbrainz;
|
||||
mod spotify;
|
||||
|
||||
use anyhow::Result;
|
||||
use chrono::Utc;
|
||||
@@ -36,7 +37,7 @@ fn main() -> Result<()> {
|
||||
Some("stats") => stats(&db),
|
||||
Some("check") => check::run(&db),
|
||||
_ => {
|
||||
eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|musicbrainz] [--backfill]\n airplay stats [--db FILE]\n airplay check [--db FILE]");
|
||||
eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|musicbrainz|spotify] [--backfill]\n airplay stats [--db FILE]\n airplay check [--db FILE]");
|
||||
std::process::exit(2);
|
||||
}
|
||||
}
|
||||
@@ -92,6 +93,12 @@ fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> {
|
||||
let r = musicbrainz::collect(db, &mb_http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 2000 }), &mut log);
|
||||
step("musicbrainz release dates", r.map(|n| format!("{n} ISRCs looked up")), &mut log);
|
||||
}
|
||||
if wants("spotify") {
|
||||
// kworb.net is a one-person site: one request a second.
|
||||
let kworb_http = http::Http::new(1000)?;
|
||||
let r = spotify::collect(db, &kworb_http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 400 }), &mut log);
|
||||
step("spotify chart history (kworb)", r.map(|(l, p)| format!("{l} songs listed, {p} histories read")), &mut log);
|
||||
}
|
||||
if wants("hitlisten") {
|
||||
let r = hitlisten::collect(db, &http, if backfill { usize::MAX } else { 60 }, &mut log);
|
||||
step("hitlisten airplay chart", r.map(|n| format!("{n} weeks fetched")), &mut log);
|
||||
@@ -131,6 +138,13 @@ fn stats(db: &db::Db) -> Result<()> {
|
||||
(SELECT count(*) FROM deezer_tracks), (SELECT count(*) FROM deezer_albums),
|
||||
(SELECT count(*) FROM deezer_albums WHERE coalesce(label, '') != '')", [],
|
||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)))?;
|
||||
spotify::create_tables(db)?;
|
||||
let (sp_tracks, sp_read, sp_rows, sp_from): (i64, i64, i64, Option<String>) = db.conn.query_row(
|
||||
"SELECT (SELECT count(*) FROM spotify_dk_tracks), (SELECT count(*) FROM spotify_dk_tracks WHERE history_fetched_at IS NOT NULL),
|
||||
(SELECT count(*) FROM spotify_chart_history WHERE country = 'DK'),
|
||||
(SELECT min(date) FROM spotify_chart_history WHERE country = 'DK' AND chart = 'daily')", [],
|
||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)))?;
|
||||
println!("Spotify Denmark: {sp_tracks} songs that charted, {sp_read} histories read, {sp_rows} DK chart rows (daily from {})", sp_from.unwrap_or_default());
|
||||
musicbrainz::create_tables(db)?;
|
||||
let (mb, mb_known): (i64, i64) = db.conn.query_row(
|
||||
"SELECT count(*), count(*) FILTER (WHERE recordings > 0) FROM mb_isrc", [], |r| Ok((r.get(0)?, r.get(1)?)))?;
|
||||
|
||||
+134
@@ -0,0 +1,134 @@
|
||||
//! Spotify chart history for Denmark, from kworb.net (which republishes Spotify's charts):
|
||||
//!
|
||||
//! /spotify/country/dk_daily_totals.html every song ever on the Danish daily chart, with its
|
||||
//! Spotify track id, days on chart, top-10 days, peak, streams
|
||||
//! /spotify/country/dk_daily.html today's Danish chart (which songs to refresh)
|
||||
//! /spotify/track/{id}.html a song's chart history: weekly and daily, position and
|
||||
//! streams per country (Global, DK, SE, NO, ...)
|
||||
//!
|
||||
//! A backfill reads every track page once (about 12,700 songs); a daily run refreshes the songs
|
||||
//! on today's chart and any new ones. No login needed.
|
||||
|
||||
use crate::{db::Db, http::Http};
|
||||
use anyhow::Result;
|
||||
use chrono::Utc;
|
||||
use regex::Regex;
|
||||
use rusqlite::params;
|
||||
|
||||
const BASE: &str = "https://kworb.net/spotify";
|
||||
|
||||
pub fn create_tables(db: &Db) -> Result<()> {
|
||||
db.conn.execute_batch(
|
||||
"CREATE TABLE IF NOT EXISTS spotify_dk_tracks (
|
||||
track_id TEXT PRIMARY KEY, artist TEXT NOT NULL, title TEXT NOT NULL, artist_ids TEXT,
|
||||
days INTEGER, top10_days INTEGER, peak INTEGER, peak_times INTEGER,
|
||||
peak_streams INTEGER, total_streams INTEGER, listed_at TEXT NOT NULL,
|
||||
history_fetched_at TEXT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS spotify_chart_history (
|
||||
track_id TEXT NOT NULL, chart TEXT NOT NULL, -- 'daily' | 'weekly'
|
||||
date TEXT NOT NULL, country TEXT NOT NULL, -- 'DK', 'Global', 'SE', ...
|
||||
position INTEGER, streams INTEGER,
|
||||
PRIMARY KEY (track_id, chart, date, country)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS spotify_history_dk ON spotify_chart_history (country, chart, date);",
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn num(s: &str) -> Option<i64> {
|
||||
let d: String = s.chars().filter(|c| c.is_ascii_digit()).collect();
|
||||
d.parse().ok()
|
||||
}
|
||||
|
||||
fn strip(s: &str) -> String {
|
||||
let tags = Regex::new(r"<[^>]+>").unwrap();
|
||||
tags.replace_all(s, "").replace("&", "&").replace("'", "'").replace(""", "\"").trim().to_string()
|
||||
}
|
||||
|
||||
/// The list of every song that has charted in Denmark.
|
||||
fn read_totals(db: &Db, http: &Http) -> Result<usize> {
|
||||
let html = http.get(&format!("{BASE}/country/dk_daily_totals.html"))?.unwrap_or_default();
|
||||
let row = Regex::new(r#"(?s)<tr[^>]*>\s*<td class="text mp"><div>(.*?)</div></td>(.*?)</tr>"#)?;
|
||||
let link = Regex::new(r#"<a href="\.\./(artist|track)/([A-Za-z0-9]{22})\.html">(.*?)</a>"#)?;
|
||||
let cell = Regex::new(r"(?s)<td[^>]*>(.*?)</td>")?;
|
||||
let now = Utc::now().to_rfc3339();
|
||||
let mut n = 0;
|
||||
for r in row.captures_iter(&html) {
|
||||
let (mut artists, mut artist_ids, mut track) = (Vec::new(), Vec::new(), None);
|
||||
for l in link.captures_iter(&r[1]) {
|
||||
if &l[1] == "artist" { artists.push(strip(&l[3])); artist_ids.push(l[2].to_string()); }
|
||||
else { track = Some((l[2].to_string(), strip(&l[3]))); }
|
||||
}
|
||||
let Some((id, title)) = track else { continue };
|
||||
// Days, T10, Pk, (x?), PkStreams, Total
|
||||
let cells: Vec<String> = cell.captures_iter(&r[2]).map(|c| strip(&c[1])).collect();
|
||||
let get = |i: usize| cells.get(i).and_then(|c| num(c));
|
||||
db.conn.execute(
|
||||
"INSERT INTO spotify_dk_tracks (track_id, artist, title, artist_ids, days, top10_days, peak, peak_times, peak_streams, total_streams, listed_at)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)
|
||||
ON CONFLICT (track_id) DO UPDATE SET days = excluded.days, top10_days = excluded.top10_days, peak = excluded.peak,
|
||||
peak_times = excluded.peak_times, peak_streams = excluded.peak_streams, total_streams = excluded.total_streams,
|
||||
listed_at = excluded.listed_at",
|
||||
params![id, artists.join(", "), title, artist_ids.join(","), get(0), get(1), get(2), get(3), get(4), get(5), now])?;
|
||||
n += 1;
|
||||
}
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
/// A song's chart history (both tables: weekly first, then daily), every country.
|
||||
fn read_history(db: &Db, http: &Http, id: &str) -> Result<usize> {
|
||||
let Some(html) = http.get(&format!("{BASE}/track/{id}.html"))? else { return Ok(0) };
|
||||
let header = Regex::new(r"(?s)<tr><th>Date</th>(.*?)</tr>")?;
|
||||
let th = Regex::new(r"<th>([^<]*)</th>")?;
|
||||
let row = Regex::new(r#"(?s)<tr[^>]*><td>(\d{4})/(\d{2})/(\d{2})</td>(.*?)</tr>"#)?;
|
||||
let cell = Regex::new(r"(?s)<td>(.*?)</td>")?;
|
||||
let pos = Regex::new(r#"<span class="p">(\d+)</span>"#)?;
|
||||
let streams = Regex::new(r#"<span class="s">([\d,]+)</span>"#)?;
|
||||
let starts: Vec<(usize, Vec<String>)> = header.captures_iter(&html)
|
||||
.map(|c| (c.get(0).unwrap().start(), th.captures_iter(&c[1]).map(|t| t[1].trim().to_string()).collect()))
|
||||
.collect();
|
||||
let tx = db.conn.unchecked_transaction()?;
|
||||
let mut n = 0;
|
||||
for (k, (start, countries)) in starts.iter().enumerate() {
|
||||
let end = html[*start..].find("</table>").map(|e| start + e).unwrap_or(html.len());
|
||||
// kworb shows the weekly table first, then the daily one.
|
||||
let chart = if k == 0 && starts.len() > 1 { "weekly" } else { "daily" };
|
||||
for r in row.captures_iter(&html[*start..end]) {
|
||||
let date = format!("{}-{}-{}", &r[1], &r[2], &r[3]);
|
||||
for (i, c) in cell.captures_iter(&r[4]).enumerate() {
|
||||
let Some(country) = countries.get(i) else { break };
|
||||
let p = pos.captures(&c[1]).and_then(|m| m[1].parse::<i64>().ok());
|
||||
let s = streams.captures(&c[1]).and_then(|m| num(&m[1]));
|
||||
if p.is_none() && s.is_none() { continue; }
|
||||
tx.execute("INSERT OR REPLACE INTO spotify_chart_history VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
|
||||
params![id, chart, date, country, p, s])?;
|
||||
n += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
tx.execute("UPDATE spotify_dk_tracks SET history_fetched_at = ?1 WHERE track_id = ?2", params![Utc::now().to_rfc3339(), id])?;
|
||||
tx.commit()?;
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
/// Refreshes the list, then reads the history of songs never read, plus (daily runs) the songs
|
||||
/// on today's chart. `limit` caps the track pages per run.
|
||||
pub fn collect(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) -> Result<(usize, usize)> {
|
||||
create_tables(db)?;
|
||||
let listed = read_totals(db, http)?;
|
||||
let today = http.get(&format!("{BASE}/country/dk_daily.html"))?.unwrap_or_default();
|
||||
let on_chart: Vec<String> = Regex::new(r"\.\./track/([A-Za-z0-9]{22})\.html")?
|
||||
.captures_iter(&today).map(|c| c[1].to_string()).collect();
|
||||
let mut todo: Vec<String> = db.conn.prepare("SELECT track_id FROM spotify_dk_tracks WHERE history_fetched_at IS NULL ORDER BY total_streams DESC")?
|
||||
.query_map([], |r| r.get(0))?.collect::<Result<_, _>>()?;
|
||||
for id in on_chart { if !todo.contains(&id) { todo.insert(0, id); } }
|
||||
let mut pages = 0;
|
||||
for id in todo.into_iter().take(limit) {
|
||||
match read_history(db, http, &id) {
|
||||
Ok(_) => pages += 1,
|
||||
Err(e) => log.push(format!("kworb track {id}: {e}")),
|
||||
}
|
||||
}
|
||||
Ok((listed, pages))
|
||||
}
|
||||
Reference in New Issue
Block a user