From 347df0d82db04e0f85f09fc4eec1ff739ea986f7 Mon Sep 17 00:00:00 2001 From: moritsbw-cpu Date: Sun, 15 Nov 2026 01:31:33 +0000 Subject: [PATCH] Spotify playlists: New Music Friday Denmark and Top 50 Denmark snapshots (embed page), New Music Friday history since 2022 per ISRC (SuperFridayChart) Co-Authored-By: Claude Opus 5.5 --- scripts/backfill.sh | 2 +- src/main.rs | 20 +++++++- src/playlists.rs | 116 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 135 insertions(+), 3 deletions(-) create mode 100644 src/playlists.rs diff --git a/scripts/backfill.sh b/scripts/backfill.sh index 2996dbf..9976907 100755 --- a/scripts/backfill.sh +++ b/scripts/backfill.sh @@ -5,7 +5,7 @@ cd "$(dirname "$0")/.." || exit 1 bin=./target/release/airplay # Wait for collectors already running (a backfill started by hand, the daily timer). while pgrep -f "release/airplay collect" >/dev/null; do sleep 60; done -for source in dr hitlisten deezer musicbrainz spotify; do +for source in dr hitlisten deezer musicbrainz spotify nmf playlists; do echo "== $(date -u +%FT%TZ) $source" $bin collect --only "$source" --backfill done diff --git a/src/main.rs b/src/main.rs index 577be6d..1eb04e0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,6 @@ //! airplay: collects what Danish radio stations play, for modelling when artists get played again. //! -//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|musicbrainz|spotify] [--backfill] +//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|musicbrainz|spotify|playlists|nmf] [--backfill] //! airplay stats [--db data/airplay.db] //! airplay check [--db data/airplay.db] gaps, duplicates, DR cross-check, chart completeness //! @@ -15,6 +15,7 @@ mod dr; mod hitlisten; mod http; mod musicbrainz; +mod playlists; mod spotify; use anyhow::Result; @@ -37,7 +38,7 @@ fn main() -> Result<()> { Some("stats") => stats(&db), Some("check") => check::run(&db), _ => { - eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|musicbrainz|spotify] [--backfill]\n airplay stats [--db FILE]\n airplay check [--db FILE]"); + eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|musicbrainz|spotify|playlists|nmf] [--backfill]\n airplay stats [--db FILE]\n airplay check [--db FILE]"); std::process::exit(2); } } @@ -99,6 +100,15 @@ fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> { let r = spotify::collect(db, &kworb_http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 400 }), &mut log); step("spotify chart history (kworb)", r.map(|(l, p)| format!("{l} songs listed, {p} histories read")), &mut log); } + if wants("playlists") { + let r = playlists::snapshot(db, &http, &mut log); + step("spotify playlists (New Music Friday DK, Top 50 DK)", r.map(|n| format!("{n} changed and saved")), &mut log); + } + if wants("nmf") { + let sfc_http = http::Http::new(1000)?; + let r = playlists::nmf_history(db, &sfc_http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 1500 }), &mut log); + step("new music friday history (superfridaychart)", r.map(|n| format!("{n} ISRCs looked up")), &mut log); + } if wants("hitlisten") { let r = hitlisten::collect(db, &http, if backfill { usize::MAX } else { 60 }, &mut log); step("hitlisten airplay chart", r.map(|n| format!("{n} weeks fetched")), &mut log); @@ -145,6 +155,12 @@ fn stats(db: &db::Db) -> Result<()> { (SELECT min(date) FROM spotify_chart_history WHERE country = 'DK' AND chart = 'daily')", [], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)))?; println!("Spotify Denmark: {sp_tracks} songs that charted, {sp_read} histories read, {sp_rows} DK chart rows (daily from {})", sp_from.unwrap_or_default()); + playlists::create_tables(db)?; + let (snaps, nmf_isrcs, nmf_dk, nmf_from): (i64, i64, i64, Option) = db.conn.query_row( + "SELECT (SELECT count(DISTINCT playlist_id || taken_at) FROM playlist_snapshots), (SELECT count(*) FROM nmf_lookups), + (SELECT count(*) FROM nmf_additions WHERE country = 'Denmark'), (SELECT min(friday) FROM nmf_additions)", [], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)))?; + println!("Playlists: {snaps} snapshots; New Music Friday history: {nmf_isrcs} songs looked up, {nmf_dk} Denmark additions (since {})", nmf_from.unwrap_or_default()); musicbrainz::create_tables(db)?; let (mb, mb_known): (i64, i64) = db.conn.query_row( "SELECT count(*), count(*) FILTER (WHERE recordings > 0) FROM mb_isrc", [], |r| Ok((r.get(0)?, r.get(1)?)))?; diff --git a/src/playlists.rs b/src/playlists.rs new file mode 100644 index 0000000..a137c38 --- /dev/null +++ b/src/playlists.rs @@ -0,0 +1,116 @@ +//! Spotify editorial playlists: +//! +//! - Snapshots, from now on: the full track list of New Music Friday Denmark and Top 50 Denmark, +//! read from Spotify's public embed page (open.spotify.com/embed/playlist/{id}, no login). +//! Daily runs store a snapshot whenever the list changed. +//! - History, since January 2022: SuperFridayChart (superfridaychart.com), which records every +//! country's New Music Friday additions. Looked up per song by ISRC (from Deezer): +//! /track/{isrc} lists each Friday the song was added, with country, playlist, followers, position. + +use crate::{db::Db, http::Http}; +use anyhow::{anyhow, Result}; +use chrono::{NaiveDate, Utc}; +use regex::Regex; +use rusqlite::params; +use serde_json::Value; + +pub const PLAYLISTS: &[(&str, &str)] = &[ + ("37i9dQZF1DWWuIAAkPM0Lu", "New Music Friday Denmark"), + ("37i9dQZEVXbL3J0k32lWnN", "Top 50 - Denmark"), +]; + +pub fn create_tables(db: &Db) -> Result<()> { + db.conn.execute_batch( + "CREATE TABLE IF NOT EXISTS playlist_snapshots ( + playlist_id TEXT NOT NULL, name TEXT NOT NULL, taken_at TEXT NOT NULL, + position INTEGER NOT NULL, track_id TEXT NOT NULL, title TEXT, artists TEXT, duration_ms INTEGER, + PRIMARY KEY (playlist_id, taken_at, position) + ); + CREATE TABLE IF NOT EXISTS nmf_additions ( -- SuperFridayChart + isrc TEXT NOT NULL, friday TEXT NOT NULL, country TEXT NOT NULL, + playlist_id TEXT, followers INTEGER, position INTEGER, + PRIMARY KEY (isrc, friday, country) + ); + CREATE TABLE IF NOT EXISTS nmf_lookups ( + isrc TEXT PRIMARY KEY, found INTEGER NOT NULL, fetched_at TEXT NOT NULL + );", + )?; + Ok(()) +} + +/// A playlist's tracks in order: (track id, title, artists, duration ms). +fn read_embed(http: &Http, id: &str) -> Result)>> { + let html = http.get(&format!("https://open.spotify.com/embed/playlist/{id}"))?.ok_or_else(|| anyhow!("playlist {id} not found"))?; + let re = Regex::new(r#"(?s)"#)?; + let data: Value = serde_json::from_str(&re.captures(&html).ok_or_else(|| anyhow!("no data in the embed page"))?[1])?; + let list = data["props"]["pageProps"]["state"]["data"]["entity"]["trackList"].as_array().cloned().unwrap_or_default(); + Ok(list.iter().map(|t| ( + t["uri"].as_str().unwrap_or_default().rsplit(':').next().unwrap_or_default().to_string(), + t["title"].as_str().unwrap_or_default().to_string(), + t["subtitle"].as_str().unwrap_or_default().replace('\u{a0}', " "), + t["duration"].as_i64(), + )).collect()) +} + +/// Stores a snapshot of each playlist if it differs from the last one stored. +pub fn snapshot(db: &Db, http: &Http, log: &mut Vec) -> Result { + create_tables(db)?; + let mut stored = 0; + for &(id, name) in PLAYLISTS { + let tracks = match read_embed(http, id) { + Ok(t) if !t.is_empty() => t, + Ok(_) => { log.push(format!("playlist {name}: empty")); continue; } + Err(e) => { log.push(format!("playlist {name}: {e}")); continue; } + }; + let last: Vec = db.conn.prepare( + "SELECT track_id FROM playlist_snapshots WHERE playlist_id = ?1 + AND taken_at = (SELECT max(taken_at) FROM playlist_snapshots WHERE playlist_id = ?1) ORDER BY position")? + .query_map([id], |r| r.get(0))?.collect::>()?; + if last == tracks.iter().map(|t| t.0.clone()).collect::>() { continue; } + let now = Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string(); + let tx = db.conn.unchecked_transaction()?; + for (i, (track, title, artists, dur)) in tracks.iter().enumerate() { + tx.execute("INSERT OR REPLACE INTO playlist_snapshots VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", + params![id, name, now, i as i64 + 1, track, title, artists, dur])?; + } + tx.commit()?; + stored += 1; + } + Ok(stored) +} + +/// New Music Friday additions per ISRC from SuperFridayChart (every ISRC Deezer gave us, once). +pub fn nmf_history(db: &Db, http: &Http, limit: usize, log: &mut Vec) -> Result { + create_tables(db)?; + crate::deezer::create_tables(db)?; + let todo: Vec = db.conn.prepare( + "SELECT DISTINCT isrc FROM deezer_tracks WHERE coalesce(isrc, '') != '' AND isrc NOT IN (SELECT isrc FROM nmf_lookups)")? + .query_map([], |r| r.get(0))?.collect::>()?; + let day = Regex::new(r#"]*>\s*(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday), ([A-Za-z]+ \d{2}, \d{4})\s*"#)?; + let row = Regex::new(r#"(?s)]*>([^<]+)

\s*]*>([\d,]+) followers\s*]*>Position (\d+)"#)?; + let mut done = 0; + for isrc in todo.into_iter().take(limit) { + let html = match http.get(&format!("https://www.superfridaychart.com/track/{isrc}")) { + Ok(h) => h, + Err(e) => { log.push(format!("superfridaychart {isrc}: {e}")); continue; } + }; + let mut found = 0; + if let Some(html) = html { + // Each Friday is an

date followed by its table of playlists. + let heads: Vec<(usize, NaiveDate)> = day.captures_iter(&html) + .filter_map(|c| Some((c.get(0)?.start(), NaiveDate::parse_from_str(&c[1], "%B %d, %Y").ok()?))) + .collect(); + for (k, (start, friday)) in heads.iter().enumerate() { + let end = heads.get(k + 1).map(|h| h.0).unwrap_or(html.len()); + for r in row.captures_iter(&html[*start..end]) { + db.conn.execute("INSERT OR REPLACE INTO nmf_additions VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + params![isrc, friday.to_string(), r[1].trim(), &r[2], r[3].replace(',', "").parse::().ok(), r[4].parse::().ok()])?; + found += 1; + } + } + } + db.conn.execute("INSERT OR REPLACE INTO nmf_lookups VALUES (?1, ?2, ?3)", params![isrc, found, Utc::now().to_rfc3339()])?; + done += 1; + } + Ok(done) +}