Spotify playlists: New Music Friday Denmark and Top 50 Denmark snapshots (embed page), New Music Friday history since 2022 per ISRC (SuperFridayChart)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+18
-2
@@ -1,6 +1,6 @@
|
||||
//! airplay: collects what Danish radio stations play, for modelling when artists get played again.
|
||||
//!
|
||||
//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|musicbrainz|spotify] [--backfill]
|
||||
//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|musicbrainz|spotify|playlists|nmf] [--backfill]
|
||||
//! airplay stats [--db data/airplay.db]
|
||||
//! airplay check [--db data/airplay.db] gaps, duplicates, DR cross-check, chart completeness
|
||||
//!
|
||||
@@ -15,6 +15,7 @@ mod dr;
|
||||
mod hitlisten;
|
||||
mod http;
|
||||
mod musicbrainz;
|
||||
mod playlists;
|
||||
mod spotify;
|
||||
|
||||
use anyhow::Result;
|
||||
@@ -37,7 +38,7 @@ fn main() -> Result<()> {
|
||||
Some("stats") => stats(&db),
|
||||
Some("check") => check::run(&db),
|
||||
_ => {
|
||||
eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|musicbrainz|spotify] [--backfill]\n airplay stats [--db FILE]\n airplay check [--db FILE]");
|
||||
eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|musicbrainz|spotify|playlists|nmf] [--backfill]\n airplay stats [--db FILE]\n airplay check [--db FILE]");
|
||||
std::process::exit(2);
|
||||
}
|
||||
}
|
||||
@@ -99,6 +100,15 @@ fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> {
|
||||
let r = spotify::collect(db, &kworb_http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 400 }), &mut log);
|
||||
step("spotify chart history (kworb)", r.map(|(l, p)| format!("{l} songs listed, {p} histories read")), &mut log);
|
||||
}
|
||||
if wants("playlists") {
|
||||
let r = playlists::snapshot(db, &http, &mut log);
|
||||
step("spotify playlists (New Music Friday DK, Top 50 DK)", r.map(|n| format!("{n} changed and saved")), &mut log);
|
||||
}
|
||||
if wants("nmf") {
|
||||
let sfc_http = http::Http::new(1000)?;
|
||||
let r = playlists::nmf_history(db, &sfc_http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 1500 }), &mut log);
|
||||
step("new music friday history (superfridaychart)", r.map(|n| format!("{n} ISRCs looked up")), &mut log);
|
||||
}
|
||||
if wants("hitlisten") {
|
||||
let r = hitlisten::collect(db, &http, if backfill { usize::MAX } else { 60 }, &mut log);
|
||||
step("hitlisten airplay chart", r.map(|n| format!("{n} weeks fetched")), &mut log);
|
||||
@@ -145,6 +155,12 @@ fn stats(db: &db::Db) -> Result<()> {
|
||||
(SELECT min(date) FROM spotify_chart_history WHERE country = 'DK' AND chart = 'daily')", [],
|
||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)))?;
|
||||
println!("Spotify Denmark: {sp_tracks} songs that charted, {sp_read} histories read, {sp_rows} DK chart rows (daily from {})", sp_from.unwrap_or_default());
|
||||
playlists::create_tables(db)?;
|
||||
let (snaps, nmf_isrcs, nmf_dk, nmf_from): (i64, i64, i64, Option<String>) = db.conn.query_row(
|
||||
"SELECT (SELECT count(DISTINCT playlist_id || taken_at) FROM playlist_snapshots), (SELECT count(*) FROM nmf_lookups),
|
||||
(SELECT count(*) FROM nmf_additions WHERE country = 'Denmark'), (SELECT min(friday) FROM nmf_additions)", [],
|
||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)))?;
|
||||
println!("Playlists: {snaps} snapshots; New Music Friday history: {nmf_isrcs} songs looked up, {nmf_dk} Denmark additions (since {})", nmf_from.unwrap_or_default());
|
||||
musicbrainz::create_tables(db)?;
|
||||
let (mb, mb_known): (i64, i64) = db.conn.query_row(
|
||||
"SELECT count(*), count(*) FILTER (WHERE recordings > 0) FROM mb_isrc", [], |r| Ok((r.get(0)?, r.get(1)?)))?;
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
//! Spotify editorial playlists:
|
||||
//!
|
||||
//! - Snapshots, from now on: the full track list of New Music Friday Denmark and Top 50 Denmark,
|
||||
//! read from Spotify's public embed page (open.spotify.com/embed/playlist/{id}, no login).
|
||||
//! Daily runs store a snapshot whenever the list changed.
|
||||
//! - History, since January 2022: SuperFridayChart (superfridaychart.com), which records every
|
||||
//! country's New Music Friday additions. Looked up per song by ISRC (from Deezer):
|
||||
//! /track/{isrc} lists each Friday the song was added, with country, playlist, followers, position.
|
||||
|
||||
use crate::{db::Db, http::Http};
|
||||
use anyhow::{anyhow, Result};
|
||||
use chrono::{NaiveDate, Utc};
|
||||
use regex::Regex;
|
||||
use rusqlite::params;
|
||||
use serde_json::Value;
|
||||
|
||||
pub const PLAYLISTS: &[(&str, &str)] = &[
|
||||
("37i9dQZF1DWWuIAAkPM0Lu", "New Music Friday Denmark"),
|
||||
("37i9dQZEVXbL3J0k32lWnN", "Top 50 - Denmark"),
|
||||
];
|
||||
|
||||
pub fn create_tables(db: &Db) -> Result<()> {
|
||||
db.conn.execute_batch(
|
||||
"CREATE TABLE IF NOT EXISTS playlist_snapshots (
|
||||
playlist_id TEXT NOT NULL, name TEXT NOT NULL, taken_at TEXT NOT NULL,
|
||||
position INTEGER NOT NULL, track_id TEXT NOT NULL, title TEXT, artists TEXT, duration_ms INTEGER,
|
||||
PRIMARY KEY (playlist_id, taken_at, position)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS nmf_additions ( -- SuperFridayChart
|
||||
isrc TEXT NOT NULL, friday TEXT NOT NULL, country TEXT NOT NULL,
|
||||
playlist_id TEXT, followers INTEGER, position INTEGER,
|
||||
PRIMARY KEY (isrc, friday, country)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS nmf_lookups (
|
||||
isrc TEXT PRIMARY KEY, found INTEGER NOT NULL, fetched_at TEXT NOT NULL
|
||||
);",
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A playlist's tracks in order: (track id, title, artists, duration ms).
|
||||
fn read_embed(http: &Http, id: &str) -> Result<Vec<(String, String, String, Option<i64>)>> {
|
||||
let html = http.get(&format!("https://open.spotify.com/embed/playlist/{id}"))?.ok_or_else(|| anyhow!("playlist {id} not found"))?;
|
||||
let re = Regex::new(r#"(?s)<script id="__NEXT_DATA__"[^>]*>(.*?)</script>"#)?;
|
||||
let data: Value = serde_json::from_str(&re.captures(&html).ok_or_else(|| anyhow!("no data in the embed page"))?[1])?;
|
||||
let list = data["props"]["pageProps"]["state"]["data"]["entity"]["trackList"].as_array().cloned().unwrap_or_default();
|
||||
Ok(list.iter().map(|t| (
|
||||
t["uri"].as_str().unwrap_or_default().rsplit(':').next().unwrap_or_default().to_string(),
|
||||
t["title"].as_str().unwrap_or_default().to_string(),
|
||||
t["subtitle"].as_str().unwrap_or_default().replace('\u{a0}', " "),
|
||||
t["duration"].as_i64(),
|
||||
)).collect())
|
||||
}
|
||||
|
||||
/// Stores a snapshot of each playlist if it differs from the last one stored.
|
||||
pub fn snapshot(db: &Db, http: &Http, log: &mut Vec<String>) -> Result<usize> {
|
||||
create_tables(db)?;
|
||||
let mut stored = 0;
|
||||
for &(id, name) in PLAYLISTS {
|
||||
let tracks = match read_embed(http, id) {
|
||||
Ok(t) if !t.is_empty() => t,
|
||||
Ok(_) => { log.push(format!("playlist {name}: empty")); continue; }
|
||||
Err(e) => { log.push(format!("playlist {name}: {e}")); continue; }
|
||||
};
|
||||
let last: Vec<String> = db.conn.prepare(
|
||||
"SELECT track_id FROM playlist_snapshots WHERE playlist_id = ?1
|
||||
AND taken_at = (SELECT max(taken_at) FROM playlist_snapshots WHERE playlist_id = ?1) ORDER BY position")?
|
||||
.query_map([id], |r| r.get(0))?.collect::<Result<_, _>>()?;
|
||||
if last == tracks.iter().map(|t| t.0.clone()).collect::<Vec<_>>() { continue; }
|
||||
let now = Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
let tx = db.conn.unchecked_transaction()?;
|
||||
for (i, (track, title, artists, dur)) in tracks.iter().enumerate() {
|
||||
tx.execute("INSERT OR REPLACE INTO playlist_snapshots VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
|
||||
params![id, name, now, i as i64 + 1, track, title, artists, dur])?;
|
||||
}
|
||||
tx.commit()?;
|
||||
stored += 1;
|
||||
}
|
||||
Ok(stored)
|
||||
}
|
||||
|
||||
/// New Music Friday additions per ISRC from SuperFridayChart (every ISRC Deezer gave us, once).
|
||||
pub fn nmf_history(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) -> Result<usize> {
|
||||
create_tables(db)?;
|
||||
crate::deezer::create_tables(db)?;
|
||||
let todo: Vec<String> = db.conn.prepare(
|
||||
"SELECT DISTINCT isrc FROM deezer_tracks WHERE coalesce(isrc, '') != '' AND isrc NOT IN (SELECT isrc FROM nmf_lookups)")?
|
||||
.query_map([], |r| r.get(0))?.collect::<Result<_, _>>()?;
|
||||
let day = Regex::new(r#"<h2[^>]*>\s*(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday), ([A-Za-z]+ \d{2}, \d{4})\s*</h2>"#)?;
|
||||
let row = Regex::new(r#"(?s)<p[^>]*>([^<]+)</p>\s*<a href="https://open\.spotify\.com/playlist/([A-Za-z0-9]{22})".*?<span[^>]*>([\d,]+) followers</span>\s*<span[^>]*>Position (\d+)</span>"#)?;
|
||||
let mut done = 0;
|
||||
for isrc in todo.into_iter().take(limit) {
|
||||
let html = match http.get(&format!("https://www.superfridaychart.com/track/{isrc}")) {
|
||||
Ok(h) => h,
|
||||
Err(e) => { log.push(format!("superfridaychart {isrc}: {e}")); continue; }
|
||||
};
|
||||
let mut found = 0;
|
||||
if let Some(html) = html {
|
||||
// Each Friday is an <h2> date followed by its table of playlists.
|
||||
let heads: Vec<(usize, NaiveDate)> = day.captures_iter(&html)
|
||||
.filter_map(|c| Some((c.get(0)?.start(), NaiveDate::parse_from_str(&c[1], "%B %d, %Y").ok()?)))
|
||||
.collect();
|
||||
for (k, (start, friday)) in heads.iter().enumerate() {
|
||||
let end = heads.get(k + 1).map(|h| h.0).unwrap_or(html.len());
|
||||
for r in row.captures_iter(&html[*start..end]) {
|
||||
db.conn.execute("INSERT OR REPLACE INTO nmf_additions VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
|
||||
params![isrc, friday.to_string(), r[1].trim(), &r[2], r[3].replace(',', "").parse::<i64>().ok(), r[4].parse::<i64>().ok()])?;
|
||||
found += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
db.conn.execute("INSERT OR REPLACE INTO nmf_lookups VALUES (?1, ?2, ?3)", params![isrc, found, Utc::now().to_rfc3339()])?;
|
||||
done += 1;
|
||||
}
|
||||
Ok(done)
|
||||
}
|
||||
Reference in New Issue
Block a user