230 lines
14 KiB
Rust
230 lines
14 KiB
Rust
//! Spotify editorial playlists:
|
|
//!
|
|
//! - Snapshots, from now on: the full track list of New Music Friday Denmark and Top 50 Denmark,
|
|
//! read from Spotify's public embed page (open.spotify.com/embed/playlist/{id}, no login).
|
|
//! Daily runs store a snapshot whenever the list changed.
|
|
//! - History, since January 2022: SuperFridayChart (superfridaychart.com), which records every
|
|
//! country's New Music Friday additions. Looked up per song by ISRC (from Deezer):
|
|
//! /track/{isrc} lists each Friday the song was added, with country, playlist, followers, position.
|
|
|
|
use crate::{db::Db, http::Http};
|
|
use anyhow::{anyhow, Result};
|
|
use chrono::{NaiveDate, Utc};
|
|
use regex::Regex;
|
|
use rusqlite::params;
|
|
use serde_json::Value;
|
|
|
|
pub const PLAYLISTS: &[(&str, &str)] = &[
|
|
("37i9dQZF1DWWuIAAkPM0Lu", "New Music Friday Denmark"),
|
|
("37i9dQZEVXbL3J0k32lWnN", "Top 50 - Denmark"),
|
|
];
|
|
|
|
pub fn create_tables(db: &Db) -> Result<()> {
|
|
db.conn.execute_batch(
|
|
"CREATE TABLE IF NOT EXISTS playlist_snapshots (
|
|
playlist_id TEXT NOT NULL, name TEXT NOT NULL, taken_at TEXT NOT NULL,
|
|
position INTEGER NOT NULL, track_id TEXT NOT NULL, title TEXT, artists TEXT, duration_ms INTEGER,
|
|
PRIMARY KEY (playlist_id, taken_at, position)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS nmf_additions ( -- SuperFridayChart
|
|
isrc TEXT NOT NULL, friday TEXT NOT NULL, country TEXT NOT NULL,
|
|
playlist_id TEXT, followers INTEGER, position INTEGER,
|
|
PRIMARY KEY (isrc, friday, country)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS nmf_lookups (
|
|
isrc TEXT PRIMARY KEY, found INTEGER NOT NULL, fetched_at TEXT NOT NULL
|
|
);",
|
|
)?;
|
|
Ok(())
|
|
}
|
|
|
|
/// A playlist's tracks in order: (track id, title, artists, duration ms).
|
|
fn read_embed(http: &Http, id: &str) -> Result<Vec<(String, String, String, Option<i64>)>> {
|
|
let html = http.get(&format!("https://open.spotify.com/embed/playlist/{id}"))?.ok_or_else(|| anyhow!("playlist {id} not found"))?;
|
|
let re = Regex::new(r#"(?s)<script id="__NEXT_DATA__"[^>]*>(.*?)</script>"#)?;
|
|
let data: Value = serde_json::from_str(&re.captures(&html).ok_or_else(|| anyhow!("no data in the embed page"))?[1])?;
|
|
let list = data["props"]["pageProps"]["state"]["data"]["entity"]["trackList"].as_array().cloned().unwrap_or_default();
|
|
Ok(list.iter().map(|t| (
|
|
t["uri"].as_str().unwrap_or_default().rsplit(':').next().unwrap_or_default().to_string(),
|
|
t["title"].as_str().unwrap_or_default().to_string(),
|
|
t["subtitle"].as_str().unwrap_or_default().replace('\u{a0}', " "),
|
|
t["duration"].as_i64(),
|
|
)).collect())
|
|
}
|
|
|
|
/// Stores a snapshot of each playlist if it differs from the last one stored.
|
|
pub fn snapshot(db: &Db, http: &Http, log: &mut Vec<String>) -> Result<usize> {
|
|
create_tables(db)?;
|
|
let mut stored = 0;
|
|
for &(id, name) in PLAYLISTS {
|
|
let tracks = match read_embed(http, id) {
|
|
Ok(t) if !t.is_empty() => t,
|
|
Ok(_) => { log.push(format!("playlist {name}: empty")); continue; }
|
|
Err(e) => { log.push(format!("playlist {name}: {e}")); continue; }
|
|
};
|
|
let last: Vec<String> = db.conn.prepare(
|
|
"SELECT track_id FROM playlist_snapshots WHERE playlist_id = ?1
|
|
AND taken_at = (SELECT max(taken_at) FROM playlist_snapshots WHERE playlist_id = ?1) ORDER BY position")?
|
|
.query_map([id], |r| r.get(0))?.collect::<Result<_, _>>()?;
|
|
if last == tracks.iter().map(|t| t.0.clone()).collect::<Vec<_>>() { continue; }
|
|
let now = Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string();
|
|
let tx = db.conn.unchecked_transaction()?;
|
|
for (i, (track, title, artists, dur)) in tracks.iter().enumerate() {
|
|
tx.execute("INSERT OR REPLACE INTO playlist_snapshots VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
|
|
params![id, name, now, i as i64 + 1, track, title, artists, dur])?;
|
|
}
|
|
tx.commit()?;
|
|
stored += 1;
|
|
}
|
|
Ok(stored)
|
|
}
|
|
|
|
/// New Music Friday additions per ISRC from SuperFridayChart (every ISRC Deezer gave us, once).
|
|
pub fn nmf_history(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) -> Result<usize> {
|
|
create_tables(db)?;
|
|
crate::deezer::create_tables(db)?;
|
|
db.conn.execute_batch("CREATE TABLE IF NOT EXISTS nmf_artist_tracks (spotify_artist_id TEXT NOT NULL, isrc TEXT NOT NULL, title TEXT,
|
|
friday TEXT NOT NULL, playlists INTEGER, followers INTEGER, PRIMARY KEY (spotify_artist_id, isrc, friday));")?;
|
|
// Title and artists of each looked-up track (added later; older rows have them empty).
|
|
for col in ["title TEXT", "artists TEXT"] {
|
|
let _ = db.conn.execute(&format!("ALTER TABLE nmf_lookups ADD COLUMN {col}"), []);
|
|
}
|
|
db.conn.execute_batch("CREATE TABLE IF NOT EXISTS nmf_global_chart (friday TEXT NOT NULL, rank INTEGER NOT NULL, isrc TEXT NOT NULL,
|
|
PRIMARY KEY (friday, rank));")?;
|
|
// Which tracks to read, most likely to be on NMF Denmark first:
|
|
// 0 artist-page placements of Danish acts (DR artist country), or one playlist the size of
|
|
// NMF Denmark that Friday (the most common single-playlist follower count among Danish acts)
|
|
// 1 SuperFridayChart's weekly global top 50
|
|
// 2 other artist-page placements on several playlists
|
|
// 3 every ISRC Deezer gave us
|
|
let todo: Vec<String> = db.conn.prepare(
|
|
"WITH danish AS (SELECT l.spotify_artist_id FROM nmf_artist_lookups l JOIN dr_artists a ON a.artist_id = l.dr_artist_id
|
|
WHERE json_extract(a.raw, '$.artist.country') = 'Danmark' AND l.spotify_artist_id IS NOT NULL),
|
|
dk_size AS (SELECT friday, followers FROM (
|
|
SELECT t.friday, t.followers, count(*) n, row_number() OVER (PARTITION BY t.friday ORDER BY count(*) DESC) rn
|
|
FROM nmf_artist_tracks t WHERE t.playlists = 1 AND t.spotify_artist_id IN (SELECT spotify_artist_id FROM danish)
|
|
GROUP BY t.friday, t.followers) WHERE rn = 1)
|
|
SELECT isrc FROM (
|
|
SELECT t.isrc, 0 AS o FROM nmf_artist_tracks t LEFT JOIN dk_size d ON d.friday = t.friday
|
|
WHERE t.spotify_artist_id IN (SELECT spotify_artist_id FROM danish)
|
|
OR (t.playlists = 1 AND abs(t.followers - d.followers) <= 0.02 * d.followers)
|
|
UNION ALL SELECT isrc, 1 FROM nmf_global_chart
|
|
UNION ALL SELECT isrc, 2 FROM nmf_artist_tracks WHERE playlists > 1
|
|
UNION ALL SELECT isrc, 3 FROM deezer_tracks WHERE coalesce(isrc, '') != '')
|
|
WHERE isrc NOT IN (SELECT isrc FROM nmf_lookups) GROUP BY isrc ORDER BY min(o)")?
|
|
.query_map([], |r| r.get(0))?.collect::<Result<_, _>>()?;
|
|
let meta = Regex::new(r#"<meta name="description" content="Discover the Spotify New Music Friday playlists that added the track "(.*?)" from (.*?)\.">"#)?;
|
|
let day = Regex::new(r#"<h2[^>]*>\s*(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday), ([A-Za-z]+ \d{2}, \d{4})\s*</h2>"#)?;
|
|
let row = Regex::new(r#"(?s)<p[^>]*>([^<]+)</p>\s*<a href="https://open\.spotify\.com/playlist/([A-Za-z0-9]{22})".*?<span[^>]*>([\d,]+) followers</span>\s*<span[^>]*>Position (\d+)</span>"#)?;
|
|
let mut done = 0;
|
|
for isrc in todo.into_iter().take(limit) {
|
|
let html = match http.get(&format!("https://www.superfridaychart.com/track/{isrc}")) {
|
|
Ok(h) => h,
|
|
Err(e) => { log.push(format!("superfridaychart {isrc}: {e}")); continue; }
|
|
};
|
|
let mut found = 0;
|
|
let (mut title, mut artists) = (None, None);
|
|
if let Some(html) = html {
|
|
if let Some(m) = meta.captures(&html) {
|
|
let un = |x: &str| x.replace("&", "&").replace("'", "'").replace(""", "\"");
|
|
title = Some(un(&m[1]));
|
|
artists = Some(un(&m[2]));
|
|
}
|
|
// Each Friday is an <h2> date followed by its table of playlists.
|
|
let heads: Vec<(usize, NaiveDate)> = day.captures_iter(&html)
|
|
.filter_map(|c| Some((c.get(0)?.start(), NaiveDate::parse_from_str(&c[1], "%B %d, %Y").ok()?)))
|
|
.collect();
|
|
for (k, (start, friday)) in heads.iter().enumerate() {
|
|
let end = heads.get(k + 1).map(|h| h.0).unwrap_or(html.len());
|
|
for r in row.captures_iter(&html[*start..end]) {
|
|
db.conn.execute("INSERT OR REPLACE INTO nmf_additions VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
|
|
params![isrc, friday.to_string(), r[1].trim(), &r[2], r[3].replace(',', "").parse::<i64>().ok(), r[4].parse::<i64>().ok()])?;
|
|
found += 1;
|
|
}
|
|
}
|
|
}
|
|
db.conn.execute("INSERT OR REPLACE INTO nmf_lookups (isrc, found, fetched_at, title, artists) VALUES (?1, ?2, ?3, ?4, ?5)",
|
|
params![isrc, found, Utc::now().to_rfc3339(), title, artists])?;
|
|
done += 1;
|
|
}
|
|
Ok(done)
|
|
}
|
|
|
|
/// New Music Friday placements per artist (SuperFridayChart's artist page, by the Spotify artist
|
|
/// id DR links on its artist page): every track of the artist added to any NMF playlist since
|
|
/// 2022, with the Friday, number of playlists and their followers. Complete per artist, so songs
|
|
/// DR never played are in it too. Which countries: the track page (`nmf_history`), fetched for
|
|
/// placements inside the modelling window.
|
|
pub fn nmf_artists(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) -> Result<usize> {
|
|
create_tables(db)?;
|
|
db.conn.execute_batch(
|
|
"CREATE TABLE IF NOT EXISTS nmf_artist_tracks (
|
|
spotify_artist_id TEXT NOT NULL, isrc TEXT NOT NULL, title TEXT, friday TEXT NOT NULL,
|
|
playlists INTEGER, followers INTEGER,
|
|
PRIMARY KEY (spotify_artist_id, isrc, friday)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS nmf_artist_lookups (
|
|
dr_artist_id TEXT PRIMARY KEY, spotify_artist_id TEXT, tracks INTEGER, fetched_at TEXT NOT NULL
|
|
);",
|
|
)?;
|
|
let link = Regex::new(r"open\.spotify\.com/artist/([A-Za-z0-9]{22})")?;
|
|
let todo: Vec<(String, String)> = db.conn.prepare(
|
|
"SELECT artist_id, raw FROM dr_artists WHERE artist_id NOT IN (SELECT dr_artist_id FROM nmf_artist_lookups)")?
|
|
.query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?.collect::<Result<_, _>>()?;
|
|
// One row per track: the track link, then its date, playlist count and followers.
|
|
let row = Regex::new(r#"(?s)/track/([A-Z0-9]{12})"[^>]*>([^<]*)</a>.*?([A-Z][a-z]{2} \d{2}, \d{4})\s*</?[^>]*>?.*?(\d+) playlists.*?([\d,]+) followers"#)?;
|
|
let mut done = 0;
|
|
for (dr_id, raw) in todo.into_iter().take(limit) {
|
|
let sid = link.captures(&raw).map(|c| c[1].to_string());
|
|
let mut n = 0;
|
|
if let Some(sid) = &sid {
|
|
match http.get(&format!("https://www.superfridaychart.com/artist/{sid}")) {
|
|
Ok(Some(html)) => {
|
|
// Split at each track link so a row's fields can't run into the next track.
|
|
let starts: Vec<usize> = html.match_indices("superfridaychart.com/track/").map(|(i, _)| i).collect();
|
|
for (k, s) in starts.iter().enumerate() {
|
|
let block = &html[s.saturating_sub(1)..starts.get(k + 1).copied().unwrap_or(html.len())];
|
|
let Some(c) = row.captures(block) else { continue };
|
|
let Ok(d) = NaiveDate::parse_from_str(&c[3], "%b %d, %Y") else { continue };
|
|
db.conn.execute("INSERT OR REPLACE INTO nmf_artist_tracks VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
|
|
params![sid, &c[1], c[2].trim().replace("&", "&").replace("'", "'"), d.to_string(),
|
|
c[4].parse::<i64>().ok(), c[5].replace(',', "").parse::<i64>().ok()])?;
|
|
n += 1;
|
|
}
|
|
}
|
|
Ok(None) => {}
|
|
Err(e) => { log.push(format!("superfridaychart artist {sid}: {e}")); continue; }
|
|
}
|
|
}
|
|
db.conn.execute("INSERT OR REPLACE INTO nmf_artist_lookups VALUES (?1, ?2, ?3, ?4)", params![dr_id, sid, n as i64, Utc::now().to_rfc3339()])?;
|
|
done += 1;
|
|
}
|
|
Ok(done)
|
|
}
|
|
|
|
/// SuperFridayChart's weekly global chart (top 50 tracks across all NMF playlists), every Friday
|
|
/// listed on its front page: candidate ISRCs for the NMF Denmark lists.
|
|
pub fn nmf_global_charts(db: &Db, http: &Http, log: &mut Vec<String>) -> Result<usize> {
|
|
db.conn.execute_batch("CREATE TABLE IF NOT EXISTS nmf_global_chart (friday TEXT NOT NULL, rank INTEGER NOT NULL, isrc TEXT NOT NULL,
|
|
PRIMARY KEY (friday, rank));")?;
|
|
let front = http.get("https://www.superfridaychart.com/")?.unwrap_or_default();
|
|
let mut fridays: Vec<String> = Regex::new(r"/chart/(\d{4}-\d{2}-\d{2})")?.captures_iter(&front).map(|c| c[1].to_string()).collect();
|
|
fridays.sort(); fridays.dedup();
|
|
let track = Regex::new(r"/track/([A-Z0-9]{12})")?;
|
|
let mut n = 0;
|
|
for f in fridays {
|
|
let have: i64 = db.conn.query_row("SELECT count(*) FROM nmf_global_chart WHERE friday = ?1", [&f], |r| r.get(0))?;
|
|
if have > 0 { continue; }
|
|
let html = match http.get(&format!("https://www.superfridaychart.com/chart/{f}")) {
|
|
Ok(Some(h)) => h, Ok(None) => continue, Err(e) => { log.push(format!("superfridaychart chart {f}: {e}")); continue; }
|
|
};
|
|
let mut seen = Vec::new();
|
|
for c in track.captures_iter(&html) { if !seen.contains(&c[1].to_string()) { seen.push(c[1].to_string()); } }
|
|
for (i, isrc) in seen.iter().enumerate() {
|
|
db.conn.execute("INSERT OR REPLACE INTO nmf_global_chart VALUES (?1, ?2, ?3)", params![f, i as i64 + 1, isrc])?;
|
|
}
|
|
n += 1;
|
|
}
|
|
Ok(n)
|
|
}
|