NMF: per-artist New Music Friday history (SuperFridayChart artist pages, complete per artist), track pages for window placements first; NMF Denmark vs DR adds analysis (airplay model nmf)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-15 03:36:05 +00:00
co-authored by Claude Opus 5.5
parent 5b7e544b1c
commit a9fd04e113
4 changed files with 285 additions and 3 deletions
+58 -1
View File
@@ -83,8 +83,13 @@ pub fn snapshot(db: &Db, http: &Http, log: &mut Vec<String>) -> Result<usize> {
pub fn nmf_history(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) -> Result<usize> {
create_tables(db)?;
crate::deezer::create_tables(db)?;
db.conn.execute_batch("CREATE TABLE IF NOT EXISTS nmf_artist_tracks (spotify_artist_id TEXT NOT NULL, isrc TEXT NOT NULL, title TEXT,
friday TEXT NOT NULL, playlists INTEGER, followers INTEGER, PRIMARY KEY (spotify_artist_id, isrc, friday));")?;
// Placements found on artist pages inside the modelling window first, then every ISRC Deezer gave us.
let todo: Vec<String> = db.conn.prepare(
"SELECT DISTINCT isrc FROM deezer_tracks WHERE coalesce(isrc, '') != '' AND isrc NOT IN (SELECT isrc FROM nmf_lookups)")?
"SELECT isrc FROM (SELECT isrc, 0 AS o FROM nmf_artist_tracks WHERE friday >= '2025-11-01'
UNION SELECT isrc, 1 FROM deezer_tracks WHERE coalesce(isrc, '') != '')
WHERE isrc NOT IN (SELECT isrc FROM nmf_lookups) GROUP BY isrc ORDER BY min(o)")?
.query_map([], |r| r.get(0))?.collect::<Result<_, _>>()?;
let day = Regex::new(r#"<h2[^>]*>\s*(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday), ([A-Za-z]+ \d{2}, \d{4})\s*</h2>"#)?;
let row = Regex::new(r#"(?s)<p[^>]*>([^<]+)</p>\s*<a href="https://open\.spotify\.com/playlist/([A-Za-z0-9]{22})".*?<span[^>]*>([\d,]+) followers</span>\s*<span[^>]*>Position (\d+)</span>"#)?;
@@ -114,3 +119,55 @@ pub fn nmf_history(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) ->
}
Ok(done)
}
/// New Music Friday placements per artist (SuperFridayChart's artist page, by the Spotify artist
/// id DR links on its artist page): every track of the artist added to any NMF playlist since
/// 2022, with the Friday, number of playlists and their followers. Complete per artist, so songs
/// DR never played are in it too. Which countries: the track page (`nmf_history`), fetched for
/// placements inside the modelling window.
pub fn nmf_artists(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) -> Result<usize> {
create_tables(db)?;
db.conn.execute_batch(
"CREATE TABLE IF NOT EXISTS nmf_artist_tracks (
spotify_artist_id TEXT NOT NULL, isrc TEXT NOT NULL, title TEXT, friday TEXT NOT NULL,
playlists INTEGER, followers INTEGER,
PRIMARY KEY (spotify_artist_id, isrc, friday)
);
CREATE TABLE IF NOT EXISTS nmf_artist_lookups (
dr_artist_id TEXT PRIMARY KEY, spotify_artist_id TEXT, tracks INTEGER, fetched_at TEXT NOT NULL
);",
)?;
let link = Regex::new(r"open\.spotify\.com/artist/([A-Za-z0-9]{22})")?;
let todo: Vec<(String, String)> = db.conn.prepare(
"SELECT artist_id, raw FROM dr_artists WHERE artist_id NOT IN (SELECT dr_artist_id FROM nmf_artist_lookups)")?
.query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?.collect::<Result<_, _>>()?;
// One row per track: the track link, then its date, playlist count and followers.
let row = Regex::new(r#"(?s)/track/([A-Z0-9]{12})"[^>]*>([^<]*)</a>.*?([A-Z][a-z]{2} \d{2}, \d{4})\s*</?[^>]*>?.*?(\d+) playlists.*?([\d,]+) followers"#)?;
let mut done = 0;
for (dr_id, raw) in todo.into_iter().take(limit) {
let sid = link.captures(&raw).map(|c| c[1].to_string());
let mut n = 0;
if let Some(sid) = &sid {
match http.get(&format!("https://www.superfridaychart.com/artist/{sid}")) {
Ok(Some(html)) => {
// Split at each track link so a row's fields can't run into the next track.
let starts: Vec<usize> = html.match_indices("superfridaychart.com/track/").map(|(i, _)| i).collect();
for (k, s) in starts.iter().enumerate() {
let block = &html[s.saturating_sub(1)..starts.get(k + 1).copied().unwrap_or(html.len())];
let Some(c) = row.captures(block) else { continue };
let Ok(d) = NaiveDate::parse_from_str(&c[3], "%b %d, %Y") else { continue };
db.conn.execute("INSERT OR REPLACE INTO nmf_artist_tracks VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
params![sid, &c[1], c[2].trim().replace("&amp;", "&").replace("&#039;", "'"), d.to_string(),
c[4].parse::<i64>().ok(), c[5].replace(',', "").parse::<i64>().ok()])?;
n += 1;
}
}
Ok(None) => {}
Err(e) => { log.push(format!("superfridaychart artist {sid}: {e}")); continue; }
}
}
db.conn.execute("INSERT OR REPLACE INTO nmf_artist_lookups VALUES (?1, ?2, ?3, ?4)", params![dr_id, sid, n as i64, Utc::now().to_rfc3339()])?;
done += 1;
}
Ok(done)
}