NMF: per-artist New Music Friday history (SuperFridayChart artist pages, complete per artist), track pages for window placements first; NMF Denmark vs DR adds analysis (airplay model nmf)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+58
-1
@@ -83,8 +83,13 @@ pub fn snapshot(db: &Db, http: &Http, log: &mut Vec<String>) -> Result<usize> {
|
||||
pub fn nmf_history(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) -> Result<usize> {
|
||||
create_tables(db)?;
|
||||
crate::deezer::create_tables(db)?;
|
||||
db.conn.execute_batch("CREATE TABLE IF NOT EXISTS nmf_artist_tracks (spotify_artist_id TEXT NOT NULL, isrc TEXT NOT NULL, title TEXT,
|
||||
friday TEXT NOT NULL, playlists INTEGER, followers INTEGER, PRIMARY KEY (spotify_artist_id, isrc, friday));")?;
|
||||
// Placements found on artist pages inside the modelling window first, then every ISRC Deezer gave us.
|
||||
let todo: Vec<String> = db.conn.prepare(
|
||||
"SELECT DISTINCT isrc FROM deezer_tracks WHERE coalesce(isrc, '') != '' AND isrc NOT IN (SELECT isrc FROM nmf_lookups)")?
|
||||
"SELECT isrc FROM (SELECT isrc, 0 AS o FROM nmf_artist_tracks WHERE friday >= '2025-11-01'
|
||||
UNION SELECT isrc, 1 FROM deezer_tracks WHERE coalesce(isrc, '') != '')
|
||||
WHERE isrc NOT IN (SELECT isrc FROM nmf_lookups) GROUP BY isrc ORDER BY min(o)")?
|
||||
.query_map([], |r| r.get(0))?.collect::<Result<_, _>>()?;
|
||||
let day = Regex::new(r#"<h2[^>]*>\s*(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday), ([A-Za-z]+ \d{2}, \d{4})\s*</h2>"#)?;
|
||||
let row = Regex::new(r#"(?s)<p[^>]*>([^<]+)</p>\s*<a href="https://open\.spotify\.com/playlist/([A-Za-z0-9]{22})".*?<span[^>]*>([\d,]+) followers</span>\s*<span[^>]*>Position (\d+)</span>"#)?;
|
||||
@@ -114,3 +119,55 @@ pub fn nmf_history(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) ->
|
||||
}
|
||||
Ok(done)
|
||||
}
|
||||
|
||||
/// New Music Friday placements per artist (SuperFridayChart's artist page, by the Spotify artist
|
||||
/// id DR links on its artist page): every track of the artist added to any NMF playlist since
|
||||
/// 2022, with the Friday, number of playlists and their followers. Complete per artist, so songs
|
||||
/// DR never played are in it too. Which countries: the track page (`nmf_history`), fetched for
|
||||
/// placements inside the modelling window.
|
||||
pub fn nmf_artists(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) -> Result<usize> {
|
||||
create_tables(db)?;
|
||||
db.conn.execute_batch(
|
||||
"CREATE TABLE IF NOT EXISTS nmf_artist_tracks (
|
||||
spotify_artist_id TEXT NOT NULL, isrc TEXT NOT NULL, title TEXT, friday TEXT NOT NULL,
|
||||
playlists INTEGER, followers INTEGER,
|
||||
PRIMARY KEY (spotify_artist_id, isrc, friday)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS nmf_artist_lookups (
|
||||
dr_artist_id TEXT PRIMARY KEY, spotify_artist_id TEXT, tracks INTEGER, fetched_at TEXT NOT NULL
|
||||
);",
|
||||
)?;
|
||||
let link = Regex::new(r"open\.spotify\.com/artist/([A-Za-z0-9]{22})")?;
|
||||
let todo: Vec<(String, String)> = db.conn.prepare(
|
||||
"SELECT artist_id, raw FROM dr_artists WHERE artist_id NOT IN (SELECT dr_artist_id FROM nmf_artist_lookups)")?
|
||||
.query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?.collect::<Result<_, _>>()?;
|
||||
// One row per track: the track link, then its date, playlist count and followers.
|
||||
let row = Regex::new(r#"(?s)/track/([A-Z0-9]{12})"[^>]*>([^<]*)</a>.*?([A-Z][a-z]{2} \d{2}, \d{4})\s*</?[^>]*>?.*?(\d+) playlists.*?([\d,]+) followers"#)?;
|
||||
let mut done = 0;
|
||||
for (dr_id, raw) in todo.into_iter().take(limit) {
|
||||
let sid = link.captures(&raw).map(|c| c[1].to_string());
|
||||
let mut n = 0;
|
||||
if let Some(sid) = &sid {
|
||||
match http.get(&format!("https://www.superfridaychart.com/artist/{sid}")) {
|
||||
Ok(Some(html)) => {
|
||||
// Split at each track link so a row's fields can't run into the next track.
|
||||
let starts: Vec<usize> = html.match_indices("superfridaychart.com/track/").map(|(i, _)| i).collect();
|
||||
for (k, s) in starts.iter().enumerate() {
|
||||
let block = &html[s.saturating_sub(1)..starts.get(k + 1).copied().unwrap_or(html.len())];
|
||||
let Some(c) = row.captures(block) else { continue };
|
||||
let Ok(d) = NaiveDate::parse_from_str(&c[3], "%b %d, %Y") else { continue };
|
||||
db.conn.execute("INSERT OR REPLACE INTO nmf_artist_tracks VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
|
||||
params![sid, &c[1], c[2].trim().replace("&", "&").replace("'", "'"), d.to_string(),
|
||||
c[4].parse::<i64>().ok(), c[5].replace(',', "").parse::<i64>().ok()])?;
|
||||
n += 1;
|
||||
}
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(e) => { log.push(format!("superfridaychart artist {sid}: {e}")); continue; }
|
||||
}
|
||||
}
|
||||
db.conn.execute("INSERT OR REPLACE INTO nmf_artist_lookups VALUES (?1, ?2, ?3, ?4)", params![dr_id, sid, n as i64, Utc::now().to_rfc3339()])?;
|
||||
done += 1;
|
||||
}
|
||||
Ok(done)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user