Wave features (time since the last DR add / Airplay Top-20 wave, peak band 13-25 weeks, 0.5-1.25x the artist's usual gap); ablation switch (model adds --without rel,wave); fix the per-channel row slice
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+60
-4
@@ -85,12 +85,35 @@ pub fn nmf_history(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) ->
|
||||
crate::deezer::create_tables(db)?;
|
||||
db.conn.execute_batch("CREATE TABLE IF NOT EXISTS nmf_artist_tracks (spotify_artist_id TEXT NOT NULL, isrc TEXT NOT NULL, title TEXT,
|
||||
friday TEXT NOT NULL, playlists INTEGER, followers INTEGER, PRIMARY KEY (spotify_artist_id, isrc, friday));")?;
|
||||
// Placements found on artist pages inside the modelling window first, then every ISRC Deezer gave us.
|
||||
// Title and artists of each looked-up track (added later; older rows have them empty).
|
||||
for col in ["title TEXT", "artists TEXT"] {
|
||||
let _ = db.conn.execute(&format!("ALTER TABLE nmf_lookups ADD COLUMN {col}"), []);
|
||||
}
|
||||
db.conn.execute_batch("CREATE TABLE IF NOT EXISTS nmf_global_chart (friday TEXT NOT NULL, rank INTEGER NOT NULL, isrc TEXT NOT NULL,
|
||||
PRIMARY KEY (friday, rank));")?;
|
||||
// Which tracks to read, most likely to be on NMF Denmark first:
|
||||
// 0 artist-page placements of Danish acts (DR artist country), or one playlist the size of
|
||||
// NMF Denmark that Friday (the most common single-playlist follower count among Danish acts)
|
||||
// 1 SuperFridayChart's weekly global top 50
|
||||
// 2 other artist-page placements on several playlists
|
||||
// 3 every ISRC Deezer gave us
|
||||
let todo: Vec<String> = db.conn.prepare(
|
||||
"SELECT isrc FROM (SELECT isrc, 0 AS o FROM nmf_artist_tracks WHERE friday >= '2025-11-01'
|
||||
UNION SELECT isrc, 1 FROM deezer_tracks WHERE coalesce(isrc, '') != '')
|
||||
"WITH danish AS (SELECT l.spotify_artist_id FROM nmf_artist_lookups l JOIN dr_artists a ON a.artist_id = l.dr_artist_id
|
||||
WHERE json_extract(a.raw, '$.artist.country') = 'Danmark' AND l.spotify_artist_id IS NOT NULL),
|
||||
dk_size AS (SELECT friday, followers FROM (
|
||||
SELECT t.friday, t.followers, count(*) n, row_number() OVER (PARTITION BY t.friday ORDER BY count(*) DESC) rn
|
||||
FROM nmf_artist_tracks t WHERE t.playlists = 1 AND t.spotify_artist_id IN (SELECT spotify_artist_id FROM danish)
|
||||
GROUP BY t.friday, t.followers) WHERE rn = 1)
|
||||
SELECT isrc FROM (
|
||||
SELECT t.isrc, 0 AS o FROM nmf_artist_tracks t LEFT JOIN dk_size d ON d.friday = t.friday
|
||||
WHERE t.spotify_artist_id IN (SELECT spotify_artist_id FROM danish)
|
||||
OR (t.playlists = 1 AND abs(t.followers - d.followers) <= 0.02 * d.followers)
|
||||
UNION ALL SELECT isrc, 1 FROM nmf_global_chart
|
||||
UNION ALL SELECT isrc, 2 FROM nmf_artist_tracks WHERE playlists > 1
|
||||
UNION ALL SELECT isrc, 3 FROM deezer_tracks WHERE coalesce(isrc, '') != '')
|
||||
WHERE isrc NOT IN (SELECT isrc FROM nmf_lookups) GROUP BY isrc ORDER BY min(o)")?
|
||||
.query_map([], |r| r.get(0))?.collect::<Result<_, _>>()?;
|
||||
let meta = Regex::new(r#"<meta name="description" content="Discover the Spotify New Music Friday playlists that added the track "(.*?)" from (.*?)\.">"#)?;
|
||||
let day = Regex::new(r#"<h2[^>]*>\s*(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday), ([A-Za-z]+ \d{2}, \d{4})\s*</h2>"#)?;
|
||||
let row = Regex::new(r#"(?s)<p[^>]*>([^<]+)</p>\s*<a href="https://open\.spotify\.com/playlist/([A-Za-z0-9]{22})".*?<span[^>]*>([\d,]+) followers</span>\s*<span[^>]*>Position (\d+)</span>"#)?;
|
||||
let mut done = 0;
|
||||
@@ -100,7 +123,13 @@ pub fn nmf_history(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) ->
|
||||
Err(e) => { log.push(format!("superfridaychart {isrc}: {e}")); continue; }
|
||||
};
|
||||
let mut found = 0;
|
||||
let (mut title, mut artists) = (None, None);
|
||||
if let Some(html) = html {
|
||||
if let Some(m) = meta.captures(&html) {
|
||||
let un = |x: &str| x.replace("&", "&").replace("'", "'").replace(""", "\"");
|
||||
title = Some(un(&m[1]));
|
||||
artists = Some(un(&m[2]));
|
||||
}
|
||||
// Each Friday is an <h2> date followed by its table of playlists.
|
||||
let heads: Vec<(usize, NaiveDate)> = day.captures_iter(&html)
|
||||
.filter_map(|c| Some((c.get(0)?.start(), NaiveDate::parse_from_str(&c[1], "%B %d, %Y").ok()?)))
|
||||
@@ -114,7 +143,8 @@ pub fn nmf_history(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) ->
|
||||
}
|
||||
}
|
||||
}
|
||||
db.conn.execute("INSERT OR REPLACE INTO nmf_lookups VALUES (?1, ?2, ?3)", params![isrc, found, Utc::now().to_rfc3339()])?;
|
||||
db.conn.execute("INSERT OR REPLACE INTO nmf_lookups (isrc, found, fetched_at, title, artists) VALUES (?1, ?2, ?3, ?4, ?5)",
|
||||
params![isrc, found, Utc::now().to_rfc3339(), title, artists])?;
|
||||
done += 1;
|
||||
}
|
||||
Ok(done)
|
||||
@@ -171,3 +201,29 @@ pub fn nmf_artists(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) ->
|
||||
}
|
||||
Ok(done)
|
||||
}
|
||||
|
||||
/// SuperFridayChart's weekly global chart (top 50 tracks across all NMF playlists), every Friday
|
||||
/// listed on its front page: candidate ISRCs for the NMF Denmark lists.
|
||||
pub fn nmf_global_charts(db: &Db, http: &Http, log: &mut Vec<String>) -> Result<usize> {
|
||||
db.conn.execute_batch("CREATE TABLE IF NOT EXISTS nmf_global_chart (friday TEXT NOT NULL, rank INTEGER NOT NULL, isrc TEXT NOT NULL,
|
||||
PRIMARY KEY (friday, rank));")?;
|
||||
let front = http.get("https://www.superfridaychart.com/")?.unwrap_or_default();
|
||||
let mut fridays: Vec<String> = Regex::new(r"/chart/(\d{4}-\d{2}-\d{2})")?.captures_iter(&front).map(|c| c[1].to_string()).collect();
|
||||
fridays.sort(); fridays.dedup();
|
||||
let track = Regex::new(r"/track/([A-Z0-9]{12})")?;
|
||||
let mut n = 0;
|
||||
for f in fridays {
|
||||
let have: i64 = db.conn.query_row("SELECT count(*) FROM nmf_global_chart WHERE friday = ?1", [&f], |r| r.get(0))?;
|
||||
if have > 0 { continue; }
|
||||
let html = match http.get(&format!("https://www.superfridaychart.com/chart/{f}")) {
|
||||
Ok(Some(h)) => h, Ok(None) => continue, Err(e) => { log.push(format!("superfridaychart chart {f}: {e}")); continue; }
|
||||
};
|
||||
let mut seen = Vec::new();
|
||||
for c in track.captures_iter(&html) { if !seen.contains(&c[1].to_string()) { seen.push(c[1].to_string()); } }
|
||||
for (i, isrc) in seen.iter().enumerate() {
|
||||
db.conn.execute("INSERT OR REPLACE INTO nmf_global_chart VALUES (?1, ?2, ?3)", params![f, i as i64 + 1, isrc])?;
|
||||
}
|
||||
n += 1;
|
||||
}
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user