Wave features (time since the last DR add / Airplay Top-20 wave, peak band 13-25 weeks, 0.5-1.25x the artist's usual gap); ablation switch (model adds --without rel,wave); fix the per-channel row slice

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-15 03:56:10 +00:00
co-authored by Claude Opus 5.5
parent 7d656834fb
commit 992cc2b157
6 changed files with 209 additions and 15 deletions
+86
View File
@@ -0,0 +1,86 @@
//! Data exports.
//!
//! airplay export nmf-denmark [--out data/nmf_denmark.csv]
//!
//! New Music Friday Denmark, one row per Friday and position, as far back as SuperFridayChart
//! goes (January 2022): title, artists, ISRC, position, playlist followers. Rebuilt from
//! SuperFridayChart's track pages, so a week is only as complete as the tracks looked up so far;
//! a second file (…_coverage.csv) lists per Friday how many positions are found and which are
//! missing. The playlist snapshots taken from 15 Nov 2026 on are complete and are added with
//! source "spotify snapshot" (Spotify track id; ISRC where SuperFridayChart has the same song).
use crate::db::Db;
use anyhow::Result;
use std::collections::{BTreeMap, HashMap};
use std::io::Write;
fn csv(s: &str) -> String {
if s.contains([',', '"', '\n']) { format!("\"{}\"", s.replace('"', "\"\"")) } else { s.to_string() }
}
pub fn run(db: &Db, args: &[String]) -> Result<()> {
match args.get(2).map(String::as_str) {
Some("nmf-denmark") => {
let out = args.iter().position(|a| a == "--out").and_then(|i| args.get(i + 1)).cloned().unwrap_or_else(|| "data/nmf_denmark.csv".into());
nmf_denmark(db, &out)
}
_ => { eprintln!("usage: airplay export nmf-denmark [--out FILE]"); std::process::exit(2); }
}
}
fn nmf_denmark(db: &Db, out: &str) -> Result<()> {
crate::playlists::create_tables(db)?;
for col in ["title TEXT", "artists TEXT"] { let _ = db.conn.execute(&format!("ALTER TABLE nmf_lookups ADD COLUMN {col}"), []); }
let mut f = std::fs::File::create(out)?;
writeln!(f, "friday,position,title,artists,isrc,spotify_track_id,playlist_followers,source")?;
let mut by_friday: BTreeMap<String, Vec<i64>> = BTreeMap::new();
let mut rows = 0;
// Title/artist -> ISRC, to give snapshot rows an ISRC when the song was looked up.
let key = |t: &str, a: &str| format!("{}|{}", crate::model::panel::key(t), crate::model::panel::key(a.split(',').next().unwrap_or(a)));
let mut isrc_of: HashMap<String, String> = HashMap::new();
let mut st = db.conn.prepare(
"SELECT n.friday, n.position, coalesce(l.title, ''), coalesce(l.artists, ''), n.isrc, n.followers
FROM nmf_additions n LEFT JOIN nmf_lookups l ON l.isrc = n.isrc
WHERE lower(n.country) = 'denmark' ORDER BY n.friday, n.position")?;
let it = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, Option<i64>>(1)?, r.get::<_, String>(2)?, r.get::<_, String>(3)?,
r.get::<_, String>(4)?, r.get::<_, Option<i64>>(5)?)))?;
for r in it {
let (friday, pos, title, artists, isrc, followers) = r?;
isrc_of.insert(key(&title, &artists), isrc.clone());
writeln!(f, "{friday},{},{},{},{isrc},,{},superfridaychart", pos.map(|p| p.to_string()).unwrap_or_default(), csv(&title), csv(&artists),
followers.map(|x| x.to_string()).unwrap_or_default())?;
if let Some(p) = pos { by_friday.entry(friday).or_default().push(p); }
rows += 1;
}
// Complete snapshots of the live playlist (each distinct list once, dated by its Friday).
let mut st = db.conn.prepare(
"SELECT taken_at, position, coalesce(title, ''), coalesce(artists, ''), track_id FROM playlist_snapshots
WHERE playlist_id = '37i9dQZF1DWWuIAAkPM0Lu' ORDER BY taken_at, position")?;
let it = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?, r.get::<_, String>(2)?, r.get::<_, String>(3)?, r.get::<_, String>(4)?)))?;
let mut snap_rows = 0;
for r in it {
let (taken, pos, title, artists, track) = r?;
// The Friday on or before the snapshot.
let d = chrono::NaiveDate::parse_from_str(&taken[..10], "%Y-%m-%d")?;
let back = (chrono::Datelike::weekday(&d).num_days_from_monday() as i64 + 3) % 7;
let friday = (d - chrono::Duration::days(back)).to_string();
let isrc = isrc_of.get(&key(&title, &artists)).cloned().unwrap_or_default();
writeln!(f, "{friday},{pos},{},{},{isrc},{track},,spotify snapshot {}", csv(&title), csv(&artists), &taken[..10])?;
snap_rows += 1;
}
// Coverage per Friday: positions run 1..N, so gaps are exactly the songs not found yet.
let cov = out.replace(".csv", "_coverage.csv");
let mut c = std::fs::File::create(&cov)?;
writeln!(c, "friday,positions_found,highest_position,missing_positions")?;
let (mut found_all, mut max_all) = (0usize, 0i64);
for (friday, mut ps) in by_friday {
ps.sort(); ps.dedup();
let max = *ps.last().unwrap_or(&0);
let missing: Vec<String> = (1..=max).filter(|p| ps.binary_search(p).is_err()).map(|p| p.to_string()).collect();
writeln!(c, "{friday},{},{max},{}", ps.len(), csv(&missing.join(" ")))?;
found_all += ps.len(); max_all += max;
}
println!("{out}: {rows} rows from SuperFridayChart track pages, {snap_rows} from playlist snapshots");
println!("{cov}: {found_all} of {max_all} positions found ({:.0}%) across the Fridays seen", 100.0 * found_all as f64 / max_all.max(1) as f64);
Ok(())
}
+11 -3
View File
@@ -1,6 +1,7 @@
//! airplay: collects what Danish radio stations play, for modelling when artists get played again. //! airplay: collects what Danish radio stations play, for modelling when artists get played again.
//! //!
//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf|nmf-artists] [--backfill] //! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks] [--backfill]
//! airplay export nmf-denmark [--out data/nmf_denmark.csv]
//! airplay stats [--db data/airplay.db] //! airplay stats [--db data/airplay.db]
//! airplay check [--db data/airplay.db] gaps, duplicates, DR cross-check, chart completeness //! airplay check [--db data/airplay.db] gaps, duplicates, DR cross-check, chart completeness
//! airplay model panel | backtest | rank the first models (src/model) //! airplay model panel | backtest | rank the first models (src/model)
@@ -14,6 +15,7 @@ mod bauer;
mod check; mod check;
mod db; mod db;
mod deezer; mod deezer;
mod export;
mod dr; mod dr;
mod hitlisten; mod hitlisten;
mod http; mod http;
@@ -45,6 +47,7 @@ fn main() -> Result<()> {
Some("check") => check::run(&db), Some("check") => check::run(&db),
Some("model") => model::run(&db, &args), Some("model") => model::run(&db, &args),
Some("serve") => web::serve(&db_path, &args), Some("serve") => web::serve(&db_path, &args),
Some("export") => export::run(&db, &args),
Some("pdfdump") => rotations::dump(std::path::Path::new(args.get(2).map(String::as_str).unwrap_or(""))), Some("pdfdump") => rotations::dump(std::path::Path::new(args.get(2).map(String::as_str).unwrap_or(""))),
Some("rotations") => { Some("rotations") => {
let dir = std::path::Path::new(&db_path).parent().unwrap_or(std::path::Path::new(".")).join("rotations"); let dir = std::path::Path::new(&db_path).parent().unwrap_or(std::path::Path::new(".")).join("rotations");
@@ -57,7 +60,8 @@ fn main() -> Result<()> {
rotations::store_all(&db, &dir, args.iter().any(|a| a == "--reparse")) rotations::store_all(&db, &dir, args.iter().any(|a| a == "--reparse"))
} }
_ => { _ => {
eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf|nmf-artists] [--backfill]\n airplay stats [--db FILE]\n airplay check [--db FILE]\n airplay model panel|backtest|rank\n airplay serve [--addr HOST:PORT] [--prefix /airplay]"); eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks] [--backfill]
//! airplay export nmf-denmark [--out data/nmf_denmark.csv]\n airplay stats [--db FILE]\n airplay check [--db FILE]\n airplay model panel|backtest|rank\n airplay serve [--addr HOST:PORT] [--prefix /airplay]");
std::process::exit(2); std::process::exit(2);
} }
} }
@@ -135,7 +139,11 @@ fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> {
let r = playlists::nmf_artists(db, &sfc_http, if backfill { usize::MAX } else { 800 }, &mut log); let r = playlists::nmf_artists(db, &sfc_http, if backfill { usize::MAX } else { 800 }, &mut log);
step("new music friday per artist (superfridaychart)", r.map(|n| format!("{n} artists looked up")), &mut log); step("new music friday per artist (superfridaychart)", r.map(|n| format!("{n} artists looked up")), &mut log);
} }
if wants("nmf") { if wants("nmf") || wants("nmf-charts") {
let r = playlists::nmf_global_charts(db, &http::Http::new(1000)?, &mut log);
step("new music friday global charts (superfridaychart)", r.map(|n| format!("{n} Fridays read")), &mut log);
}
if wants("nmf") || wants("nmf-tracks") {
let sfc_http = http::Http::new(1000)?; let sfc_http = http::Http::new(1000)?;
let r = playlists::nmf_history(db, &sfc_http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 1500 }), &mut log); let r = playlists::nmf_history(db, &sfc_http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 1500 }), &mut log);
step("new music friday history (superfridaychart)", r.map(|n| format!("{n} ISRCs looked up")), &mut log); step("new music friday history (superfridaychart)", r.map(|n| format!("{n} ISRCs looked up")), &mut log);
+35 -7
View File
@@ -27,6 +27,31 @@ pub const GAP: usize = 4;
pub const W: usize = 8; pub const W: usize = 8;
/// Per-channel rows: the panel features plus these. /// Per-channel rows: the panel features plus these.
/// Wave covariates: the chance of a next wave peaks 13–25 weeks after the last one and at
/// 0.5–1.25x the artist's usual gap (25 years of Airplay Top-20: weekly hazard 1.7% at 13–25
/// weeks vs 0.5% before and 0.3–0.4% after a year). Appended after the release block.
pub const WAVE: usize = 6;
/// Feature blocks switched off for an ablation run ("rel", "wave").
pub static WITHOUT: std::sync::OnceLock<Vec<String>> = std::sync::OnceLock::new();
fn off(block: &str) -> bool { WITHOUT.get().is_some_and(|v| v.iter().any(|b| b == block)) }
pub const WAVE_NAMES: &[&str] = &["DR: weeks since last add 13-25", "DR: last add 26+ weeks ago", "DR: no add in the panel yet",
"Airplay Top-20: last new song 13-25 weeks ago", "Airplay Top-20: 0.5-1.25x own usual gap", "Airplay Top-20: last new song 2+ years ago"];
/// Weeks since the artist's last add on any DR channel (as of week t; None = none in the panel),
/// and the IFPI wave position.
pub fn wave_feats(p: &Panel, r: &Rot, a: usize, t: usize) -> [f64; WAVE] {
if off("wave") { return [0.0; WAVE]; }
let since = (0..=t).find(|&i| r.add_any(a, t - i));
let ch = p.chart(a, t);
let f = |b: bool| b as i32 as f64;
let air = ch.air_since as f64;
let known = ch.air_since != u16::MAX;
let ratio_ok = known && ch.air_gap > 0 && { let x = air / ch.air_gap as f64; (0.5..=1.25).contains(&x) };
[f(since.is_some_and(|s| (13..=25).contains(&s))), f(since.is_some_and(|s| s >= 26)), f(since.is_none()),
f(known && (13.0..=25.0).contains(&air)), f(ratio_ok), f(known && air >= 104.0)]
}
/// Release covariates (Deezer discography), appended to both row kinds. /// Release covariates (Deezer discography), appended to both row kinds.
pub const REL: usize = 6; pub const REL: usize = 6;
pub const REL_NAMES: &[&str] = &["discography known", "release recency 4w", "release recency 12w", "log releases last 52w", pub const REL_NAMES: &[&str] = &["discography known", "release recency 4w", "release recency 12w", "log releases last 52w",
@@ -36,6 +61,7 @@ pub const REL_NAMES: &[&str] = &["discography known", "release recency 4w", "rel
/// the window t+h+1 ..= t+h+W (the actual one when `rel` is None: training and backtests; else the /// the window t+h+1 ..= t+h+W (the actual one when `rel` is None: training and backtests; else the
/// scenario "releases / doesn't release in the window"). /// scenario "releases / doesn't release in the window").
pub fn rel_feats(p: &Panel, a: usize, t: usize, h: usize, rel: Option<bool>, in_rot_now: bool) -> [f64; REL] { pub fn rel_feats(p: &Panel, a: usize, t: usize, h: usize, rel: Option<bool>, in_rot_now: bool) -> [f64; REL] {
if off("rel") { return [0.0; REL]; }
let Some(dates) = p.releases[a].as_ref() else { return [0.0; REL] }; let Some(dates) = p.releases[a].as_ref() else { return [0.0; REL] };
let end_t = p.weeks[t] + chrono::Duration::days(6); let end_t = p.weeks[t] + chrono::Duration::days(6);
let since = dates.iter().rev().find(|d| **d <= end_t).map(|d| (end_t - *d).num_days() as f64 / 7.0); let since = dates.iter().rev().find(|d| **d <= end_t).map(|d| (end_t - *d).num_days() as f64 / 7.0);
@@ -62,7 +88,7 @@ pub const EXTRA: &[&str] = &["in rotation on channel now", "in rotation anywhere
"run on channel = 1 week", "run = 2 weeks", "run = 3-5 weeks", "run >= 6 weeks", "run on channel = 1 week", "run = 2 weeks", "run = 3-5 weeks", "run >= 6 weeks",
"near threshold (>= half of tau, not in rotation)", "plays / tau (capped 3, /3)", "near threshold (>= half of tau, not in rotation)", "plays / tau (capped 3, /3)",
"weeks since in rotation on channel (/12)"]; "weeks since in rotation on channel (/12)"];
pub const PC: usize = P + 10 + REL; pub const PC: usize = P + 10 + REL + WAVE;
/// Artist-level ("any channel") rows. /// Artist-level ("any channel") rows.
pub const ANY_NAMES: &[&str] = &[ pub const ANY_NAMES: &[&str] = &[
@@ -72,7 +98,7 @@ pub const ANY_NAMES: &[&str] = &[
"log Track Top-40 hits", "log Airplay Top-20 hits", "Danish", "group", "female", "career years (/50)", "debut unknown", "log Track Top-40 hits", "log Airplay Top-20 hits", "Danish", "group", "female", "career years (/50)", "debut unknown",
"Christmas in window", "Christmas in window",
]; ];
pub const PA: usize = 21 + REL; pub const PA: usize = 21 + REL + WAVE;
/// Positions of the chart covariates (for the what-if): (t40 score, air score, recency 6w, recency 26w). /// Positions of the chart covariates (for the what-if): (t40 score, air score, recency 6w, recency 26w).
pub const CHART_IDX_CHANNEL: [usize; 4] = [10, 11, 12, 13]; pub const CHART_IDX_CHANNEL: [usize; 4] = [10, 11, 12, 13];
pub const CHART_IDX_ANY: [usize; 4] = [9, 10, 11, 12]; pub const CHART_IDX_ANY: [usize; 4] = [9, 10, 11, 12];
@@ -124,10 +150,11 @@ pub fn row_channel(p: &Panel, r: &Rot, a: usize, c: usize, t: usize, h: usize, r
let on = r.on(a, c, t); let on = r.on(a, c, t);
let ratio = p.y(a, c, t) / tau_of(p, r, a, c, t); let ratio = p.y(a, c, t) / tau_of(p, r, a, c, t);
let f = |b: bool| b as i32 as f64; let f = |b: bool| b as i32 as f64;
out[P..PC].copy_from_slice(&[f(on), f(r.any(a, t)), n_on / C as f64, out[P..P + 10].copy_from_slice(&[f(on), f(r.any(a, t)), n_on / C as f64,
f(weeks == 1), f(weeks == 2), f((3..=5).contains(&weeks)), f(weeks >= 6), f(weeks == 1), f(weeks == 2), f((3..=5).contains(&weeks)), f(weeks >= 6),
f(!on && ratio >= 0.5), ratio.min(3.0) / 3.0, since]); f(!on && ratio >= 0.5), ratio.min(3.0) / 3.0, since]);
out[P + 10..PC].copy_from_slice(&rel_feats(p, a, t, h, rel, on)); out[P + 10..P + 10 + REL].copy_from_slice(&rel_feats(p, a, t, h, rel, on));
out[P + 10 + REL..PC].copy_from_slice(&wave_feats(p, r, a, t));
} }
pub fn row_any(p: &Panel, r: &Rot, a: usize, t: usize, h: usize, rel: Option<bool>, out: &mut [f64]) { pub fn row_any(p: &Panel, r: &Rot, a: usize, t: usize, h: usize, rel: Option<bool>, out: &mut [f64]) {
@@ -151,7 +178,8 @@ pub fn row_any(p: &Panel, r: &Rot, a: usize, t: usize, h: usize, rel: Option<boo
christmas_in_window(p, t, h) as i32 as f64, christmas_in_window(p, t, h) as i32 as f64,
]; ];
out[..21].copy_from_slice(&v); out[..21].copy_from_slice(&v);
out[21..PA].copy_from_slice(&rel_feats(p, a, t, h, rel, r.any(a, t))); out[21..21 + REL].copy_from_slice(&rel_feats(p, a, t, h, rel, r.any(a, t)));
out[21 + REL..PA].copy_from_slice(&wave_feats(p, r, a, t));
} }
/// Logit models for one h, trained on origins whose whole window is observed by `last_week`. /// Logit models for one h, trained on origins whose whole window is observed by `last_week`.
@@ -376,9 +404,9 @@ pub fn backtest(p: &Panel, tau: &[u32; C], horizons: &[usize], start: usize, md:
let m = AddModel::fit(p, &r, 0, p.w() - 1); let m = AddModel::fit(p, &r, 0, p.w() - 1);
writeln!(md, "## Coefficients, window starting now (h = 0), all closed windows\n\n| Any-channel feature | Coefficient |\n|---|---|")?; writeln!(md, "## Coefficients, window starting now (h = 0), all closed windows\n\n| Any-channel feature | Coefficient |\n|---|---|")?;
for (n, b) in ANY_NAMES.iter().zip(&m.any) { writeln!(md, "| {n} | {b:+.3} |")?; } for (n, b) in ANY_NAMES.iter().zip(&m.any) { writeln!(md, "| {n} | {b:+.3} |")?; }
for (n, b) in REL_NAMES.iter().zip(&m.any[21..]) { writeln!(md, "| {n} | {b:+.3} |")?; } for (n, b) in REL_NAMES.iter().chain(WAVE_NAMES).zip(&m.any[21..]) { writeln!(md, "| {n} | {b:+.3} |")?; }
writeln!(md, "\n| Per-channel feature | Coefficient |\n|---|---|")?; writeln!(md, "\n| Per-channel feature | Coefficient |\n|---|---|")?;
for (n, b) in features::NAMES.iter().chain(EXTRA).chain(REL_NAMES).zip(&m.channel) { writeln!(md, "| {n} | {b:+.3} |")?; } for (n, b) in features::NAMES.iter().chain(EXTRA).chain(REL_NAMES).chain(WAVE_NAMES).zip(&m.channel) { writeln!(md, "| {n} | {b:+.3} |")?; }
writeln!(md)?; writeln!(md)?;
Ok(rows) Ok(rows)
} }
+1
View File
@@ -76,6 +76,7 @@ pub fn run(db: &Db, args: &[String]) -> Result<()> {
print!("{md}"); print!("{md}");
} }
Some("adds") => { Some("adds") => {
if let Some(w) = arg(args, "--without") { let _ = adds::WITHOUT.set(w.split(',').map(str::to_string).collect()); }
let horizons: Vec<usize> = arg(args, "--h").unwrap_or_else(|| "0,4".into()).split(',').filter_map(|s| s.trim().parse().ok()).collect(); let horizons: Vec<usize> = arg(args, "--h").unwrap_or_else(|| "0,4".into()).split(',').filter_map(|s| s.trim().parse().ok()).collect();
let tau = common::thresholds(&p, 26); let tau = common::thresholds(&p, 26);
let mut md = String::new(); let mut md = String::new();
+16 -1
View File
@@ -41,6 +41,10 @@ pub struct Chart {
/// Distinct songs on the Track Top-40 / Airplay Top-20 so far. /// Distinct songs on the Track Top-40 / Airplay Top-20 so far.
pub t40_hits: u16, pub t40_hits: u16,
pub air_hits: u16, pub air_hits: u16,
/// Airplay Top-20 "waves": weeks since the artist's latest new song on the Airplay Top-20
/// (u16::MAX = never), and the median gap between their earlier new songs (0 = fewer than 2 gaps).
pub air_since: u16,
pub air_gap: u16,
} }
pub struct Panel { pub struct Panel {
@@ -170,10 +174,12 @@ fn chart_states(db: &Db, weeks: &[NaiveDate], names: &HashMap<String, Vec<usize>
for entries in &by_artist { for entries in &by_artist {
for w in 0..nw { for w in 0..nw {
let cutoff = cutoffs[w]; let cutoff = cutoffs[w];
let mut s = Chart { t40_rank: 0, air_rank: 0, weeks_since_entry: u16::MAX, t40_hits: 0, air_hits: 0 }; let mut s = Chart { t40_rank: 0, air_rank: 0, weeks_since_entry: u16::MAX, t40_hits: 0, air_hits: 0, air_since: u16::MAX, air_gap: 0 };
let mut air_firsts: BTreeSet<NaiveDate> = BTreeSet::new();
let (mut t40_songs, mut air_songs) = (BTreeSet::new(), BTreeSet::new()); let (mut t40_songs, mut air_songs) = (BTreeSet::new(), BTreeSet::new());
for e in entries.iter().filter(|e| e.monday <= cutoff) { for e in entries.iter().filter(|e| e.monday <= cutoff) {
if e.t40 { t40_songs.insert(&e.song); } else { air_songs.insert(&e.song); } if e.t40 { t40_songs.insert(&e.song); } else { air_songs.insert(&e.song); }
if !e.t40 && e.new { air_firsts.insert(e.monday); }
let best = |cur: u8| if cur == 0 { e.rank } else { cur.min(e.rank) }; let best = |cur: u8| if cur == 0 { e.rank } else { cur.min(e.rank) };
if e.t40 && Some(e.monday) == latest_t40[w] { s.t40_rank = best(s.t40_rank); } if e.t40 && Some(e.monday) == latest_t40[w] { s.t40_rank = best(s.t40_rank); }
if !e.t40 && Some(e.monday) == latest_air[w] { s.air_rank = best(s.air_rank); } if !e.t40 && Some(e.monday) == latest_air[w] { s.air_rank = best(s.air_rank); }
@@ -184,6 +190,15 @@ fn chart_states(db: &Db, weeks: &[NaiveDate], names: &HashMap<String, Vec<usize>
} }
s.t40_hits = t40_songs.len() as u16; s.t40_hits = t40_songs.len() as u16;
s.air_hits = air_songs.len() as u16; s.air_hits = air_songs.len() as u16;
// Waves: new songs at least 2 weeks apart.
let mut waves: Vec<NaiveDate> = Vec::new();
for d in &air_firsts { if waves.last().is_none_or(|l| (*d - *l).num_days() >= 14) { waves.push(*d); } }
if let Some(last) = waves.last() { s.air_since = ((cutoff - *last).num_days() / 7).clamp(0, u16::MAX as i64 - 1) as u16; }
if waves.len() >= 3 {
let mut gaps: Vec<i64> = waves.windows(2).map(|w| (w[1] - w[0]).num_days() / 7).collect();
gaps.sort();
s.air_gap = gaps[gaps.len() / 2].clamp(1, 2000) as u16;
}
out.push(s); out.push(s);
} }
} }
+60 -4
View File
@@ -85,12 +85,35 @@ pub fn nmf_history(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) ->
crate::deezer::create_tables(db)?; crate::deezer::create_tables(db)?;
db.conn.execute_batch("CREATE TABLE IF NOT EXISTS nmf_artist_tracks (spotify_artist_id TEXT NOT NULL, isrc TEXT NOT NULL, title TEXT, db.conn.execute_batch("CREATE TABLE IF NOT EXISTS nmf_artist_tracks (spotify_artist_id TEXT NOT NULL, isrc TEXT NOT NULL, title TEXT,
friday TEXT NOT NULL, playlists INTEGER, followers INTEGER, PRIMARY KEY (spotify_artist_id, isrc, friday));")?; friday TEXT NOT NULL, playlists INTEGER, followers INTEGER, PRIMARY KEY (spotify_artist_id, isrc, friday));")?;
// Placements found on artist pages inside the modelling window first, then every ISRC Deezer gave us. // Title and artists of each looked-up track (added later; older rows have them empty).
for col in ["title TEXT", "artists TEXT"] {
let _ = db.conn.execute(&format!("ALTER TABLE nmf_lookups ADD COLUMN {col}"), []);
}
db.conn.execute_batch("CREATE TABLE IF NOT EXISTS nmf_global_chart (friday TEXT NOT NULL, rank INTEGER NOT NULL, isrc TEXT NOT NULL,
PRIMARY KEY (friday, rank));")?;
// Which tracks to read, most likely to be on NMF Denmark first:
// 0 artist-page placements of Danish acts (DR artist country), or one playlist the size of
// NMF Denmark that Friday (the most common single-playlist follower count among Danish acts)
// 1 SuperFridayChart's weekly global top 50
// 2 other artist-page placements on several playlists
// 3 every ISRC Deezer gave us
let todo: Vec<String> = db.conn.prepare( let todo: Vec<String> = db.conn.prepare(
"SELECT isrc FROM (SELECT isrc, 0 AS o FROM nmf_artist_tracks WHERE friday >= '2025-11-01' "WITH danish AS (SELECT l.spotify_artist_id FROM nmf_artist_lookups l JOIN dr_artists a ON a.artist_id = l.dr_artist_id
UNION SELECT isrc, 1 FROM deezer_tracks WHERE coalesce(isrc, '') != '') WHERE json_extract(a.raw, '$.artist.country') = 'Danmark' AND l.spotify_artist_id IS NOT NULL),
dk_size AS (SELECT friday, followers FROM (
SELECT t.friday, t.followers, count(*) n, row_number() OVER (PARTITION BY t.friday ORDER BY count(*) DESC) rn
FROM nmf_artist_tracks t WHERE t.playlists = 1 AND t.spotify_artist_id IN (SELECT spotify_artist_id FROM danish)
GROUP BY t.friday, t.followers) WHERE rn = 1)
SELECT isrc FROM (
SELECT t.isrc, 0 AS o FROM nmf_artist_tracks t LEFT JOIN dk_size d ON d.friday = t.friday
WHERE t.spotify_artist_id IN (SELECT spotify_artist_id FROM danish)
OR (t.playlists = 1 AND abs(t.followers - d.followers) <= 0.02 * d.followers)
UNION ALL SELECT isrc, 1 FROM nmf_global_chart
UNION ALL SELECT isrc, 2 FROM nmf_artist_tracks WHERE playlists > 1
UNION ALL SELECT isrc, 3 FROM deezer_tracks WHERE coalesce(isrc, '') != '')
WHERE isrc NOT IN (SELECT isrc FROM nmf_lookups) GROUP BY isrc ORDER BY min(o)")? WHERE isrc NOT IN (SELECT isrc FROM nmf_lookups) GROUP BY isrc ORDER BY min(o)")?
.query_map([], |r| r.get(0))?.collect::<Result<_, _>>()?; .query_map([], |r| r.get(0))?.collect::<Result<_, _>>()?;
let meta = Regex::new(r#"<meta name="description" content="Discover the Spotify New Music Friday playlists that added the track &quot;(.*?)&quot; from (.*?)\.">"#)?;
let day = Regex::new(r#"<h2[^>]*>\s*(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday), ([A-Za-z]+ \d{2}, \d{4})\s*</h2>"#)?; let day = Regex::new(r#"<h2[^>]*>\s*(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday), ([A-Za-z]+ \d{2}, \d{4})\s*</h2>"#)?;
let row = Regex::new(r#"(?s)<p[^>]*>([^<]+)</p>\s*<a href="https://open\.spotify\.com/playlist/([A-Za-z0-9]{22})".*?<span[^>]*>([\d,]+) followers</span>\s*<span[^>]*>Position (\d+)</span>"#)?; let row = Regex::new(r#"(?s)<p[^>]*>([^<]+)</p>\s*<a href="https://open\.spotify\.com/playlist/([A-Za-z0-9]{22})".*?<span[^>]*>([\d,]+) followers</span>\s*<span[^>]*>Position (\d+)</span>"#)?;
let mut done = 0; let mut done = 0;
@@ -100,7 +123,13 @@ pub fn nmf_history(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) ->
Err(e) => { log.push(format!("superfridaychart {isrc}: {e}")); continue; } Err(e) => { log.push(format!("superfridaychart {isrc}: {e}")); continue; }
}; };
let mut found = 0; let mut found = 0;
let (mut title, mut artists) = (None, None);
if let Some(html) = html { if let Some(html) = html {
if let Some(m) = meta.captures(&html) {
let un = |x: &str| x.replace("&amp;", "&").replace("&#039;", "'").replace("&quot;", "\"");
title = Some(un(&m[1]));
artists = Some(un(&m[2]));
}
// Each Friday is an <h2> date followed by its table of playlists. // Each Friday is an <h2> date followed by its table of playlists.
let heads: Vec<(usize, NaiveDate)> = day.captures_iter(&html) let heads: Vec<(usize, NaiveDate)> = day.captures_iter(&html)
.filter_map(|c| Some((c.get(0)?.start(), NaiveDate::parse_from_str(&c[1], "%B %d, %Y").ok()?))) .filter_map(|c| Some((c.get(0)?.start(), NaiveDate::parse_from_str(&c[1], "%B %d, %Y").ok()?)))
@@ -114,7 +143,8 @@ pub fn nmf_history(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) ->
} }
} }
} }
db.conn.execute("INSERT OR REPLACE INTO nmf_lookups VALUES (?1, ?2, ?3)", params![isrc, found, Utc::now().to_rfc3339()])?; db.conn.execute("INSERT OR REPLACE INTO nmf_lookups (isrc, found, fetched_at, title, artists) VALUES (?1, ?2, ?3, ?4, ?5)",
params![isrc, found, Utc::now().to_rfc3339(), title, artists])?;
done += 1; done += 1;
} }
Ok(done) Ok(done)
@@ -171,3 +201,29 @@ pub fn nmf_artists(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) ->
} }
Ok(done) Ok(done)
} }
/// SuperFridayChart's weekly global chart (top 50 tracks across all NMF playlists), every Friday
/// listed on its front page: candidate ISRCs for the NMF Denmark lists.
pub fn nmf_global_charts(db: &Db, http: &Http, log: &mut Vec<String>) -> Result<usize> {
db.conn.execute_batch("CREATE TABLE IF NOT EXISTS nmf_global_chart (friday TEXT NOT NULL, rank INTEGER NOT NULL, isrc TEXT NOT NULL,
PRIMARY KEY (friday, rank));")?;
let front = http.get("https://www.superfridaychart.com/")?.unwrap_or_default();
let mut fridays: Vec<String> = Regex::new(r"/chart/(\d{4}-\d{2}-\d{2})")?.captures_iter(&front).map(|c| c[1].to_string()).collect();
fridays.sort(); fridays.dedup();
let track = Regex::new(r"/track/([A-Z0-9]{12})")?;
let mut n = 0;
for f in fridays {
let have: i64 = db.conn.query_row("SELECT count(*) FROM nmf_global_chart WHERE friday = ?1", [&f], |r| r.get(0))?;
if have > 0 { continue; }
let html = match http.get(&format!("https://www.superfridaychart.com/chart/{f}")) {
Ok(Some(h)) => h, Ok(None) => continue, Err(e) => { log.push(format!("superfridaychart chart {f}: {e}")); continue; }
};
let mut seen = Vec::new();
for c in track.captures_iter(&html) { if !seen.contains(&c[1].to_string()) { seen.push(c[1].to_string()); } }
for (i, isrc) in seen.iter().enumerate() {
db.conn.execute("INSERT OR REPLACE INTO nmf_global_chart VALUES (?1, ?2, ?3)", params![f, i as i64 + 1, isrc])?;
}
n += 1;
}
Ok(n)
}