Wave features (time since the last DR add / Airplay Top-20 wave, peak band 13-25 weeks, 0.5-1.25x the artist's usual gap); ablation switch (model adds --without rel,wave); fix the per-channel row slice
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,86 @@
|
||||
//! Data exports.
|
||||
//!
|
||||
//! airplay export nmf-denmark [--out data/nmf_denmark.csv]
|
||||
//!
|
||||
//! New Music Friday Denmark, one row per Friday and position, as far back as SuperFridayChart
|
||||
//! goes (January 2022): title, artists, ISRC, position, playlist followers. Rebuilt from
|
||||
//! SuperFridayChart's track pages, so a week is only as complete as the tracks looked up so far;
|
||||
//! a second file (…_coverage.csv) lists per Friday how many positions are found and which are
|
||||
//! missing. The playlist snapshots taken from 15 Nov 2026 on are complete and are added with
|
||||
//! source "spotify snapshot" (Spotify track id; ISRC where SuperFridayChart has the same song).
|
||||
|
||||
use crate::db::Db;
|
||||
use anyhow::Result;
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::io::Write;
|
||||
|
||||
fn csv(s: &str) -> String {
|
||||
if s.contains([',', '"', '\n']) { format!("\"{}\"", s.replace('"', "\"\"")) } else { s.to_string() }
|
||||
}
|
||||
|
||||
pub fn run(db: &Db, args: &[String]) -> Result<()> {
|
||||
match args.get(2).map(String::as_str) {
|
||||
Some("nmf-denmark") => {
|
||||
let out = args.iter().position(|a| a == "--out").and_then(|i| args.get(i + 1)).cloned().unwrap_or_else(|| "data/nmf_denmark.csv".into());
|
||||
nmf_denmark(db, &out)
|
||||
}
|
||||
_ => { eprintln!("usage: airplay export nmf-denmark [--out FILE]"); std::process::exit(2); }
|
||||
}
|
||||
}
|
||||
|
||||
fn nmf_denmark(db: &Db, out: &str) -> Result<()> {
|
||||
crate::playlists::create_tables(db)?;
|
||||
for col in ["title TEXT", "artists TEXT"] { let _ = db.conn.execute(&format!("ALTER TABLE nmf_lookups ADD COLUMN {col}"), []); }
|
||||
let mut f = std::fs::File::create(out)?;
|
||||
writeln!(f, "friday,position,title,artists,isrc,spotify_track_id,playlist_followers,source")?;
|
||||
let mut by_friday: BTreeMap<String, Vec<i64>> = BTreeMap::new();
|
||||
let mut rows = 0;
|
||||
// Title/artist -> ISRC, to give snapshot rows an ISRC when the song was looked up.
|
||||
let key = |t: &str, a: &str| format!("{}|{}", crate::model::panel::key(t), crate::model::panel::key(a.split(',').next().unwrap_or(a)));
|
||||
let mut isrc_of: HashMap<String, String> = HashMap::new();
|
||||
let mut st = db.conn.prepare(
|
||||
"SELECT n.friday, n.position, coalesce(l.title, ''), coalesce(l.artists, ''), n.isrc, n.followers
|
||||
FROM nmf_additions n LEFT JOIN nmf_lookups l ON l.isrc = n.isrc
|
||||
WHERE lower(n.country) = 'denmark' ORDER BY n.friday, n.position")?;
|
||||
let it = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, Option<i64>>(1)?, r.get::<_, String>(2)?, r.get::<_, String>(3)?,
|
||||
r.get::<_, String>(4)?, r.get::<_, Option<i64>>(5)?)))?;
|
||||
for r in it {
|
||||
let (friday, pos, title, artists, isrc, followers) = r?;
|
||||
isrc_of.insert(key(&title, &artists), isrc.clone());
|
||||
writeln!(f, "{friday},{},{},{},{isrc},,{},superfridaychart", pos.map(|p| p.to_string()).unwrap_or_default(), csv(&title), csv(&artists),
|
||||
followers.map(|x| x.to_string()).unwrap_or_default())?;
|
||||
if let Some(p) = pos { by_friday.entry(friday).or_default().push(p); }
|
||||
rows += 1;
|
||||
}
|
||||
// Complete snapshots of the live playlist (each distinct list once, dated by its Friday).
|
||||
let mut st = db.conn.prepare(
|
||||
"SELECT taken_at, position, coalesce(title, ''), coalesce(artists, ''), track_id FROM playlist_snapshots
|
||||
WHERE playlist_id = '37i9dQZF1DWWuIAAkPM0Lu' ORDER BY taken_at, position")?;
|
||||
let it = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?, r.get::<_, String>(2)?, r.get::<_, String>(3)?, r.get::<_, String>(4)?)))?;
|
||||
let mut snap_rows = 0;
|
||||
for r in it {
|
||||
let (taken, pos, title, artists, track) = r?;
|
||||
// The Friday on or before the snapshot.
|
||||
let d = chrono::NaiveDate::parse_from_str(&taken[..10], "%Y-%m-%d")?;
|
||||
let back = (chrono::Datelike::weekday(&d).num_days_from_monday() as i64 + 3) % 7;
|
||||
let friday = (d - chrono::Duration::days(back)).to_string();
|
||||
let isrc = isrc_of.get(&key(&title, &artists)).cloned().unwrap_or_default();
|
||||
writeln!(f, "{friday},{pos},{},{},{isrc},{track},,spotify snapshot {}", csv(&title), csv(&artists), &taken[..10])?;
|
||||
snap_rows += 1;
|
||||
}
|
||||
// Coverage per Friday: positions run 1..N, so gaps are exactly the songs not found yet.
|
||||
let cov = out.replace(".csv", "_coverage.csv");
|
||||
let mut c = std::fs::File::create(&cov)?;
|
||||
writeln!(c, "friday,positions_found,highest_position,missing_positions")?;
|
||||
let (mut found_all, mut max_all) = (0usize, 0i64);
|
||||
for (friday, mut ps) in by_friday {
|
||||
ps.sort(); ps.dedup();
|
||||
let max = *ps.last().unwrap_or(&0);
|
||||
let missing: Vec<String> = (1..=max).filter(|p| ps.binary_search(p).is_err()).map(|p| p.to_string()).collect();
|
||||
writeln!(c, "{friday},{},{max},{}", ps.len(), csv(&missing.join(" ")))?;
|
||||
found_all += ps.len(); max_all += max;
|
||||
}
|
||||
println!("{out}: {rows} rows from SuperFridayChart track pages, {snap_rows} from playlist snapshots");
|
||||
println!("{cov}: {found_all} of {max_all} positions found ({:.0}%) across the Fridays seen", 100.0 * found_all as f64 / max_all.max(1) as f64);
|
||||
Ok(())
|
||||
}
|
||||
+11
-3
@@ -1,6 +1,7 @@
|
||||
//! airplay: collects what Danish radio stations play, for modelling when artists get played again.
|
||||
//!
|
||||
//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf|nmf-artists] [--backfill]
|
||||
//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks] [--backfill]
|
||||
//! airplay export nmf-denmark [--out data/nmf_denmark.csv]
|
||||
//! airplay stats [--db data/airplay.db]
|
||||
//! airplay check [--db data/airplay.db] gaps, duplicates, DR cross-check, chart completeness
|
||||
//! airplay model panel | backtest | rank the first models (src/model)
|
||||
@@ -14,6 +15,7 @@ mod bauer;
|
||||
mod check;
|
||||
mod db;
|
||||
mod deezer;
|
||||
mod export;
|
||||
mod dr;
|
||||
mod hitlisten;
|
||||
mod http;
|
||||
@@ -45,6 +47,7 @@ fn main() -> Result<()> {
|
||||
Some("check") => check::run(&db),
|
||||
Some("model") => model::run(&db, &args),
|
||||
Some("serve") => web::serve(&db_path, &args),
|
||||
Some("export") => export::run(&db, &args),
|
||||
Some("pdfdump") => rotations::dump(std::path::Path::new(args.get(2).map(String::as_str).unwrap_or(""))),
|
||||
Some("rotations") => {
|
||||
let dir = std::path::Path::new(&db_path).parent().unwrap_or(std::path::Path::new(".")).join("rotations");
|
||||
@@ -57,7 +60,8 @@ fn main() -> Result<()> {
|
||||
rotations::store_all(&db, &dir, args.iter().any(|a| a == "--reparse"))
|
||||
}
|
||||
_ => {
|
||||
eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf|nmf-artists] [--backfill]\n airplay stats [--db FILE]\n airplay check [--db FILE]\n airplay model panel|backtest|rank\n airplay serve [--addr HOST:PORT] [--prefix /airplay]");
|
||||
eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks] [--backfill]
|
||||
//! airplay export nmf-denmark [--out data/nmf_denmark.csv]\n airplay stats [--db FILE]\n airplay check [--db FILE]\n airplay model panel|backtest|rank\n airplay serve [--addr HOST:PORT] [--prefix /airplay]");
|
||||
std::process::exit(2);
|
||||
}
|
||||
}
|
||||
@@ -135,7 +139,11 @@ fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> {
|
||||
let r = playlists::nmf_artists(db, &sfc_http, if backfill { usize::MAX } else { 800 }, &mut log);
|
||||
step("new music friday per artist (superfridaychart)", r.map(|n| format!("{n} artists looked up")), &mut log);
|
||||
}
|
||||
if wants("nmf") {
|
||||
if wants("nmf") || wants("nmf-charts") {
|
||||
let r = playlists::nmf_global_charts(db, &http::Http::new(1000)?, &mut log);
|
||||
step("new music friday global charts (superfridaychart)", r.map(|n| format!("{n} Fridays read")), &mut log);
|
||||
}
|
||||
if wants("nmf") || wants("nmf-tracks") {
|
||||
let sfc_http = http::Http::new(1000)?;
|
||||
let r = playlists::nmf_history(db, &sfc_http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 1500 }), &mut log);
|
||||
step("new music friday history (superfridaychart)", r.map(|n| format!("{n} ISRCs looked up")), &mut log);
|
||||
|
||||
+35
-7
@@ -27,6 +27,31 @@ pub const GAP: usize = 4;
|
||||
pub const W: usize = 8;
|
||||
|
||||
/// Per-channel rows: the panel features plus these.
|
||||
/// Wave covariates: the chance of a next wave peaks 13–25 weeks after the last one and at
|
||||
/// 0.5–1.25x the artist's usual gap (25 years of Airplay Top-20: weekly hazard 1.7% at 13–25
|
||||
/// weeks vs 0.5% before and 0.3–0.4% after a year). Appended after the release block.
|
||||
pub const WAVE: usize = 6;
|
||||
|
||||
/// Feature blocks switched off for an ablation run ("rel", "wave").
|
||||
pub static WITHOUT: std::sync::OnceLock<Vec<String>> = std::sync::OnceLock::new();
|
||||
fn off(block: &str) -> bool { WITHOUT.get().is_some_and(|v| v.iter().any(|b| b == block)) }
|
||||
pub const WAVE_NAMES: &[&str] = &["DR: weeks since last add 13-25", "DR: last add 26+ weeks ago", "DR: no add in the panel yet",
|
||||
"Airplay Top-20: last new song 13-25 weeks ago", "Airplay Top-20: 0.5-1.25x own usual gap", "Airplay Top-20: last new song 2+ years ago"];
|
||||
|
||||
/// Weeks since the artist's last add on any DR channel (as of week t; None = none in the panel),
|
||||
/// and the IFPI wave position.
|
||||
pub fn wave_feats(p: &Panel, r: &Rot, a: usize, t: usize) -> [f64; WAVE] {
|
||||
if off("wave") { return [0.0; WAVE]; }
|
||||
let since = (0..=t).find(|&i| r.add_any(a, t - i));
|
||||
let ch = p.chart(a, t);
|
||||
let f = |b: bool| b as i32 as f64;
|
||||
let air = ch.air_since as f64;
|
||||
let known = ch.air_since != u16::MAX;
|
||||
let ratio_ok = known && ch.air_gap > 0 && { let x = air / ch.air_gap as f64; (0.5..=1.25).contains(&x) };
|
||||
[f(since.is_some_and(|s| (13..=25).contains(&s))), f(since.is_some_and(|s| s >= 26)), f(since.is_none()),
|
||||
f(known && (13.0..=25.0).contains(&air)), f(ratio_ok), f(known && air >= 104.0)]
|
||||
}
|
||||
|
||||
/// Release covariates (Deezer discography), appended to both row kinds.
|
||||
pub const REL: usize = 6;
|
||||
pub const REL_NAMES: &[&str] = &["discography known", "release recency 4w", "release recency 12w", "log releases last 52w",
|
||||
@@ -36,6 +61,7 @@ pub const REL_NAMES: &[&str] = &["discography known", "release recency 4w", "rel
|
||||
/// the window t+h+1 ..= t+h+W (the actual one when `rel` is None: training and backtests; else the
|
||||
/// scenario "releases / doesn't release in the window").
|
||||
pub fn rel_feats(p: &Panel, a: usize, t: usize, h: usize, rel: Option<bool>, in_rot_now: bool) -> [f64; REL] {
|
||||
if off("rel") { return [0.0; REL]; }
|
||||
let Some(dates) = p.releases[a].as_ref() else { return [0.0; REL] };
|
||||
let end_t = p.weeks[t] + chrono::Duration::days(6);
|
||||
let since = dates.iter().rev().find(|d| **d <= end_t).map(|d| (end_t - *d).num_days() as f64 / 7.0);
|
||||
@@ -62,7 +88,7 @@ pub const EXTRA: &[&str] = &["in rotation on channel now", "in rotation anywhere
|
||||
"run on channel = 1 week", "run = 2 weeks", "run = 3-5 weeks", "run >= 6 weeks",
|
||||
"near threshold (>= half of tau, not in rotation)", "plays / tau (capped 3, /3)",
|
||||
"weeks since in rotation on channel (/12)"];
|
||||
pub const PC: usize = P + 10 + REL;
|
||||
pub const PC: usize = P + 10 + REL + WAVE;
|
||||
|
||||
/// Artist-level ("any channel") rows.
|
||||
pub const ANY_NAMES: &[&str] = &[
|
||||
@@ -72,7 +98,7 @@ pub const ANY_NAMES: &[&str] = &[
|
||||
"log Track Top-40 hits", "log Airplay Top-20 hits", "Danish", "group", "female", "career years (/50)", "debut unknown",
|
||||
"Christmas in window",
|
||||
];
|
||||
pub const PA: usize = 21 + REL;
|
||||
pub const PA: usize = 21 + REL + WAVE;
|
||||
/// Positions of the chart covariates (for the what-if): (t40 score, air score, recency 6w, recency 26w).
|
||||
pub const CHART_IDX_CHANNEL: [usize; 4] = [10, 11, 12, 13];
|
||||
pub const CHART_IDX_ANY: [usize; 4] = [9, 10, 11, 12];
|
||||
@@ -124,10 +150,11 @@ pub fn row_channel(p: &Panel, r: &Rot, a: usize, c: usize, t: usize, h: usize, r
|
||||
let on = r.on(a, c, t);
|
||||
let ratio = p.y(a, c, t) / tau_of(p, r, a, c, t);
|
||||
let f = |b: bool| b as i32 as f64;
|
||||
out[P..PC].copy_from_slice(&[f(on), f(r.any(a, t)), n_on / C as f64,
|
||||
out[P..P + 10].copy_from_slice(&[f(on), f(r.any(a, t)), n_on / C as f64,
|
||||
f(weeks == 1), f(weeks == 2), f((3..=5).contains(&weeks)), f(weeks >= 6),
|
||||
f(!on && ratio >= 0.5), ratio.min(3.0) / 3.0, since]);
|
||||
out[P + 10..PC].copy_from_slice(&rel_feats(p, a, t, h, rel, on));
|
||||
out[P + 10..P + 10 + REL].copy_from_slice(&rel_feats(p, a, t, h, rel, on));
|
||||
out[P + 10 + REL..PC].copy_from_slice(&wave_feats(p, r, a, t));
|
||||
}
|
||||
|
||||
pub fn row_any(p: &Panel, r: &Rot, a: usize, t: usize, h: usize, rel: Option<bool>, out: &mut [f64]) {
|
||||
@@ -151,7 +178,8 @@ pub fn row_any(p: &Panel, r: &Rot, a: usize, t: usize, h: usize, rel: Option<boo
|
||||
christmas_in_window(p, t, h) as i32 as f64,
|
||||
];
|
||||
out[..21].copy_from_slice(&v);
|
||||
out[21..PA].copy_from_slice(&rel_feats(p, a, t, h, rel, r.any(a, t)));
|
||||
out[21..21 + REL].copy_from_slice(&rel_feats(p, a, t, h, rel, r.any(a, t)));
|
||||
out[21 + REL..PA].copy_from_slice(&wave_feats(p, r, a, t));
|
||||
}
|
||||
|
||||
/// Logit models for one h, trained on origins whose whole window is observed by `last_week`.
|
||||
@@ -376,9 +404,9 @@ pub fn backtest(p: &Panel, tau: &[u32; C], horizons: &[usize], start: usize, md:
|
||||
let m = AddModel::fit(p, &r, 0, p.w() - 1);
|
||||
writeln!(md, "## Coefficients, window starting now (h = 0), all closed windows\n\n| Any-channel feature | Coefficient |\n|---|---|")?;
|
||||
for (n, b) in ANY_NAMES.iter().zip(&m.any) { writeln!(md, "| {n} | {b:+.3} |")?; }
|
||||
for (n, b) in REL_NAMES.iter().zip(&m.any[21..]) { writeln!(md, "| {n} | {b:+.3} |")?; }
|
||||
for (n, b) in REL_NAMES.iter().chain(WAVE_NAMES).zip(&m.any[21..]) { writeln!(md, "| {n} | {b:+.3} |")?; }
|
||||
writeln!(md, "\n| Per-channel feature | Coefficient |\n|---|---|")?;
|
||||
for (n, b) in features::NAMES.iter().chain(EXTRA).chain(REL_NAMES).zip(&m.channel) { writeln!(md, "| {n} | {b:+.3} |")?; }
|
||||
for (n, b) in features::NAMES.iter().chain(EXTRA).chain(REL_NAMES).chain(WAVE_NAMES).zip(&m.channel) { writeln!(md, "| {n} | {b:+.3} |")?; }
|
||||
writeln!(md)?;
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
@@ -76,6 +76,7 @@ pub fn run(db: &Db, args: &[String]) -> Result<()> {
|
||||
print!("{md}");
|
||||
}
|
||||
Some("adds") => {
|
||||
if let Some(w) = arg(args, "--without") { let _ = adds::WITHOUT.set(w.split(',').map(str::to_string).collect()); }
|
||||
let horizons: Vec<usize> = arg(args, "--h").unwrap_or_else(|| "0,4".into()).split(',').filter_map(|s| s.trim().parse().ok()).collect();
|
||||
let tau = common::thresholds(&p, 26);
|
||||
let mut md = String::new();
|
||||
|
||||
+16
-1
@@ -41,6 +41,10 @@ pub struct Chart {
|
||||
/// Distinct songs on the Track Top-40 / Airplay Top-20 so far.
|
||||
pub t40_hits: u16,
|
||||
pub air_hits: u16,
|
||||
/// Airplay Top-20 "waves": weeks since the artist's latest new song on the Airplay Top-20
|
||||
/// (u16::MAX = never), and the median gap between their earlier new songs (0 = fewer than 2 gaps).
|
||||
pub air_since: u16,
|
||||
pub air_gap: u16,
|
||||
}
|
||||
|
||||
pub struct Panel {
|
||||
@@ -170,10 +174,12 @@ fn chart_states(db: &Db, weeks: &[NaiveDate], names: &HashMap<String, Vec<usize>
|
||||
for entries in &by_artist {
|
||||
for w in 0..nw {
|
||||
let cutoff = cutoffs[w];
|
||||
let mut s = Chart { t40_rank: 0, air_rank: 0, weeks_since_entry: u16::MAX, t40_hits: 0, air_hits: 0 };
|
||||
let mut s = Chart { t40_rank: 0, air_rank: 0, weeks_since_entry: u16::MAX, t40_hits: 0, air_hits: 0, air_since: u16::MAX, air_gap: 0 };
|
||||
let mut air_firsts: BTreeSet<NaiveDate> = BTreeSet::new();
|
||||
let (mut t40_songs, mut air_songs) = (BTreeSet::new(), BTreeSet::new());
|
||||
for e in entries.iter().filter(|e| e.monday <= cutoff) {
|
||||
if e.t40 { t40_songs.insert(&e.song); } else { air_songs.insert(&e.song); }
|
||||
if !e.t40 && e.new { air_firsts.insert(e.monday); }
|
||||
let best = |cur: u8| if cur == 0 { e.rank } else { cur.min(e.rank) };
|
||||
if e.t40 && Some(e.monday) == latest_t40[w] { s.t40_rank = best(s.t40_rank); }
|
||||
if !e.t40 && Some(e.monday) == latest_air[w] { s.air_rank = best(s.air_rank); }
|
||||
@@ -184,6 +190,15 @@ fn chart_states(db: &Db, weeks: &[NaiveDate], names: &HashMap<String, Vec<usize>
|
||||
}
|
||||
s.t40_hits = t40_songs.len() as u16;
|
||||
s.air_hits = air_songs.len() as u16;
|
||||
// Waves: new songs at least 2 weeks apart.
|
||||
let mut waves: Vec<NaiveDate> = Vec::new();
|
||||
for d in &air_firsts { if waves.last().is_none_or(|l| (*d - *l).num_days() >= 14) { waves.push(*d); } }
|
||||
if let Some(last) = waves.last() { s.air_since = ((cutoff - *last).num_days() / 7).clamp(0, u16::MAX as i64 - 1) as u16; }
|
||||
if waves.len() >= 3 {
|
||||
let mut gaps: Vec<i64> = waves.windows(2).map(|w| (w[1] - w[0]).num_days() / 7).collect();
|
||||
gaps.sort();
|
||||
s.air_gap = gaps[gaps.len() / 2].clamp(1, 2000) as u16;
|
||||
}
|
||||
out.push(s);
|
||||
}
|
||||
}
|
||||
|
||||
+60
-4
@@ -85,12 +85,35 @@ pub fn nmf_history(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) ->
|
||||
crate::deezer::create_tables(db)?;
|
||||
db.conn.execute_batch("CREATE TABLE IF NOT EXISTS nmf_artist_tracks (spotify_artist_id TEXT NOT NULL, isrc TEXT NOT NULL, title TEXT,
|
||||
friday TEXT NOT NULL, playlists INTEGER, followers INTEGER, PRIMARY KEY (spotify_artist_id, isrc, friday));")?;
|
||||
// Placements found on artist pages inside the modelling window first, then every ISRC Deezer gave us.
|
||||
// Title and artists of each looked-up track (added later; older rows have them empty).
|
||||
for col in ["title TEXT", "artists TEXT"] {
|
||||
let _ = db.conn.execute(&format!("ALTER TABLE nmf_lookups ADD COLUMN {col}"), []);
|
||||
}
|
||||
db.conn.execute_batch("CREATE TABLE IF NOT EXISTS nmf_global_chart (friday TEXT NOT NULL, rank INTEGER NOT NULL, isrc TEXT NOT NULL,
|
||||
PRIMARY KEY (friday, rank));")?;
|
||||
// Which tracks to read, most likely to be on NMF Denmark first:
|
||||
// 0 artist-page placements of Danish acts (DR artist country), or one playlist the size of
|
||||
// NMF Denmark that Friday (the most common single-playlist follower count among Danish acts)
|
||||
// 1 SuperFridayChart's weekly global top 50
|
||||
// 2 other artist-page placements on several playlists
|
||||
// 3 every ISRC Deezer gave us
|
||||
let todo: Vec<String> = db.conn.prepare(
|
||||
"SELECT isrc FROM (SELECT isrc, 0 AS o FROM nmf_artist_tracks WHERE friday >= '2025-11-01'
|
||||
UNION SELECT isrc, 1 FROM deezer_tracks WHERE coalesce(isrc, '') != '')
|
||||
"WITH danish AS (SELECT l.spotify_artist_id FROM nmf_artist_lookups l JOIN dr_artists a ON a.artist_id = l.dr_artist_id
|
||||
WHERE json_extract(a.raw, '$.artist.country') = 'Danmark' AND l.spotify_artist_id IS NOT NULL),
|
||||
dk_size AS (SELECT friday, followers FROM (
|
||||
SELECT t.friday, t.followers, count(*) n, row_number() OVER (PARTITION BY t.friday ORDER BY count(*) DESC) rn
|
||||
FROM nmf_artist_tracks t WHERE t.playlists = 1 AND t.spotify_artist_id IN (SELECT spotify_artist_id FROM danish)
|
||||
GROUP BY t.friday, t.followers) WHERE rn = 1)
|
||||
SELECT isrc FROM (
|
||||
SELECT t.isrc, 0 AS o FROM nmf_artist_tracks t LEFT JOIN dk_size d ON d.friday = t.friday
|
||||
WHERE t.spotify_artist_id IN (SELECT spotify_artist_id FROM danish)
|
||||
OR (t.playlists = 1 AND abs(t.followers - d.followers) <= 0.02 * d.followers)
|
||||
UNION ALL SELECT isrc, 1 FROM nmf_global_chart
|
||||
UNION ALL SELECT isrc, 2 FROM nmf_artist_tracks WHERE playlists > 1
|
||||
UNION ALL SELECT isrc, 3 FROM deezer_tracks WHERE coalesce(isrc, '') != '')
|
||||
WHERE isrc NOT IN (SELECT isrc FROM nmf_lookups) GROUP BY isrc ORDER BY min(o)")?
|
||||
.query_map([], |r| r.get(0))?.collect::<Result<_, _>>()?;
|
||||
let meta = Regex::new(r#"<meta name="description" content="Discover the Spotify New Music Friday playlists that added the track "(.*?)" from (.*?)\.">"#)?;
|
||||
let day = Regex::new(r#"<h2[^>]*>\s*(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday), ([A-Za-z]+ \d{2}, \d{4})\s*</h2>"#)?;
|
||||
let row = Regex::new(r#"(?s)<p[^>]*>([^<]+)</p>\s*<a href="https://open\.spotify\.com/playlist/([A-Za-z0-9]{22})".*?<span[^>]*>([\d,]+) followers</span>\s*<span[^>]*>Position (\d+)</span>"#)?;
|
||||
let mut done = 0;
|
||||
@@ -100,7 +123,13 @@ pub fn nmf_history(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) ->
|
||||
Err(e) => { log.push(format!("superfridaychart {isrc}: {e}")); continue; }
|
||||
};
|
||||
let mut found = 0;
|
||||
let (mut title, mut artists) = (None, None);
|
||||
if let Some(html) = html {
|
||||
if let Some(m) = meta.captures(&html) {
|
||||
let un = |x: &str| x.replace("&", "&").replace("'", "'").replace(""", "\"");
|
||||
title = Some(un(&m[1]));
|
||||
artists = Some(un(&m[2]));
|
||||
}
|
||||
// Each Friday is an <h2> date followed by its table of playlists.
|
||||
let heads: Vec<(usize, NaiveDate)> = day.captures_iter(&html)
|
||||
.filter_map(|c| Some((c.get(0)?.start(), NaiveDate::parse_from_str(&c[1], "%B %d, %Y").ok()?)))
|
||||
@@ -114,7 +143,8 @@ pub fn nmf_history(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) ->
|
||||
}
|
||||
}
|
||||
}
|
||||
db.conn.execute("INSERT OR REPLACE INTO nmf_lookups VALUES (?1, ?2, ?3)", params![isrc, found, Utc::now().to_rfc3339()])?;
|
||||
db.conn.execute("INSERT OR REPLACE INTO nmf_lookups (isrc, found, fetched_at, title, artists) VALUES (?1, ?2, ?3, ?4, ?5)",
|
||||
params![isrc, found, Utc::now().to_rfc3339(), title, artists])?;
|
||||
done += 1;
|
||||
}
|
||||
Ok(done)
|
||||
@@ -171,3 +201,29 @@ pub fn nmf_artists(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) ->
|
||||
}
|
||||
Ok(done)
|
||||
}
|
||||
|
||||
/// SuperFridayChart's weekly global chart (top 50 tracks across all NMF playlists), every Friday
|
||||
/// listed on its front page: candidate ISRCs for the NMF Denmark lists.
|
||||
pub fn nmf_global_charts(db: &Db, http: &Http, log: &mut Vec<String>) -> Result<usize> {
|
||||
db.conn.execute_batch("CREATE TABLE IF NOT EXISTS nmf_global_chart (friday TEXT NOT NULL, rank INTEGER NOT NULL, isrc TEXT NOT NULL,
|
||||
PRIMARY KEY (friday, rank));")?;
|
||||
let front = http.get("https://www.superfridaychart.com/")?.unwrap_or_default();
|
||||
let mut fridays: Vec<String> = Regex::new(r"/chart/(\d{4}-\d{2}-\d{2})")?.captures_iter(&front).map(|c| c[1].to_string()).collect();
|
||||
fridays.sort(); fridays.dedup();
|
||||
let track = Regex::new(r"/track/([A-Z0-9]{12})")?;
|
||||
let mut n = 0;
|
||||
for f in fridays {
|
||||
let have: i64 = db.conn.query_row("SELECT count(*) FROM nmf_global_chart WHERE friday = ?1", [&f], |r| r.get(0))?;
|
||||
if have > 0 { continue; }
|
||||
let html = match http.get(&format!("https://www.superfridaychart.com/chart/{f}")) {
|
||||
Ok(Some(h)) => h, Ok(None) => continue, Err(e) => { log.push(format!("superfridaychart chart {f}: {e}")); continue; }
|
||||
};
|
||||
let mut seen = Vec::new();
|
||||
for c in track.captures_iter(&html) { if !seen.contains(&c[1].to_string()) { seen.push(c[1].to_string()); } }
|
||||
for (i, isrc) in seen.iter().enumerate() {
|
||||
db.conn.execute("INSERT OR REPLACE INTO nmf_global_chart VALUES (?1, ?2, ?3)", params![f, i as i64 + 1, isrc])?;
|
||||
}
|
||||
n += 1;
|
||||
}
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user