NMF: per-artist New Music Friday history (SuperFridayChart artist pages, complete per artist), track pages for window placements first; NMF Denmark vs DR adds analysis (airplay model nmf)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+7
-2
@@ -1,6 +1,6 @@
|
||||
//! airplay: collects what Danish radio stations play, for modelling when artists get played again.
|
||||
//!
|
||||
//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf] [--backfill]
|
||||
//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf|nmf-artists] [--backfill]
|
||||
//! airplay stats [--db data/airplay.db]
|
||||
//! airplay check [--db data/airplay.db] gaps, duplicates, DR cross-check, chart completeness
|
||||
//! airplay model panel | backtest | rank the first models (src/model)
|
||||
@@ -57,7 +57,7 @@ fn main() -> Result<()> {
|
||||
rotations::store_all(&db, &dir, args.iter().any(|a| a == "--reparse"))
|
||||
}
|
||||
_ => {
|
||||
eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf] [--backfill]\n airplay stats [--db FILE]\n airplay check [--db FILE]\n airplay model panel|backtest|rank\n airplay serve [--addr HOST:PORT] [--prefix /airplay]");
|
||||
eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf|nmf-artists] [--backfill]\n airplay stats [--db FILE]\n airplay check [--db FILE]\n airplay model panel|backtest|rank\n airplay serve [--addr HOST:PORT] [--prefix /airplay]");
|
||||
std::process::exit(2);
|
||||
}
|
||||
}
|
||||
@@ -130,6 +130,11 @@ fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> {
|
||||
let r = playlists::snapshot(db, &http, &mut log);
|
||||
step("spotify playlists (New Music Friday DK, Top 50 DK)", r.map(|n| format!("{n} changed and saved")), &mut log);
|
||||
}
|
||||
if wants("nmf") || wants("nmf-artists") {
|
||||
let sfc_http = http::Http::new(1000)?;
|
||||
let r = playlists::nmf_artists(db, &sfc_http, if backfill { usize::MAX } else { 800 }, &mut log);
|
||||
step("new music friday per artist (superfridaychart)", r.map(|n| format!("{n} artists looked up")), &mut log);
|
||||
}
|
||||
if wants("nmf") {
|
||||
let sfc_http = http::Http::new(1000)?;
|
||||
let r = playlists::nmf_history(db, &sfc_http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 1500 }), &mut log);
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
//! airplay model panel what the panel holds
|
||||
//! airplay model backtest [--h 1,4,8] [--refit 1] [--out docs/BACKTEST.md]
|
||||
//! airplay model calibrate [--out docs/CALIBRATION.md] rotation thresholds from DR's lists
|
||||
//! airplay model nmf [--out docs/NMF.md] New Music Friday Denmark vs DR adds
|
||||
//! airplay model ifpi the 20-year IFPI test bed -> docs/IFPI.md
|
||||
//! airplay model adds [--h 0,4] [--out docs/BACKTEST-ADDS.md] backtest of the new-add model
|
||||
//! airplay model rank [--h 4] [--max-h 12] [--top 50]
|
||||
@@ -16,6 +17,7 @@ pub mod features;
|
||||
pub mod glm;
|
||||
pub mod ifpi;
|
||||
pub mod kalman;
|
||||
pub mod nmf;
|
||||
pub mod panel;
|
||||
pub mod rank;
|
||||
|
||||
@@ -57,6 +59,11 @@ pub fn run(db: &Db, args: &[String]) -> Result<()> {
|
||||
print!("{}", report.markdown);
|
||||
eprintln!("written to {out}");
|
||||
}
|
||||
Some("nmf") => {
|
||||
let md = nmf::run(db, &p)?;
|
||||
if let Some(out) = arg(args, "--out") { std::fs::write(out, &md)?; }
|
||||
print!("{md}");
|
||||
}
|
||||
Some("ifpi") => {
|
||||
let md = ifpi::run(db)?;
|
||||
std::fs::write("docs/IFPI.md", &md)?;
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
//! Does New Music Friday Denmark lead to DR airplay? (`airplay model nmf` -> docs/NMF.md)
|
||||
//!
|
||||
//! Events: every release (Deezer discography: single, EP, album) by an artist in the DR panel,
|
||||
//! inside the panel window and with 8 weeks of follow-up. Each release is matched to the artist's
|
||||
//! New Music Friday placements (SuperFridayChart artist page) within a week of the release date:
|
||||
//!
|
||||
//! NMF Denmark the song was on New Music Friday Denmark (track page says Denmark, or, when the
|
||||
//! track page isn't read yet, one playlist with NMF Denmark's follower count)
|
||||
//! NMF elsewhere on New Music Friday in other countries only
|
||||
//! none not on any New Music Friday
|
||||
//!
|
||||
//! Outcome: a new DR add (adds.rs: entering rotation after 4+ weeks out) within 8 weeks of the
|
||||
//! release, on any channel and per channel group. Compared raw, by subgroup, and in a logit with
|
||||
//! controls (current airplay, in rotation now, Danish, chart history). Observational: songs are
|
||||
//! picked for NMF by the same people and signals that drive radio, so this measures association.
|
||||
|
||||
use super::adds::{Rot, GAP};
|
||||
use super::common::thresholds;
|
||||
use super::glm::{self, Design};
|
||||
use super::panel::{is_p4, Panel, C};
|
||||
use crate::db::Db;
|
||||
use anyhow::Result;
|
||||
use chrono::{Duration, NaiveDate};
|
||||
use std::collections::HashMap;
|
||||
use std::fmt::Write as _;
|
||||
|
||||
const WIN: usize = 8;
|
||||
|
||||
#[derive(Clone, Copy, PartialEq)]
|
||||
enum Nmf { Dk, Elsewhere, None }
|
||||
|
||||
pub fn run(db: &Db, p: &Panel) -> Result<String> {
|
||||
let tau = thresholds(p, p.w());
|
||||
let r = Rot::new(p, &tau);
|
||||
let id_of: HashMap<&str, usize> = p.artists.iter().enumerate().map(|(i, a)| (a.id.as_str(), i)).collect();
|
||||
|
||||
// Denmark per ISRC + Friday from the track pages.
|
||||
let mut dk_track: HashMap<(String, String), Option<i64>> = HashMap::new(); // -> position
|
||||
let mut seen_track: HashMap<(String, String), bool> = HashMap::new();
|
||||
let has = |t: &str| -> Result<bool> { Ok(db.conn.query_row("SELECT count(*) FROM sqlite_master WHERE name = ?1", [t], |r| r.get::<_, i64>(0))? > 0) };
|
||||
if has("nmf_additions")? {
|
||||
for row in db.conn.prepare("SELECT isrc, friday, country, position FROM nmf_additions")?
|
||||
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, Option<i64>>(3)?)))? {
|
||||
let (isrc, friday, country, pos) = row?;
|
||||
seen_track.insert((isrc.clone(), friday.clone()), true);
|
||||
if country.eq_ignore_ascii_case("denmark") { dk_track.insert((isrc, friday), pos); }
|
||||
}
|
||||
}
|
||||
// NMF Denmark's follower count per Friday (from the track pages), to recognise it on artist pages.
|
||||
let mut dk_followers: Vec<(NaiveDate, f64)> = Vec::new();
|
||||
if has("nmf_additions")? {
|
||||
for row in db.conn.prepare("SELECT friday, max(followers) FROM nmf_additions WHERE lower(country) = 'denmark' GROUP BY friday")?
|
||||
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, Option<f64>>(1)?)))? {
|
||||
let (f, n) = row?;
|
||||
if let (Ok(d), Some(n)) = (NaiveDate::parse_from_str(&f, "%Y-%m-%d"), n) { dk_followers.push((d, n)); }
|
||||
}
|
||||
}
|
||||
// Before the track pages are read: on each Friday, the most common follower count among
|
||||
// single-playlist placements of Danish acts is NMF Denmark.
|
||||
if has("nmf_artist_tracks")? {
|
||||
let mut counts: HashMap<(String, i64), usize> = HashMap::new();
|
||||
let danish: std::collections::HashSet<&str> = p.artists.iter().filter(|a| a.danish).map(|a| a.id.as_str()).collect();
|
||||
for row in db.conn.prepare("SELECT l.dr_artist_id, t.friday, t.followers FROM nmf_artist_tracks t JOIN nmf_artist_lookups l ON l.spotify_artist_id = t.spotify_artist_id WHERE t.playlists = 1")?
|
||||
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, Option<i64>>(2)?)))? {
|
||||
let (id, f, fo) = row?;
|
||||
if let (true, Some(fo)) = (danish.contains(id.as_str()), fo) { *counts.entry((f, fo)).or_insert(0) += 1; }
|
||||
}
|
||||
let mut best: HashMap<String, (i64, usize)> = HashMap::new();
|
||||
for ((f, fo), n) in counts { let e = best.entry(f).or_insert((fo, 0)); if n > e.1 { *e = (fo, n); } }
|
||||
for (f, (fo, n)) in best {
|
||||
if n < 2 { continue; }
|
||||
if let Ok(d) = NaiveDate::parse_from_str(&f, "%Y-%m-%d") {
|
||||
if !dk_followers.iter().any(|x| x.0 == d) { dk_followers.push((d, fo as f64)); }
|
||||
}
|
||||
}
|
||||
}
|
||||
dk_followers.sort_by_key(|x| x.0);
|
||||
let dk_size = |d: NaiveDate| dk_followers.iter().min_by_key(|(x, _)| (*x - d).num_days().abs()).map(|x| x.1);
|
||||
|
||||
// Placements per DR artist: (friday, isrc, playlists, followers).
|
||||
let mut placements: HashMap<usize, Vec<(NaiveDate, String, i64, i64)>> = HashMap::new();
|
||||
let mut looked_up = vec![false; p.a()];
|
||||
if has("nmf_artist_tracks")? {
|
||||
for row in db.conn.prepare("SELECT l.dr_artist_id FROM nmf_artist_lookups l WHERE l.spotify_artist_id IS NOT NULL")?.query_map([], |r| r.get::<_, String>(0))? {
|
||||
if let Some(&a) = id_of.get(row?.as_str()) { looked_up[a] = true; }
|
||||
}
|
||||
for row in db.conn.prepare(
|
||||
"SELECT l.dr_artist_id, t.friday, t.isrc, coalesce(t.playlists, 0), coalesce(t.followers, 0)
|
||||
FROM nmf_artist_tracks t JOIN nmf_artist_lookups l ON l.spotify_artist_id = t.spotify_artist_id")?
|
||||
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, i64>(3)?, r.get::<_, i64>(4)?)))? {
|
||||
let (id, f, isrc, pl, fo) = row?;
|
||||
if let (Some(&a), Ok(d)) = (id_of.get(id.as_str()), NaiveDate::parse_from_str(&f, "%Y-%m-%d")) {
|
||||
placements.entry(a).or_default().push((d, isrc, pl, fo));
|
||||
}
|
||||
}
|
||||
}
|
||||
let classify = |a: usize, rel: NaiveDate| -> (Nmf, Option<i64>, &'static str) {
|
||||
let mut best = (Nmf::None, None, "");
|
||||
for (d, isrc, pl, fo) in placements.get(&a).into_iter().flatten() {
|
||||
if (*d - rel).num_days().abs() > 7 { continue; }
|
||||
let key = (isrc.clone(), d.to_string());
|
||||
if let Some(pos) = dk_track.get(&key) { return (Nmf::Dk, *pos, "track page"); }
|
||||
if seen_track.contains_key(&key) { if best.0 == Nmf::None { best = (Nmf::Elsewhere, None, "track page"); } continue; }
|
||||
// Not read on the track page: one playlist the size of NMF Denmark that week, Danish act.
|
||||
let looks_dk = *pl >= 1 && dk_size(*d).is_some_and(|n| (*fo as f64 - n).abs() <= 0.02 * n || (*pl > 1 && *fo as f64 >= n)) && p.artists[a].danish;
|
||||
if looks_dk { best = (Nmf::Dk, None, "followers"); } else if best.0 == Nmf::None { best = (Nmf::Elsewhere, None, "followers"); }
|
||||
}
|
||||
best
|
||||
};
|
||||
|
||||
// Events.
|
||||
struct Ev { nmf: Nmf, pos: Option<i64>, add_any: bool, add_grp: [bool; 5], weeks_to_add: Option<usize>, x: Vec<f64>, in_rot: bool, danish: bool }
|
||||
let week_of: HashMap<NaiveDate, usize> = p.weeks.iter().enumerate().map(|(i, d)| (*d, i)).collect();
|
||||
let mut evs: Vec<Ev> = Vec::new();
|
||||
let mut via_followers = 0;
|
||||
for a in 0..p.a() {
|
||||
if !looked_up[a] { continue; }
|
||||
let Some(rels) = p.releases[a].as_ref() else { continue };
|
||||
let mut last: Option<usize> = None;
|
||||
for rel in rels {
|
||||
// The DR week containing the release (Monday start).
|
||||
let monday = *rel - Duration::days(chrono::Datelike::weekday(rel).num_days_from_monday() as i64);
|
||||
let Some(&w) = week_of.get(&monday) else { continue };
|
||||
if w < 12 || w + WIN >= p.w() { continue; }
|
||||
if last == Some(w) { continue; } // one event per artist-week
|
||||
last = Some(w);
|
||||
let (nmf, pos, how) = classify(a, *rel);
|
||||
if how == "followers" && nmf == Nmf::Dk { via_followers += 1; }
|
||||
// Outcome: an add in the release week or the 8 after it (artist level, any channel).
|
||||
let first_add = (w..=w + WIN).find(|&k| r.add_any(a, k));
|
||||
let groups: [Vec<usize>; 5] = [vec![0], (0..C).filter(|&c| is_p4(c)).collect(), vec![11], vec![12], vec![13]];
|
||||
let add_grp = groups.clone().map(|g| (w..=w + WIN).any(|k| g.iter().any(|&c| r.add(a, c, k))));
|
||||
let t = w - 1; // known before the release week
|
||||
let tot = |k: usize| (0..C).map(|c| p.y(a, c, k)).sum::<f64>();
|
||||
let lvl = ((0..12).map(|i| tot(t - i)).sum::<f64>() / 12.0).ln_1p();
|
||||
let in_rot = r.any(a, t);
|
||||
let ch = p.chart(a, t);
|
||||
let x = vec![1.0, (nmf == Nmf::Dk) as i32 as f64, (nmf == Nmf::Elsewhere) as i32 as f64, lvl, in_rot as i32 as f64,
|
||||
p.artists[a].danish as i32 as f64, (ch.air_hits as f64).ln_1p(), (ch.t40_hits as f64).ln_1p()];
|
||||
evs.push(Ev { nmf, pos, add_any: first_add.is_some(), add_grp, weeks_to_add: first_add.map(|k| k - w), x, in_rot, danish: p.artists[a].danish });
|
||||
}
|
||||
}
|
||||
|
||||
let mut md = String::new();
|
||||
writeln!(md, "# New Music Friday Denmark and DR adds\n")?;
|
||||
writeln!(md, "Releases by artists in the DR panel (Deezer discographies), in DR weeks {} to {} with {WIN} weeks of follow-up, whose New Music Friday history was looked up. An add = entering rotation on a DR channel after at least {GAP} weeks out of rotation everywhere (rotation thresholds from DR's lists). Matched to NMF placements within a week of the release date. NMF Denmark recognised from the track page or, before that's read, by one playlist the size of NMF Denmark for a Danish act ({via_followers} events).\n",
|
||||
p.weeks[12], p.weeks[p.w() - 1 - WIN])?;
|
||||
if evs.is_empty() { writeln!(md, "No events yet: discographies and NMF artist pages are still being collected.")?; return Ok(md); }
|
||||
let rate = |f: &dyn Fn(&Ev) -> bool, out: &dyn Fn(&Ev) -> bool| { let v: Vec<&Ev> = evs.iter().filter(|e| f(e)).collect(); (v.len(), if v.is_empty() { f64::NAN } else { v.iter().filter(|e| out(e)).count() as f64 / v.len() as f64 }) };
|
||||
writeln!(md, "| Group | Releases | DR add within {WIN} weeks | P3 | P4 | P5 | P6 | P8 |\n|---|---|---|---|---|---|---|---|")?;
|
||||
let groups: Vec<(&str, Box<dyn Fn(&Ev) -> bool>)> = vec![
|
||||
("NMF Denmark", Box::new(|e: &Ev| e.nmf == Nmf::Dk)),
|
||||
("NMF Denmark, top 10 position", Box::new(|e: &Ev| e.nmf == Nmf::Dk && e.pos.is_some_and(|p| p <= 10))),
|
||||
("NMF elsewhere only", Box::new(|e: &Ev| e.nmf == Nmf::Elsewhere)),
|
||||
("Not on NMF", Box::new(|e: &Ev| e.nmf == Nmf::None)),
|
||||
("Danish acts: NMF Denmark", Box::new(|e: &Ev| e.danish && e.nmf == Nmf::Dk)),
|
||||
("Danish acts: not on NMF", Box::new(|e: &Ev| e.danish && e.nmf == Nmf::None)),
|
||||
("Not in rotation at release: NMF Denmark", Box::new(|e: &Ev| !e.in_rot && e.nmf == Nmf::Dk)),
|
||||
("Not in rotation at release: not on NMF", Box::new(|e: &Ev| !e.in_rot && e.nmf == Nmf::None)),
|
||||
];
|
||||
for (name, f) in &groups {
|
||||
let (n, any) = rate(f.as_ref(), &|e| e.add_any);
|
||||
let g: Vec<String> = (0..5).map(|k| { let (_, v) = rate(f.as_ref(), &|e| e.add_grp[k]); if v.is_nan() { "—".into() } else { format!("{:.0}%", 100.0 * v) } }).collect();
|
||||
writeln!(md, "| {name} | {n} | {} | {} |", if any.is_nan() { "—".into() } else { format!("{:.1}%", 100.0 * any) }, g.join(" | "))?;
|
||||
}
|
||||
// Timing.
|
||||
let mut lag: Vec<usize> = evs.iter().filter(|e| e.nmf == Nmf::Dk).filter_map(|e| e.weeks_to_add).collect();
|
||||
lag.sort();
|
||||
if !lag.is_empty() {
|
||||
let hist: Vec<String> = (0..=WIN).map(|k| format!("wk {k}: {}", lag.iter().filter(|&&l| l == k).count())).collect();
|
||||
writeln!(md, "\nWeeks from the release to the add, NMF Denmark songs that got one: median {}; {}.\n", lag[lag.len() / 2], hist.join(", "))?;
|
||||
}
|
||||
// Logit with controls; standard errors from the information matrix.
|
||||
let names = ["intercept", "NMF Denmark", "NMF elsewhere only", "log DR plays/week, last 12 weeks", "in rotation at release", "Danish act", "log Airplay Top-20 hits", "log Track Top-40 hits"];
|
||||
let mut d = Design::new(names.len());
|
||||
for e in &evs { d.push(&e.x, e.add_any as i32 as f64, 1.0); }
|
||||
let b = glm::logit(&d, 0.1);
|
||||
let se = std_errors(&evs.iter().map(|e| e.x.clone()).collect::<Vec<_>>(), &b);
|
||||
writeln!(md, "\nLogit for a DR add within {WIN} weeks of a release ({} releases):\n\n| Covariate | Coefficient | Odds ratio (95% CI) |\n|---|---|---|", evs.len())?;
|
||||
for (k, n) in names.iter().enumerate() {
|
||||
let (lo, hi) = (b[k] - 1.96 * se[k], b[k] + 1.96 * se[k]);
|
||||
writeln!(md, "| {n} | {:+.3} | {:.2} ({:.2}–{:.2}) |", b[k], b[k].exp(), lo.exp(), hi.exp())?;
|
||||
}
|
||||
writeln!(md, "\nObservational: NMF picks and radio adds share causes (label push, artist momentum), so the odds ratio is an association after the listed controls, not the effect of being picked.\n")?;
|
||||
Ok(md)
|
||||
}
|
||||
|
||||
fn std_errors(xs: &[Vec<f64>], b: &[f64]) -> Vec<f64> {
|
||||
let p = b.len();
|
||||
let mut h = vec![0.0; p * p];
|
||||
for x in xs {
|
||||
let mu = glm::sigmoid(glm::dot(b, x));
|
||||
let w = mu * (1.0 - mu);
|
||||
for i in 0..p { for j in 0..p { h[i * p + j] += w * x[i] * x[j]; } }
|
||||
}
|
||||
for i in 0..p { h[i * p + i] += 1e-6; }
|
||||
// invert by Gauss-Jordan
|
||||
let mut inv = vec![0.0; p * p];
|
||||
for i in 0..p { inv[i * p + i] = 1.0; }
|
||||
for c in 0..p {
|
||||
let piv = (c..p).max_by(|&a, &bb| h[a * p + c].abs().total_cmp(&h[bb * p + c].abs())).unwrap();
|
||||
for k in 0..p { h.swap(c * p + k, piv * p + k); inv.swap(c * p + k, piv * p + k); }
|
||||
let d = h[c * p + c];
|
||||
if d.abs() < 1e-12 { return vec![f64::NAN; p]; }
|
||||
for k in 0..p { h[c * p + k] /= d; inv[c * p + k] /= d; }
|
||||
for rr in 0..p {
|
||||
if rr == c { continue; }
|
||||
let f = h[rr * p + c];
|
||||
for k in 0..p { h[rr * p + k] -= f * h[c * p + k]; inv[rr * p + k] -= f * inv[c * p + k]; }
|
||||
}
|
||||
}
|
||||
(0..p).map(|i| inv[i * p + i].max(0.0).sqrt()).collect()
|
||||
}
|
||||
+58
-1
@@ -83,8 +83,13 @@ pub fn snapshot(db: &Db, http: &Http, log: &mut Vec<String>) -> Result<usize> {
|
||||
pub fn nmf_history(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) -> Result<usize> {
|
||||
create_tables(db)?;
|
||||
crate::deezer::create_tables(db)?;
|
||||
db.conn.execute_batch("CREATE TABLE IF NOT EXISTS nmf_artist_tracks (spotify_artist_id TEXT NOT NULL, isrc TEXT NOT NULL, title TEXT,
|
||||
friday TEXT NOT NULL, playlists INTEGER, followers INTEGER, PRIMARY KEY (spotify_artist_id, isrc, friday));")?;
|
||||
// Placements found on artist pages inside the modelling window first, then every ISRC Deezer gave us.
|
||||
let todo: Vec<String> = db.conn.prepare(
|
||||
"SELECT DISTINCT isrc FROM deezer_tracks WHERE coalesce(isrc, '') != '' AND isrc NOT IN (SELECT isrc FROM nmf_lookups)")?
|
||||
"SELECT isrc FROM (SELECT isrc, 0 AS o FROM nmf_artist_tracks WHERE friday >= '2025-11-01'
|
||||
UNION SELECT isrc, 1 FROM deezer_tracks WHERE coalesce(isrc, '') != '')
|
||||
WHERE isrc NOT IN (SELECT isrc FROM nmf_lookups) GROUP BY isrc ORDER BY min(o)")?
|
||||
.query_map([], |r| r.get(0))?.collect::<Result<_, _>>()?;
|
||||
let day = Regex::new(r#"<h2[^>]*>\s*(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday), ([A-Za-z]+ \d{2}, \d{4})\s*</h2>"#)?;
|
||||
let row = Regex::new(r#"(?s)<p[^>]*>([^<]+)</p>\s*<a href="https://open\.spotify\.com/playlist/([A-Za-z0-9]{22})".*?<span[^>]*>([\d,]+) followers</span>\s*<span[^>]*>Position (\d+)</span>"#)?;
|
||||
@@ -114,3 +119,55 @@ pub fn nmf_history(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) ->
|
||||
}
|
||||
Ok(done)
|
||||
}
|
||||
|
||||
/// New Music Friday placements per artist (SuperFridayChart's artist page, by the Spotify artist
|
||||
/// id DR links on its artist page): every track of the artist added to any NMF playlist since
|
||||
/// 2022, with the Friday, number of playlists and their followers. Complete per artist, so songs
|
||||
/// DR never played are in it too. Which countries: the track page (`nmf_history`), fetched for
|
||||
/// placements inside the modelling window.
|
||||
pub fn nmf_artists(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) -> Result<usize> {
|
||||
create_tables(db)?;
|
||||
db.conn.execute_batch(
|
||||
"CREATE TABLE IF NOT EXISTS nmf_artist_tracks (
|
||||
spotify_artist_id TEXT NOT NULL, isrc TEXT NOT NULL, title TEXT, friday TEXT NOT NULL,
|
||||
playlists INTEGER, followers INTEGER,
|
||||
PRIMARY KEY (spotify_artist_id, isrc, friday)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS nmf_artist_lookups (
|
||||
dr_artist_id TEXT PRIMARY KEY, spotify_artist_id TEXT, tracks INTEGER, fetched_at TEXT NOT NULL
|
||||
);",
|
||||
)?;
|
||||
let link = Regex::new(r"open\.spotify\.com/artist/([A-Za-z0-9]{22})")?;
|
||||
let todo: Vec<(String, String)> = db.conn.prepare(
|
||||
"SELECT artist_id, raw FROM dr_artists WHERE artist_id NOT IN (SELECT dr_artist_id FROM nmf_artist_lookups)")?
|
||||
.query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?.collect::<Result<_, _>>()?;
|
||||
// One row per track: the track link, then its date, playlist count and followers.
|
||||
let row = Regex::new(r#"(?s)/track/([A-Z0-9]{12})"[^>]*>([^<]*)</a>.*?([A-Z][a-z]{2} \d{2}, \d{4})\s*</?[^>]*>?.*?(\d+) playlists.*?([\d,]+) followers"#)?;
|
||||
let mut done = 0;
|
||||
for (dr_id, raw) in todo.into_iter().take(limit) {
|
||||
let sid = link.captures(&raw).map(|c| c[1].to_string());
|
||||
let mut n = 0;
|
||||
if let Some(sid) = &sid {
|
||||
match http.get(&format!("https://www.superfridaychart.com/artist/{sid}")) {
|
||||
Ok(Some(html)) => {
|
||||
// Split at each track link so a row's fields can't run into the next track.
|
||||
let starts: Vec<usize> = html.match_indices("superfridaychart.com/track/").map(|(i, _)| i).collect();
|
||||
for (k, s) in starts.iter().enumerate() {
|
||||
let block = &html[s.saturating_sub(1)..starts.get(k + 1).copied().unwrap_or(html.len())];
|
||||
let Some(c) = row.captures(block) else { continue };
|
||||
let Ok(d) = NaiveDate::parse_from_str(&c[3], "%b %d, %Y") else { continue };
|
||||
db.conn.execute("INSERT OR REPLACE INTO nmf_artist_tracks VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
|
||||
params![sid, &c[1], c[2].trim().replace("&", "&").replace("'", "'"), d.to_string(),
|
||||
c[4].parse::<i64>().ok(), c[5].replace(',', "").parse::<i64>().ok()])?;
|
||||
n += 1;
|
||||
}
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(e) => { log.push(format!("superfridaychart artist {sid}: {e}")); continue; }
|
||||
}
|
||||
}
|
||||
db.conn.execute("INSERT OR REPLACE INTO nmf_artist_lookups VALUES (?1, ?2, ?3, ?4)", params![dr_id, sid, n as i64, Utc::now().to_rfc3339()])?;
|
||||
done += 1;
|
||||
}
|
||||
Ok(done)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user