From 992cc2b1576929f8ca506ae0d8387f839b2c4eac Mon Sep 17 00:00:00 2001 From: moritsbw-cpu Date: Sun, 15 Nov 2026 03:56:10 +0000 Subject: [PATCH] Wave features (time since the last DR add / Airplay Top-20 wave, peak band 13-25 weeks, 0.5-1.25x the artist's usual gap); ablation switch (model adds --without rel,wave); fix the per-channel row slice Co-Authored-By: Claude Opus 5.5 --- src/export.rs | 86 ++++++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 14 ++++++-- src/model/adds.rs | 42 ++++++++++++++++++---- src/model/mod.rs | 1 + src/model/panel.rs | 17 ++++++++- src/playlists.rs | 64 +++++++++++++++++++++++++++++++--- 6 files changed, 209 insertions(+), 15 deletions(-) create mode 100644 src/export.rs diff --git a/src/export.rs b/src/export.rs new file mode 100644 index 0000000..3557a5c --- /dev/null +++ b/src/export.rs @@ -0,0 +1,86 @@ +//! Data exports. +//! +//! airplay export nmf-denmark [--out data/nmf_denmark.csv] +//! +//! New Music Friday Denmark, one row per Friday and position, as far back as SuperFridayChart +//! goes (January 2022): title, artists, ISRC, position, playlist followers. Rebuilt from +//! SuperFridayChart's track pages, so a week is only as complete as the tracks looked up so far; +//! a second file (…_coverage.csv) lists per Friday how many positions are found and which are +//! missing. The playlist snapshots taken from 15 Nov 2026 on are complete and are added with +//! source "spotify snapshot" (Spotify track id; ISRC where SuperFridayChart has the same song). + +use crate::db::Db; +use anyhow::Result; +use std::collections::{BTreeMap, HashMap}; +use std::io::Write; + +fn csv(s: &str) -> String { + if s.contains([',', '"', '\n']) { format!("\"{}\"", s.replace('"', "\"\"")) } else { s.to_string() } +} + +pub fn run(db: &Db, args: &[String]) -> Result<()> { + match args.get(2).map(String::as_str) { + Some("nmf-denmark") => { + let out = args.iter().position(|a| a == "--out").and_then(|i| args.get(i + 1)).cloned().unwrap_or_else(|| "data/nmf_denmark.csv".into()); + nmf_denmark(db, &out) + } + _ => { eprintln!("usage: airplay export nmf-denmark [--out FILE]"); std::process::exit(2); } + } +} + +fn nmf_denmark(db: &Db, out: &str) -> Result<()> { + crate::playlists::create_tables(db)?; + for col in ["title TEXT", "artists TEXT"] { let _ = db.conn.execute(&format!("ALTER TABLE nmf_lookups ADD COLUMN {col}"), []); } + let mut f = std::fs::File::create(out)?; + writeln!(f, "friday,position,title,artists,isrc,spotify_track_id,playlist_followers,source")?; + let mut by_friday: BTreeMap> = BTreeMap::new(); + let mut rows = 0; + // Title/artist -> ISRC, to give snapshot rows an ISRC when the song was looked up. + let key = |t: &str, a: &str| format!("{}|{}", crate::model::panel::key(t), crate::model::panel::key(a.split(',').next().unwrap_or(a))); + let mut isrc_of: HashMap = HashMap::new(); + let mut st = db.conn.prepare( + "SELECT n.friday, n.position, coalesce(l.title, ''), coalesce(l.artists, ''), n.isrc, n.followers + FROM nmf_additions n LEFT JOIN nmf_lookups l ON l.isrc = n.isrc + WHERE lower(n.country) = 'denmark' ORDER BY n.friday, n.position")?; + let it = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, Option>(1)?, r.get::<_, String>(2)?, r.get::<_, String>(3)?, + r.get::<_, String>(4)?, r.get::<_, Option>(5)?)))?; + for r in it { + let (friday, pos, title, artists, isrc, followers) = r?; + isrc_of.insert(key(&title, &artists), isrc.clone()); + writeln!(f, "{friday},{},{},{},{isrc},,{},superfridaychart", pos.map(|p| p.to_string()).unwrap_or_default(), csv(&title), csv(&artists), + followers.map(|x| x.to_string()).unwrap_or_default())?; + if let Some(p) = pos { by_friday.entry(friday).or_default().push(p); } + rows += 1; + } + // Complete snapshots of the live playlist (each distinct list once, dated by its Friday). + let mut st = db.conn.prepare( + "SELECT taken_at, position, coalesce(title, ''), coalesce(artists, ''), track_id FROM playlist_snapshots + WHERE playlist_id = '37i9dQZF1DWWuIAAkPM0Lu' ORDER BY taken_at, position")?; + let it = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?, r.get::<_, String>(2)?, r.get::<_, String>(3)?, r.get::<_, String>(4)?)))?; + let mut snap_rows = 0; + for r in it { + let (taken, pos, title, artists, track) = r?; + // The Friday on or before the snapshot. + let d = chrono::NaiveDate::parse_from_str(&taken[..10], "%Y-%m-%d")?; + let back = (chrono::Datelike::weekday(&d).num_days_from_monday() as i64 + 3) % 7; + let friday = (d - chrono::Duration::days(back)).to_string(); + let isrc = isrc_of.get(&key(&title, &artists)).cloned().unwrap_or_default(); + writeln!(f, "{friday},{pos},{},{},{isrc},{track},,spotify snapshot {}", csv(&title), csv(&artists), &taken[..10])?; + snap_rows += 1; + } + // Coverage per Friday: positions run 1..N, so gaps are exactly the songs not found yet. + let cov = out.replace(".csv", "_coverage.csv"); + let mut c = std::fs::File::create(&cov)?; + writeln!(c, "friday,positions_found,highest_position,missing_positions")?; + let (mut found_all, mut max_all) = (0usize, 0i64); + for (friday, mut ps) in by_friday { + ps.sort(); ps.dedup(); + let max = *ps.last().unwrap_or(&0); + let missing: Vec = (1..=max).filter(|p| ps.binary_search(p).is_err()).map(|p| p.to_string()).collect(); + writeln!(c, "{friday},{},{max},{}", ps.len(), csv(&missing.join(" ")))?; + found_all += ps.len(); max_all += max; + } + println!("{out}: {rows} rows from SuperFridayChart track pages, {snap_rows} from playlist snapshots"); + println!("{cov}: {found_all} of {max_all} positions found ({:.0}%) across the Fridays seen", 100.0 * found_all as f64 / max_all.max(1) as f64); + Ok(()) +} diff --git a/src/main.rs b/src/main.rs index 0300ca9..456c956 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,7 @@ //! airplay: collects what Danish radio stations play, for modelling when artists get played again. //! -//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf|nmf-artists] [--backfill] +//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks] [--backfill] +//! airplay export nmf-denmark [--out data/nmf_denmark.csv] //! airplay stats [--db data/airplay.db] //! airplay check [--db data/airplay.db] gaps, duplicates, DR cross-check, chart completeness //! airplay model panel | backtest | rank the first models (src/model) @@ -14,6 +15,7 @@ mod bauer; mod check; mod db; mod deezer; +mod export; mod dr; mod hitlisten; mod http; @@ -45,6 +47,7 @@ fn main() -> Result<()> { Some("check") => check::run(&db), Some("model") => model::run(&db, &args), Some("serve") => web::serve(&db_path, &args), + Some("export") => export::run(&db, &args), Some("pdfdump") => rotations::dump(std::path::Path::new(args.get(2).map(String::as_str).unwrap_or(""))), Some("rotations") => { let dir = std::path::Path::new(&db_path).parent().unwrap_or(std::path::Path::new(".")).join("rotations"); @@ -57,7 +60,8 @@ fn main() -> Result<()> { rotations::store_all(&db, &dir, args.iter().any(|a| a == "--reparse")) } _ => { - eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf|nmf-artists] [--backfill]\n airplay stats [--db FILE]\n airplay check [--db FILE]\n airplay model panel|backtest|rank\n airplay serve [--addr HOST:PORT] [--prefix /airplay]"); + eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks] [--backfill] +//! airplay export nmf-denmark [--out data/nmf_denmark.csv]\n airplay stats [--db FILE]\n airplay check [--db FILE]\n airplay model panel|backtest|rank\n airplay serve [--addr HOST:PORT] [--prefix /airplay]"); std::process::exit(2); } } @@ -135,7 +139,11 @@ fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> { let r = playlists::nmf_artists(db, &sfc_http, if backfill { usize::MAX } else { 800 }, &mut log); step("new music friday per artist (superfridaychart)", r.map(|n| format!("{n} artists looked up")), &mut log); } - if wants("nmf") { + if wants("nmf") || wants("nmf-charts") { + let r = playlists::nmf_global_charts(db, &http::Http::new(1000)?, &mut log); + step("new music friday global charts (superfridaychart)", r.map(|n| format!("{n} Fridays read")), &mut log); + } + if wants("nmf") || wants("nmf-tracks") { let sfc_http = http::Http::new(1000)?; let r = playlists::nmf_history(db, &sfc_http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 1500 }), &mut log); step("new music friday history (superfridaychart)", r.map(|n| format!("{n} ISRCs looked up")), &mut log); diff --git a/src/model/adds.rs b/src/model/adds.rs index c963a0b..f149903 100644 --- a/src/model/adds.rs +++ b/src/model/adds.rs @@ -27,6 +27,31 @@ pub const GAP: usize = 4; pub const W: usize = 8; /// Per-channel rows: the panel features plus these. +/// Wave covariates: the chance of a next wave peaks 13–25 weeks after the last one and at +/// 0.5–1.25x the artist's usual gap (25 years of Airplay Top-20: weekly hazard 1.7% at 13–25 +/// weeks vs 0.5% before and 0.3–0.4% after a year). Appended after the release block. +pub const WAVE: usize = 6; + +/// Feature blocks switched off for an ablation run ("rel", "wave"). +pub static WITHOUT: std::sync::OnceLock> = std::sync::OnceLock::new(); +fn off(block: &str) -> bool { WITHOUT.get().is_some_and(|v| v.iter().any(|b| b == block)) } +pub const WAVE_NAMES: &[&str] = &["DR: weeks since last add 13-25", "DR: last add 26+ weeks ago", "DR: no add in the panel yet", + "Airplay Top-20: last new song 13-25 weeks ago", "Airplay Top-20: 0.5-1.25x own usual gap", "Airplay Top-20: last new song 2+ years ago"]; + +/// Weeks since the artist's last add on any DR channel (as of week t; None = none in the panel), +/// and the IFPI wave position. +pub fn wave_feats(p: &Panel, r: &Rot, a: usize, t: usize) -> [f64; WAVE] { + if off("wave") { return [0.0; WAVE]; } + let since = (0..=t).find(|&i| r.add_any(a, t - i)); + let ch = p.chart(a, t); + let f = |b: bool| b as i32 as f64; + let air = ch.air_since as f64; + let known = ch.air_since != u16::MAX; + let ratio_ok = known && ch.air_gap > 0 && { let x = air / ch.air_gap as f64; (0.5..=1.25).contains(&x) }; + [f(since.is_some_and(|s| (13..=25).contains(&s))), f(since.is_some_and(|s| s >= 26)), f(since.is_none()), + f(known && (13.0..=25.0).contains(&air)), f(ratio_ok), f(known && air >= 104.0)] +} + /// Release covariates (Deezer discography), appended to both row kinds. pub const REL: usize = 6; pub const REL_NAMES: &[&str] = &["discography known", "release recency 4w", "release recency 12w", "log releases last 52w", @@ -36,6 +61,7 @@ pub const REL_NAMES: &[&str] = &["discography known", "release recency 4w", "rel /// the window t+h+1 ..= t+h+W (the actual one when `rel` is None: training and backtests; else the /// scenario "releases / doesn't release in the window"). pub fn rel_feats(p: &Panel, a: usize, t: usize, h: usize, rel: Option, in_rot_now: bool) -> [f64; REL] { + if off("rel") { return [0.0; REL]; } let Some(dates) = p.releases[a].as_ref() else { return [0.0; REL] }; let end_t = p.weeks[t] + chrono::Duration::days(6); let since = dates.iter().rev().find(|d| **d <= end_t).map(|d| (end_t - *d).num_days() as f64 / 7.0); @@ -62,7 +88,7 @@ pub const EXTRA: &[&str] = &["in rotation on channel now", "in rotation anywhere "run on channel = 1 week", "run = 2 weeks", "run = 3-5 weeks", "run >= 6 weeks", "near threshold (>= half of tau, not in rotation)", "plays / tau (capped 3, /3)", "weeks since in rotation on channel (/12)"]; -pub const PC: usize = P + 10 + REL; +pub const PC: usize = P + 10 + REL + WAVE; /// Artist-level ("any channel") rows. pub const ANY_NAMES: &[&str] = &[ @@ -72,7 +98,7 @@ pub const ANY_NAMES: &[&str] = &[ "log Track Top-40 hits", "log Airplay Top-20 hits", "Danish", "group", "female", "career years (/50)", "debut unknown", "Christmas in window", ]; -pub const PA: usize = 21 + REL; +pub const PA: usize = 21 + REL + WAVE; /// Positions of the chart covariates (for the what-if): (t40 score, air score, recency 6w, recency 26w). pub const CHART_IDX_CHANNEL: [usize; 4] = [10, 11, 12, 13]; pub const CHART_IDX_ANY: [usize; 4] = [9, 10, 11, 12]; @@ -124,10 +150,11 @@ pub fn row_channel(p: &Panel, r: &Rot, a: usize, c: usize, t: usize, h: usize, r let on = r.on(a, c, t); let ratio = p.y(a, c, t) / tau_of(p, r, a, c, t); let f = |b: bool| b as i32 as f64; - out[P..PC].copy_from_slice(&[f(on), f(r.any(a, t)), n_on / C as f64, + out[P..P + 10].copy_from_slice(&[f(on), f(r.any(a, t)), n_on / C as f64, f(weeks == 1), f(weeks == 2), f((3..=5).contains(&weeks)), f(weeks >= 6), f(!on && ratio >= 0.5), ratio.min(3.0) / 3.0, since]); - out[P + 10..PC].copy_from_slice(&rel_feats(p, a, t, h, rel, on)); + out[P + 10..P + 10 + REL].copy_from_slice(&rel_feats(p, a, t, h, rel, on)); + out[P + 10 + REL..PC].copy_from_slice(&wave_feats(p, r, a, t)); } pub fn row_any(p: &Panel, r: &Rot, a: usize, t: usize, h: usize, rel: Option, out: &mut [f64]) { @@ -151,7 +178,8 @@ pub fn row_any(p: &Panel, r: &Rot, a: usize, t: usize, h: usize, rel: Option Result<()> { print!("{md}"); } Some("adds") => { + if let Some(w) = arg(args, "--without") { let _ = adds::WITHOUT.set(w.split(',').map(str::to_string).collect()); } let horizons: Vec = arg(args, "--h").unwrap_or_else(|| "0,4".into()).split(',').filter_map(|s| s.trim().parse().ok()).collect(); let tau = common::thresholds(&p, 26); let mut md = String::new(); diff --git a/src/model/panel.rs b/src/model/panel.rs index a9b54f5..a4a9583 100644 --- a/src/model/panel.rs +++ b/src/model/panel.rs @@ -41,6 +41,10 @@ pub struct Chart { /// Distinct songs on the Track Top-40 / Airplay Top-20 so far. pub t40_hits: u16, pub air_hits: u16, + /// Airplay Top-20 "waves": weeks since the artist's latest new song on the Airplay Top-20 + /// (u16::MAX = never), and the median gap between their earlier new songs (0 = fewer than 2 gaps). + pub air_since: u16, + pub air_gap: u16, } pub struct Panel { @@ -170,10 +174,12 @@ fn chart_states(db: &Db, weeks: &[NaiveDate], names: &HashMap for entries in &by_artist { for w in 0..nw { let cutoff = cutoffs[w]; - let mut s = Chart { t40_rank: 0, air_rank: 0, weeks_since_entry: u16::MAX, t40_hits: 0, air_hits: 0 }; + let mut s = Chart { t40_rank: 0, air_rank: 0, weeks_since_entry: u16::MAX, t40_hits: 0, air_hits: 0, air_since: u16::MAX, air_gap: 0 }; + let mut air_firsts: BTreeSet = BTreeSet::new(); let (mut t40_songs, mut air_songs) = (BTreeSet::new(), BTreeSet::new()); for e in entries.iter().filter(|e| e.monday <= cutoff) { if e.t40 { t40_songs.insert(&e.song); } else { air_songs.insert(&e.song); } + if !e.t40 && e.new { air_firsts.insert(e.monday); } let best = |cur: u8| if cur == 0 { e.rank } else { cur.min(e.rank) }; if e.t40 && Some(e.monday) == latest_t40[w] { s.t40_rank = best(s.t40_rank); } if !e.t40 && Some(e.monday) == latest_air[w] { s.air_rank = best(s.air_rank); } @@ -184,6 +190,15 @@ fn chart_states(db: &Db, weeks: &[NaiveDate], names: &HashMap } s.t40_hits = t40_songs.len() as u16; s.air_hits = air_songs.len() as u16; + // Waves: new songs at least 2 weeks apart. + let mut waves: Vec = Vec::new(); + for d in &air_firsts { if waves.last().is_none_or(|l| (*d - *l).num_days() >= 14) { waves.push(*d); } } + if let Some(last) = waves.last() { s.air_since = ((cutoff - *last).num_days() / 7).clamp(0, u16::MAX as i64 - 1) as u16; } + if waves.len() >= 3 { + let mut gaps: Vec = waves.windows(2).map(|w| (w[1] - w[0]).num_days() / 7).collect(); + gaps.sort(); + s.air_gap = gaps[gaps.len() / 2].clamp(1, 2000) as u16; + } out.push(s); } } diff --git a/src/playlists.rs b/src/playlists.rs index 6044f9b..f48d6e4 100644 --- a/src/playlists.rs +++ b/src/playlists.rs @@ -85,12 +85,35 @@ pub fn nmf_history(db: &Db, http: &Http, limit: usize, log: &mut Vec) -> crate::deezer::create_tables(db)?; db.conn.execute_batch("CREATE TABLE IF NOT EXISTS nmf_artist_tracks (spotify_artist_id TEXT NOT NULL, isrc TEXT NOT NULL, title TEXT, friday TEXT NOT NULL, playlists INTEGER, followers INTEGER, PRIMARY KEY (spotify_artist_id, isrc, friday));")?; - // Placements found on artist pages inside the modelling window first, then every ISRC Deezer gave us. + // Title and artists of each looked-up track (added later; older rows have them empty). + for col in ["title TEXT", "artists TEXT"] { + let _ = db.conn.execute(&format!("ALTER TABLE nmf_lookups ADD COLUMN {col}"), []); + } + db.conn.execute_batch("CREATE TABLE IF NOT EXISTS nmf_global_chart (friday TEXT NOT NULL, rank INTEGER NOT NULL, isrc TEXT NOT NULL, + PRIMARY KEY (friday, rank));")?; + // Which tracks to read, most likely to be on NMF Denmark first: + // 0 artist-page placements of Danish acts (DR artist country), or one playlist the size of + // NMF Denmark that Friday (the most common single-playlist follower count among Danish acts) + // 1 SuperFridayChart's weekly global top 50 + // 2 other artist-page placements on several playlists + // 3 every ISRC Deezer gave us let todo: Vec = db.conn.prepare( - "SELECT isrc FROM (SELECT isrc, 0 AS o FROM nmf_artist_tracks WHERE friday >= '2025-11-01' - UNION SELECT isrc, 1 FROM deezer_tracks WHERE coalesce(isrc, '') != '') + "WITH danish AS (SELECT l.spotify_artist_id FROM nmf_artist_lookups l JOIN dr_artists a ON a.artist_id = l.dr_artist_id + WHERE json_extract(a.raw, '$.artist.country') = 'Danmark' AND l.spotify_artist_id IS NOT NULL), + dk_size AS (SELECT friday, followers FROM ( + SELECT t.friday, t.followers, count(*) n, row_number() OVER (PARTITION BY t.friday ORDER BY count(*) DESC) rn + FROM nmf_artist_tracks t WHERE t.playlists = 1 AND t.spotify_artist_id IN (SELECT spotify_artist_id FROM danish) + GROUP BY t.friday, t.followers) WHERE rn = 1) + SELECT isrc FROM ( + SELECT t.isrc, 0 AS o FROM nmf_artist_tracks t LEFT JOIN dk_size d ON d.friday = t.friday + WHERE t.spotify_artist_id IN (SELECT spotify_artist_id FROM danish) + OR (t.playlists = 1 AND abs(t.followers - d.followers) <= 0.02 * d.followers) + UNION ALL SELECT isrc, 1 FROM nmf_global_chart + UNION ALL SELECT isrc, 2 FROM nmf_artist_tracks WHERE playlists > 1 + UNION ALL SELECT isrc, 3 FROM deezer_tracks WHERE coalesce(isrc, '') != '') WHERE isrc NOT IN (SELECT isrc FROM nmf_lookups) GROUP BY isrc ORDER BY min(o)")? .query_map([], |r| r.get(0))?.collect::>()?; + let meta = Regex::new(r#""#)?; let day = Regex::new(r#"]*>\s*(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday), ([A-Za-z]+ \d{2}, \d{4})\s*"#)?; let row = Regex::new(r#"(?s)]*>([^<]+)

\s*]*>([\d,]+) followers\s*]*>Position (\d+)"#)?; let mut done = 0; @@ -100,7 +123,13 @@ pub fn nmf_history(db: &Db, http: &Http, limit: usize, log: &mut Vec) -> Err(e) => { log.push(format!("superfridaychart {isrc}: {e}")); continue; } }; let mut found = 0; + let (mut title, mut artists) = (None, None); if let Some(html) = html { + if let Some(m) = meta.captures(&html) { + let un = |x: &str| x.replace("&", "&").replace("'", "'").replace(""", "\""); + title = Some(un(&m[1])); + artists = Some(un(&m[2])); + } // Each Friday is an

date followed by its table of playlists. let heads: Vec<(usize, NaiveDate)> = day.captures_iter(&html) .filter_map(|c| Some((c.get(0)?.start(), NaiveDate::parse_from_str(&c[1], "%B %d, %Y").ok()?))) @@ -114,7 +143,8 @@ pub fn nmf_history(db: &Db, http: &Http, limit: usize, log: &mut Vec) -> } } } - db.conn.execute("INSERT OR REPLACE INTO nmf_lookups VALUES (?1, ?2, ?3)", params![isrc, found, Utc::now().to_rfc3339()])?; + db.conn.execute("INSERT OR REPLACE INTO nmf_lookups (isrc, found, fetched_at, title, artists) VALUES (?1, ?2, ?3, ?4, ?5)", + params![isrc, found, Utc::now().to_rfc3339(), title, artists])?; done += 1; } Ok(done) @@ -171,3 +201,29 @@ pub fn nmf_artists(db: &Db, http: &Http, limit: usize, log: &mut Vec) -> } Ok(done) } + +/// SuperFridayChart's weekly global chart (top 50 tracks across all NMF playlists), every Friday +/// listed on its front page: candidate ISRCs for the NMF Denmark lists. +pub fn nmf_global_charts(db: &Db, http: &Http, log: &mut Vec) -> Result { + db.conn.execute_batch("CREATE TABLE IF NOT EXISTS nmf_global_chart (friday TEXT NOT NULL, rank INTEGER NOT NULL, isrc TEXT NOT NULL, + PRIMARY KEY (friday, rank));")?; + let front = http.get("https://www.superfridaychart.com/")?.unwrap_or_default(); + let mut fridays: Vec = Regex::new(r"/chart/(\d{4}-\d{2}-\d{2})")?.captures_iter(&front).map(|c| c[1].to_string()).collect(); + fridays.sort(); fridays.dedup(); + let track = Regex::new(r"/track/([A-Z0-9]{12})")?; + let mut n = 0; + for f in fridays { + let have: i64 = db.conn.query_row("SELECT count(*) FROM nmf_global_chart WHERE friday = ?1", [&f], |r| r.get(0))?; + if have > 0 { continue; } + let html = match http.get(&format!("https://www.superfridaychart.com/chart/{f}")) { + Ok(Some(h)) => h, Ok(None) => continue, Err(e) => { log.push(format!("superfridaychart chart {f}: {e}")); continue; } + }; + let mut seen = Vec::new(); + for c in track.captures_iter(&html) { if !seen.contains(&c[1].to_string()) { seen.push(c[1].to_string()); } } + for (i, isrc) in seen.iter().enumerate() { + db.conn.execute("INSERT OR REPLACE INTO nmf_global_chart VALUES (?1, ?2, ?3)", params![f, i as i64 + 1, isrc])?; + } + n += 1; + } + Ok(n) +}