Synthetic lists: top-N rule (N = typical DR list size), 4-week blocks (weekly samples are noise); long-history features (archive presence 2012-21, DR lists before t); NMF analysis on full artist histories
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+22
-6
@@ -52,6 +52,20 @@ pub fn wave_feats(p: &Panel, r: &Rot, a: usize, t: usize) -> [f64; WAVE] {
|
||||
f(known && (13.0..=25.0).contains(&air)), f(ratio_ok), f(known && air >= 104.0)]
|
||||
}
|
||||
|
||||
/// Long-history covariates: DR's 2012–2021 archive and DR's real rotation lists before week t.
|
||||
pub const LONG: usize = 4;
|
||||
pub const LONG_NAMES: &[&str] = &["log archive plays per 100 sampled hours (2012-21)", "log DR lists with the artist before t (2023-)",
|
||||
"DR list recency 26w", "DR lists known before t at all"];
|
||||
|
||||
pub fn long_feats(p: &Panel, a: usize, t: usize) -> [f64; LONG] {
|
||||
if off("long") { return [0.0; LONG]; }
|
||||
let end_t = p.weeks[t] + chrono::Duration::days(6);
|
||||
// A list published on a Friday is known from then.
|
||||
let before: Vec<&chrono::NaiveDate> = p.listed_on[a].iter().filter(|d| **d <= end_t).collect();
|
||||
let since = before.last().map(|d| (end_t - **d).num_days() as f64 / 7.0);
|
||||
[p.archive_rate[a].ln_1p(), (before.len() as f64).ln_1p(), since.map(|s| (-s / 26.0).exp()).unwrap_or(0.0), (!before.is_empty()) as i32 as f64]
|
||||
}
|
||||
|
||||
/// Release covariates (Deezer discography), appended to both row kinds.
|
||||
pub const REL: usize = 6;
|
||||
pub const REL_NAMES: &[&str] = &["discography known", "release recency 4w", "release recency 12w", "log releases last 52w",
|
||||
@@ -88,7 +102,7 @@ pub const EXTRA: &[&str] = &["in rotation on channel now", "in rotation anywhere
|
||||
"run on channel = 1 week", "run = 2 weeks", "run = 3-5 weeks", "run >= 6 weeks",
|
||||
"near threshold (>= half of tau, not in rotation)", "plays / tau (capped 3, /3)",
|
||||
"weeks since in rotation on channel (/12)"];
|
||||
pub const PC: usize = P + 10 + REL + WAVE;
|
||||
pub const PC: usize = P + 10 + REL + WAVE + LONG;
|
||||
|
||||
/// Artist-level ("any channel") rows.
|
||||
pub const ANY_NAMES: &[&str] = &[
|
||||
@@ -98,7 +112,7 @@ pub const ANY_NAMES: &[&str] = &[
|
||||
"log Track Top-40 hits", "log Airplay Top-20 hits", "Danish", "group", "female", "career years (/50)", "debut unknown",
|
||||
"Christmas in window",
|
||||
];
|
||||
pub const PA: usize = 21 + REL + WAVE;
|
||||
pub const PA: usize = 21 + REL + WAVE + LONG;
|
||||
/// Positions of the chart covariates (for the what-if): (t40 score, air score, recency 6w, recency 26w).
|
||||
pub const CHART_IDX_CHANNEL: [usize; 4] = [10, 11, 12, 13];
|
||||
pub const CHART_IDX_ANY: [usize; 4] = [9, 10, 11, 12];
|
||||
@@ -161,7 +175,8 @@ pub fn row_channel(p: &Panel, r: &Rot, a: usize, c: usize, t: usize, h: usize, r
|
||||
f(weeks == 1), f(weeks == 2), f((3..=5).contains(&weeks)), f(weeks >= 6),
|
||||
f(!on && ratio >= 0.5), ratio.min(3.0) / 3.0, since]);
|
||||
out[P + 10..P + 10 + REL].copy_from_slice(&rel_feats(p, a, t, h, rel, on));
|
||||
out[P + 10 + REL..PC].copy_from_slice(&wave_feats(p, r, a, t));
|
||||
out[P + 10 + REL..P + 10 + REL + WAVE].copy_from_slice(&wave_feats(p, r, a, t));
|
||||
out[P + 10 + REL + WAVE..PC].copy_from_slice(&long_feats(p, a, t));
|
||||
}
|
||||
|
||||
pub fn row_any(p: &Panel, r: &Rot, a: usize, t: usize, h: usize, rel: Option<bool>, out: &mut [f64]) {
|
||||
@@ -186,7 +201,8 @@ pub fn row_any(p: &Panel, r: &Rot, a: usize, t: usize, h: usize, rel: Option<boo
|
||||
];
|
||||
out[..21].copy_from_slice(&v);
|
||||
out[21..21 + REL].copy_from_slice(&rel_feats(p, a, t, h, rel, r.any(a, t)));
|
||||
out[21 + REL..PA].copy_from_slice(&wave_feats(p, r, a, t));
|
||||
out[21 + REL..21 + REL + WAVE].copy_from_slice(&wave_feats(p, r, a, t));
|
||||
out[21 + REL + WAVE..PA].copy_from_slice(&long_feats(p, a, t));
|
||||
}
|
||||
|
||||
/// Logit models for one h, trained on origins whose whole window is observed by `last_week`.
|
||||
@@ -416,9 +432,9 @@ pub fn backtest(p: &Panel, tau: &[u32; C], horizons: &[usize], start: usize, md:
|
||||
let m = AddModel::fit(p, &r, 0, p.w() - 1);
|
||||
writeln!(md, "## Coefficients, window starting now (h = 0), all closed windows\n\n| Any-channel feature | Coefficient |\n|---|---|")?;
|
||||
for (n, b) in ANY_NAMES.iter().zip(&m.any) { writeln!(md, "| {n} | {b:+.3} |")?; }
|
||||
for (n, b) in REL_NAMES.iter().chain(WAVE_NAMES).zip(&m.any[21..]) { writeln!(md, "| {n} | {b:+.3} |")?; }
|
||||
for (n, b) in REL_NAMES.iter().chain(WAVE_NAMES).chain(LONG_NAMES).zip(&m.any[21..]) { writeln!(md, "| {n} | {b:+.3} |")?; }
|
||||
writeln!(md, "\n| Per-channel feature | Coefficient |\n|---|---|")?;
|
||||
for (n, b) in features::NAMES.iter().chain(EXTRA).chain(REL_NAMES).chain(WAVE_NAMES).zip(&m.channel) { writeln!(md, "| {n} | {b:+.3} |")?; }
|
||||
for (n, b) in features::NAMES.iter().chain(EXTRA).chain(REL_NAMES).chain(WAVE_NAMES).chain(LONG_NAMES).zip(&m.channel) { writeln!(md, "| {n} | {b:+.3} |")?; }
|
||||
writeln!(md)?;
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
+31
-1
@@ -75,6 +75,10 @@ pub struct Panel {
|
||||
pub list_tau: Option<[u32; C]>,
|
||||
/// Release dates per artist (Deezer discography: singles, EPs, albums), sorted; None = not looked up / not found.
|
||||
pub releases: Vec<Option<Vec<NaiveDate>>>,
|
||||
/// Long history: the artist's plays per 100 archived hours on DR (Wayback, 2012–2021), and the
|
||||
/// dates of DR's real rotation lists (2023–) that had the artist on any channel.
|
||||
pub archive_rate: Vec<f64>,
|
||||
pub listed_on: Vec<Vec<NaiveDate>>,
|
||||
}
|
||||
|
||||
impl Panel {
|
||||
@@ -139,7 +143,8 @@ impl Panel {
|
||||
let chart = chart_states(db, &weeks, &names, artists.len())?;
|
||||
let list_tau = super::calibrate::stored(db)?;
|
||||
let releases = load_releases(db, &id_of, artists.len())?;
|
||||
Ok(Panel { weeks, artists, y, chart, list_tau, releases })
|
||||
let (archive_rate, listed_on) = load_long_history(db, &id_of, &names, artists.len())?;
|
||||
Ok(Panel { weeks, artists, y, chart, list_tau, releases, archive_rate, listed_on })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -242,3 +247,28 @@ fn load_releases(db: &Db, id_of: &HashMap<String, usize>, n: usize) -> Result<Ve
|
||||
for v in out.iter_mut().flatten() { v.sort(); v.dedup(); }
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn load_long_history(db: &Db, id_of: &HashMap<String, usize>, names: &HashMap<String, Vec<usize>>, n: usize) -> Result<(Vec<f64>, Vec<Vec<NaiveDate>>)> {
|
||||
let mut rate = vec![0.0; n];
|
||||
let has = |t: &str| -> Result<bool> { Ok(db.conn.query_row("SELECT count(*) FROM sqlite_master WHERE name = ?1", [t], |r| r.get::<_, i64>(0))? > 0) };
|
||||
if has("archive_plays")? {
|
||||
let hours: f64 = db.conn.query_row("SELECT count(DISTINCT station || substr(played_at, 1, 13)) FROM archive_plays", [], |r| r.get(0))?;
|
||||
let mut st = db.conn.prepare("SELECT artist_ids FROM archive_plays WHERE artist_ids != ''")?;
|
||||
let rows = st.query_map([], |r| r.get::<_, String>(0))?;
|
||||
for r in rows {
|
||||
for id in r?.split(',') { if let Some(&a) = id_of.get(id) { rate[a] += 1.0; } }
|
||||
}
|
||||
if hours > 0.0 { for v in rate.iter_mut() { *v *= 100.0 / hours; } }
|
||||
}
|
||||
let mut listed: Vec<Vec<NaiveDate>> = vec![Vec::new(); n];
|
||||
if has("rotation_lists")? {
|
||||
for r in db.conn.prepare("SELECT DISTINCT list_date, artist FROM rotation_lists")?.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? {
|
||||
let (d, artist) = r?;
|
||||
let Ok(d) = NaiveDate::parse_from_str(&d, "%Y-%m-%d") else { continue };
|
||||
let list_artist = artist.replace(" [+] ", ", ");
|
||||
for a in chart_artists(&list_artist, names) { listed[a].push(d); }
|
||||
}
|
||||
}
|
||||
for v in listed.iter_mut() { v.sort(); v.dedup(); }
|
||||
Ok((rate, listed))
|
||||
}
|
||||
|
||||
+41
-8
@@ -150,6 +150,33 @@ pub fn run(db: &Db, p: &Panel) -> Result<String> {
|
||||
let false_pos: usize = hit[3][..3].iter().sum();
|
||||
writeln!(md, "\nIn-sample check (song-weeks): listed songs found {listed_found} of {listed_total}; songs wrongly put on the list {false_pos}. Thresholds are refitted as exact plays and lists for the same weeks accumulate.\n")?;
|
||||
|
||||
// Typical list size per channel (DR's real lists: A/B/C or one list), the era-robust rule:
|
||||
// on the list = among the channel's N most played songs that week.
|
||||
let mut list_size: HashMap<String, usize> = HashMap::new();
|
||||
for lc in ["p3", "p4", "p5", "p6beat", "p8jazz"] {
|
||||
let mut v: Vec<i64> = db.conn.prepare("SELECT count(*) FROM rotation_lists WHERE channel = ?1 AND category IN ('A', 'B', 'C', 'LIST') GROUP BY list_date")?
|
||||
.query_map([lc], |r| r.get(0))?.collect::<Result<_, _>>()?;
|
||||
v.sort();
|
||||
if !v.is_empty() { list_size.insert(lc.to_string(), v[v.len() / 2] as usize); }
|
||||
}
|
||||
// Check the rule where the real list is known: top N by plays vs the list.
|
||||
let (mut tp, mut fp, mut fnn) = (0usize, 0usize, 0usize);
|
||||
for (week, by_station) in &plays {
|
||||
let days = days_seen.get(week).map(|s| s.len()).unwrap_or(0);
|
||||
if days < 5 { continue; }
|
||||
for (station, songs) in by_station {
|
||||
let Some(lc) = list_channel(station) else { continue };
|
||||
let (Some((list, _)), Some(&n)) = (list_for(*week, lc), list_size.get(lc)) else { continue };
|
||||
let mut ranked: Vec<(&String, &f64)> = songs.iter().collect();
|
||||
ranked.sort_by(|a, b| b.1.total_cmp(a.1));
|
||||
let top: std::collections::HashSet<&String> = ranked.iter().take(n).map(|x| x.0).collect();
|
||||
for s in &top { if list.contains_key(*s) { tp += 1 } else { fp += 1 } }
|
||||
fnn += list.keys().filter(|s| !top.contains(s)).count();
|
||||
}
|
||||
}
|
||||
writeln!(md, "Era-robust rule used for the archive: on the list = among the channel's N most played songs that week, N = the median size of DR's lists ({}). Where the real list is known it finds {tp} of {} listed songs, with {fp} songs wrongly on (precision {:.0}%, recall {:.0}%).\n",
|
||||
list_size.iter().map(|(k, v)| format!("{k} {v}")).collect::<Vec<_>>().join(", "), tp + fnn, 100.0 * tp as f64 / (tp + fp).max(1) as f64, 100.0 * tp as f64 / (tp + fnn).max(1) as f64)?;
|
||||
|
||||
// ---- 2. apply to the archive
|
||||
let has_archive: i64 = db.conn.query_row("SELECT count(*) FROM sqlite_master WHERE name = 'archive_plays'", [], |r| r.get(0))?;
|
||||
writeln!(md, "## 2. Archived playlists → synthetic lists\n")?;
|
||||
@@ -163,7 +190,12 @@ pub fn run(db: &Db, p: &Panel) -> Result<String> {
|
||||
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, String>(3)?)))? {
|
||||
let (st, at, a, t) = r?;
|
||||
let Ok(d) = NaiveDate::parse_from_str(&at[..10], "%Y-%m-%d") else { continue };
|
||||
let w = weeks.entry((st, d.iso_week().year(), d.iso_week().week())).or_insert_with(|| Wk { hours: HashMap::new(), songs: HashMap::new() });
|
||||
// 4-week blocks (from Monday 3 Jan 2000): a few sampled days a week are too noisy for a
|
||||
// weekly list (two 36-hour samples of one week's P3 differ by ~9 of the top 32 songs),
|
||||
// pooled over 4 weeks the list holds; songs stay on the list ~7+ weeks anyway.
|
||||
let epoch = NaiveDate::from_ymd_opt(2000, 1, 3).unwrap();
|
||||
let block = epoch + Duration::weeks(((d - epoch).num_days() / 7 / 4) * 4);
|
||||
let w = weeks.entry((st, block.iso_week().year(), block.iso_week().week())).or_insert_with(|| Wk { hours: HashMap::new(), songs: HashMap::new() });
|
||||
w.hours.entry(at[..10].to_string()).or_default().insert(at[11..13].to_string());
|
||||
let e = w.songs.entry(song(&a, &t)).or_insert((0.0, a.clone(), t.clone()));
|
||||
e.0 += 1.0;
|
||||
@@ -173,17 +205,18 @@ pub fn run(db: &Db, p: &Panel) -> Result<String> {
|
||||
let mut summary: BTreeMap<(String, i32), (usize, usize)> = BTreeMap::new(); // (station, year) -> (weeks, songs listed)
|
||||
for ((st, y, w), wk) in &weeks {
|
||||
let Some(lc) = list_channel(st) else { continue };
|
||||
let Some(th) = thresholds.get(lc) else { continue };
|
||||
if !thresholds.contains_key(lc) { continue; }
|
||||
// Only daytime-complete days count fully; scale plays by covered hours / 24.
|
||||
let hours: usize = wk.hours.values().map(|h| h.len()).sum();
|
||||
let days = wk.hours.values().filter(|h| h.len() >= 12).count();
|
||||
if hours < 36 { continue; } // less than a day and a half: too thin to tell a C song from chance
|
||||
if hours < 96 { continue; } // under 4 days' worth in the block: too thin (two 72-hour samples of one week still differ by ~5 of 32)
|
||||
let day_equiv = hours as f64 / 24.0;
|
||||
let Some(&size) = list_size.get(lc) else { continue };
|
||||
let mut ranked: Vec<(&String, &(f64, String, String))> = wk.songs.iter().filter(|(_, v)| v.0 >= 3.0).collect();
|
||||
ranked.sort_by(|a, b| b.1 .0.total_cmp(&a.1 .0));
|
||||
let mut n = 0;
|
||||
for (s, (cnt, a, t)) in &wk.songs {
|
||||
if *cnt < 3.0 { continue; } // a song needs 3+ plays in the sample to count
|
||||
for (s, (cnt, a, t)) in ranked.into_iter().take(size) {
|
||||
let ppd = cnt / day_equiv;
|
||||
if ppd < th[2] { continue; }
|
||||
tx.execute("INSERT OR REPLACE INTO synthetic_rotation VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, 'wayback')",
|
||||
rusqlite::params![lc, y, w, s, a, t, ppd, "listed", days as i64, hours as i64])?;
|
||||
n += 1;
|
||||
@@ -192,9 +225,9 @@ pub fn run(db: &Db, p: &Panel) -> Result<String> {
|
||||
e.0 += 1; e.1 += n;
|
||||
}
|
||||
tx.commit()?;
|
||||
writeln!(md, "| Station | Year | Weeks with a synthetic list | Avg songs listed per week |\n|---|---|---|---|")?;
|
||||
writeln!(md, "| Station | Year | 4-week blocks with a synthetic list | Avg songs listed |\n|---|---|---|---|")?;
|
||||
for ((st, y), (wks, n)) in &summary { writeln!(md, "| {st} | {y} | {wks} | {:.0} |", *n as f64 / *wks as f64)?; }
|
||||
writeln!(md, "\nA week needs at least 36 archived hours and a song at least 3 plays in them; plays are scaled to a full day by the hours covered.\n")?;
|
||||
writeln!(md, "\nLists are per 4-week block (iso_year/iso_week = its first week): a block needs at least 96 archived hours; a song needs 3+ plays and is then ranked against the block's other songs (top N = on the list in that block). Weekly lists from a few sampled days are mostly sampling noise.\n")?;
|
||||
}
|
||||
|
||||
// ---- 3. IFPI Airplay Top-20 position vs DR plays
|
||||
|
||||
Reference in New Issue
Block a user