From bc222d3cd02acc8f8e3c409049f4b0846a50cc94 Mon Sep 17 00:00:00 2001 From: moritsbw-cpu Date: Sun, 15 Nov 2026 13:00:34 +0000 Subject: [PATCH] Channel affinity (share of the artist's DR plays per station before t, never played there) and genre fit (DR genre x station) in the per-station rows; Best week: any date range up to 12 months Co-Authored-By: Claude Opus 5.5 --- docs/BACKTEST-EVENTS.md | 50 ++++++++++++++--------------- src/model/adds.rs | 29 ++++++++++++++--- src/model/panel.rs | 71 ++++++++++++++++++++++++++++++++++++++++- src/web.rs | 8 ++++- src/web/index.html | 17 ++++++++-- 5 files changed, 141 insertions(+), 34 deletions(-) diff --git a/docs/BACKTEST-EVENTS.md b/docs/BACKTEST-EVENTS.md index 215512a..c20689c 100644 --- a/docs/BACKTEST-EVENTS.md +++ b/docs/BACKTEST-EVENTS.md @@ -6,29 +6,29 @@ Events: an artist in the panel releases something (Deezer) in week t0+1, with no |---|---|---|---|---| | rate by in-rotation-now | 0.1791 | 0.042 | 0.0572 | 0.011 | | rate by last-12-weeks level | 0.1604 | 0.116 | 0.0434 | 0.123 | -| event model | 0.1584 | 0.179 | 0.0383 | 0.286 | +| event model | 0.1555 | 0.186 | 0.0373 | 0.273 | -Skill of the event model, any channel: 0.116 (0.062 to 0.184) vs in-rotation-now, 0.013 (-0.020 to 0.060) vs last-12-weeks level. +Skill of the event model, any channel: 0.132 (0.090 to 0.200) vs in-rotation-now, 0.031 (0.010 to 0.079) vs last-12-weeks level. | Any-channel feature (all events) | Coefficient | |---|---| -| intercept | -5.514 | +| intercept | -5.512 | | log DR plays t | +0.487 | | log DR plays 4w | +0.863 | -| log DR plays 12w | -0.819 | -| log max DR plays 12w | +0.447 | +| log DR plays 12w | -0.818 | +| log max DR plays 12w | +0.446 | | in rotation anywhere now | -1.356 | | station groups in rotation now (/5) | -0.502 | -| current run anywhere (/12) | -1.525 | -| weeks since in rotation anywhere (/12) | +0.129 | -| Track Top-40 score | -0.489 | +| current run anywhere (/12) | -1.526 | +| weeks since in rotation anywhere (/12) | +0.130 | +| Track Top-40 score | -0.490 | | Airplay Top-20 score | -0.429 | -| chart entry recency 6w | +0.330 | -| chart entry recency 26w | -0.124 | -| log Track Top-40 hits | -0.307 | +| chart entry recency 6w | +0.331 | +| chart entry recency 26w | -0.122 | +| log Track Top-40 hits | -0.308 | | log Airplay Top-20 hits | -0.100 | -| Danish | +0.758 | -| group | -0.007 | +| Danish | +0.759 | +| group | -0.008 | | female | +0.124 | | career years (/50) | +0.528 | | debut unknown | +0.596 | @@ -36,21 +36,21 @@ Skill of the event model, any channel: 0.116 (0.062 to 0.184) vs in-rotation-now | discography known | -0.000 | | release recency 4w | -0.124 | | release recency 12w | -0.866 | -| log releases last 52w | -0.175 | +| log releases last 52w | -0.177 | | releases in the window | -0.000 | | releases in window x not in rotation now | +1.356 | | DR: weeks since last add 13-25 | +0.144 | | DR: last add 26+ weeks ago | +0.659 | | DR: no add in the panel yet | -0.622 | -| Airplay Top-20: last new song 13-25 weeks ago | +0.399 | -| Airplay Top-20: 0.5-1.25x own usual gap | -0.497 | -| Airplay Top-20: last new song 2+ years ago | -0.069 | -| log archive plays per 100 sampled hours (2012-21) | +0.316 | +| Airplay Top-20: last new song 13-25 weeks ago | +0.398 | +| Airplay Top-20: 0.5-1.25x own usual gap | -0.496 | +| Airplay Top-20: last new song 2+ years ago | -0.070 | +| log archive plays per 100 sampled hours (2012-21) | +0.317 | | log DR lists with the artist before t (2023-) | +0.431 | -| DR list recency 26w | +0.049 | -| DR lists known before t at all | -0.016 | -| NMF Denmark in the window | +0.774 | -| NMF Denmark top 10 in the window | +0.784 | -| log NMF Denmark placements last 52w | -0.067 | -| share of releases on NMF Denmark (2 years) | -0.590 | -| season index of the window (log, IFPI 2002-24) | +0.452 | +| DR list recency 26w | +0.046 | +| DR lists known before t at all | -0.014 | +| NMF Denmark in the window | +0.769 | +| NMF Denmark top 10 in the window | +0.783 | +| log NMF Denmark placements last 52w | -0.056 | +| share of releases on NMF Denmark (2 years) | -0.600 | +| season index of the window (log, IFPI 2002-24) | +0.451 | diff --git a/src/model/adds.rs b/src/model/adds.rs index ae47757..cfa8b7b 100644 --- a/src/model/adds.rs +++ b/src/model/adds.rs @@ -153,6 +153,26 @@ pub fn season_feat(p: &Panel, t: usize, h: usize, win: usize) -> [f64; SEASON] { [v] } +/// Channel affinity and genre fit (per-channel rows only): the share of the artist's DR plays that +/// were on this station group (all-time top tracks, minus panel plays after week t), the log of +/// those plays, "never played here", and how much DR plays the artist's genre on this group. +pub const AFF: usize = 4; +pub const AFF_NAMES: &[&str] = &["share of the artist's DR plays on this station (before t)", "log DR plays on this station (before t)", + "never played on this station (before t)", "genre fit for this station"]; +pub fn aff_feats(p: &Panel, a: usize, c: usize, t: usize) -> [f64; AFF] { + if off("aff") { return [0.0; AFF]; } + // Plays after week t (the future, as seen from t) taken out of the all-time counts. + let after = |g: usize| -> f64 { + let cs: Vec = (0..C).filter(|&k| group(k) == g).collect(); + let n = cs.len() as f64; + (t + 1..p.w()).map(|w| cs.iter().map(|&k| p.y(a, k, w)).sum::() / n).sum() + }; + let before: [f64; GROUPS] = std::array::from_fn(|g| (p.alltime[a][g] - after(g)).max(0.0)); + let tot: f64 = before.iter().sum(); + let g = group(c); + [if tot > 0.0 { before[g] / tot } else { 0.0 }, before[g].ln_1p(), (before[g] == 0.0) as i32 as f64, p.genre_fit[a][g]] +} + /// Release covariates (Deezer discography), appended to both row kinds. pub const REL: usize = 6; pub const REL_NAMES: &[&str] = &["discography known", "release recency 4w", "release recency 12w", "log releases last 52w", @@ -190,7 +210,7 @@ pub const EXTRA: &[&str] = &["in rotation on channel now", "in rotation anywhere "run on channel = 1 week", "run = 2 weeks", "run = 3-5 weeks", "run >= 6 weeks", "near threshold (>= half of tau, not in rotation)", "plays / tau (capped 3, /3)", "weeks since in rotation on channel (/12)"]; -pub const PC: usize = P + 10 + REL + WAVE + LONG + NMF + SEASON; +pub const PC: usize = P + 10 + REL + WAVE + LONG + NMF + SEASON + AFF; /// Artist-level ("any channel") rows. pub const ANY_NAMES: &[&str] = &[ @@ -267,8 +287,9 @@ pub fn row_channel(p: &Panel, r: &Rot, a: usize, c: usize, t: usize, h: usize, s out[P + 10..P + 10 + REL].copy_from_slice(&rel_feats(p, a, t, h, sc, on)); out[P + 10 + REL..P + 10 + REL + WAVE].copy_from_slice(&wave_feats(p, r, a, t, sc.shift)); out[P + 10 + REL + WAVE..P + 10 + REL + WAVE + LONG].copy_from_slice(&long_feats(p, a, t, sc.shift)); - out[P + 10 + REL + WAVE + LONG..PC - SEASON].copy_from_slice(&nmf_feats(p, a, t, h, sc)); - out[PC - SEASON..PC].copy_from_slice(&season_feat(p, t, h, sc.win)); + out[P + 10 + REL + WAVE + LONG..PC - SEASON - AFF].copy_from_slice(&nmf_feats(p, a, t, h, sc)); + out[PC - SEASON - AFF..PC - AFF].copy_from_slice(&season_feat(p, t, h, sc.win)); + out[PC - AFF..PC].copy_from_slice(&aff_feats(p, a, c, t)); } pub fn row_any(p: &Panel, r: &Rot, a: usize, t: usize, h: usize, sc: Scenario, out: &mut [f64]) { @@ -551,7 +572,7 @@ pub fn backtest(p: &Panel, tau: &[u32; C], horizons: &[usize], start: usize, md: for (n, b) in ANY_NAMES.iter().zip(&m.any) { writeln!(md, "| {n} | {b:+.3} |")?; } for (n, b) in REL_NAMES.iter().chain(WAVE_NAMES).chain(LONG_NAMES).chain(NMF_NAMES).chain(SEASON_NAMES).zip(&m.any[21..]) { writeln!(md, "| {n} | {b:+.3} |")?; } writeln!(md, "\n| Per-channel feature | Coefficient |\n|---|---|")?; - for (n, b) in features::NAMES.iter().chain(EXTRA).chain(REL_NAMES).chain(WAVE_NAMES).chain(LONG_NAMES).chain(NMF_NAMES).chain(SEASON_NAMES).zip(&m.channel) { writeln!(md, "| {n} | {b:+.3} |")?; } + for (n, b) in features::NAMES.iter().chain(EXTRA).chain(REL_NAMES).chain(WAVE_NAMES).chain(LONG_NAMES).chain(NMF_NAMES).chain(SEASON_NAMES).chain(AFF_NAMES).zip(&m.channel) { writeln!(md, "| {n} | {b:+.3} |")?; } writeln!(md)?; Ok(rows) } diff --git a/src/model/panel.rs b/src/model/panel.rs index f627c52..a6e3a2e 100644 --- a/src/model/panel.rs +++ b/src/model/panel.rs @@ -81,6 +81,12 @@ pub struct Panel { pub listed_on: Vec>, /// New Music Friday Denmark placements per artist: (Friday, position if known), sorted. pub nmf_dk: Vec)>>, + /// All-time DR plays per station group (P3, P4, P5, P6, P8) from the artist page's most played + /// tracks (P4: the national count, or the regional average when only regions are given). + pub alltime: Vec<[f64; GROUPS]>, + /// How much DR plays the artist's genre on each station group, relative to all music: ln of + /// (genre's share of plays on the group / the group's share of all plays); 0 = genre unknown. + pub genre_fit: Vec<[f64; GROUPS]>, /// Season index per ISO week (1..=53): new songs reaching the Airplay Top-20 that week of the /// year, relative to an average week, 2002–2024 (before the DR panel), smoothed over 5 weeks. pub season: [f64; 54], @@ -151,7 +157,8 @@ impl Panel { let (archive_rate, listed_on) = load_long_history(db, &id_of, &names, artists.len())?; let nmf_dk = load_nmf_dk(db, &id_of, &artists)?; let season = load_season(db)?; - Ok(Panel { weeks, artists, y, chart, list_tau, releases, archive_rate, listed_on, nmf_dk, season }) + let (alltime, genre_fit) = load_affinity(db, &id_of, artists.len())?; + Ok(Panel { weeks, artists, y, chart, list_tau, releases, archive_rate, listed_on, nmf_dk, season, alltime, genre_fit }) } } @@ -336,3 +343,65 @@ fn load_season(db: &Db) -> Result<[f64; 54]> { } Ok(out) } + +/// Broad genre from DR's description ("Sanger (Afrobeats/Pop/R&B)" -> the first genre that maps). +pub fn genre_bucket(description: &str) -> Option { + let inner = description.split('(').nth(1)?.split(')').next()?; + for tok in inner.split('/') { + let t = tok.trim().to_lowercase(); + let b = if ["hip hop", "hiphop", "rap", "grime", "trap", "drill"].iter().any(|k| t.contains(k)) { 1 } + else if ["jazz", "fusion", "bebop", "swing"].iter().any(|k| t.contains(k)) { 4 } + else if ["klassisk", "filmmusik", "musical", "opera", "kor"].iter().any(|k| t.contains(k)) { 7 } + else if ["r&b", "soul", "funk", "gospel", "afrobeat", "dancehall", "reggae"].iter().any(|k| t.contains(k)) { 5 } + else if ["house", "techno", "elektronisk", "electronica", "electro", "dubstep", "dance", "disco"].iter().any(|k| t.contains(k)) { 3 } + else if ["rock", "punk", "metal", "indie", "new wave", "blues", "grunge"].iter().any(|k| t.contains(k)) { 2 } + else if ["folk", "country", "vise", "singer"].iter().any(|k| t.contains(k)) { 6 } + else if ["latin", "world", "verdens"].iter().any(|k| t.contains(k)) { 8 } + else if ["pop", "dansktop", "schlager", "underholdning"].iter().any(|k| t.contains(k)) { 0 } + else { continue }; + return Some(b); + } + None +} +const GENRES: usize = 9; + +fn load_affinity(db: &Db, id_of: &HashMap, n: usize) -> Result<(Vec<[f64; GROUPS]>, Vec<[f64; GROUPS]>)> { + let mut alltime = vec![[0.0f64; GROUPS]; n]; + let mut genre: Vec> = vec![None; n]; + for r in db.conn.prepare("SELECT artist_id, raw FROM dr_artists")?.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? { + let (id, raw) = r?; + let Some(&a) = id_of.get(&id) else { continue }; + let v: Value = serde_json::from_str(&raw).unwrap_or(Value::Null); + genre[a] = v["artist"]["description"].as_str().and_then(genre_bucket); + let (mut p4_nat, mut p4_reg, mut n_reg) = (0.0, 0.0, 0.0); + for t in v["mostPlayed"].as_array().into_iter().flatten() { + for s in t["statisticsByChannels"].as_array().into_iter().flatten() { + let cnt = s["count"].as_f64().unwrap_or(0.0); + match s["channel"]["slug"].as_str().unwrap_or("") { + "p3" => alltime[a][0] += cnt, "p5" => alltime[a][2] += cnt, "p6beat" => alltime[a][3] += cnt, "p8jazz" => alltime[a][4] += cnt, + "p4" => p4_nat += cnt, + s if s.starts_with("p4") => { p4_reg += cnt; n_reg += 1.0; } + _ => {} + } + } + } + alltime[a][1] = p4_nat.max(if n_reg > 0.0 { p4_reg / 10.0 } else { 0.0 }); + } + // Genre x station group: share of the genre's plays on each group vs the group's share overall. + let mut by = [[0.0f64; GROUPS]; GENRES]; + let mut all = [0.0f64; GROUPS]; + for a in 0..n { + for g in 0..GROUPS { all[g] += alltime[a][g]; if let Some(b) = genre[a] { by[b][g] += alltime[a][g]; } } + } + let tot: f64 = all.iter().sum(); + let mut fit = vec![[0.0f64; GROUPS]; n]; + for a in 0..n { + let Some(b) = genre[a] else { continue }; + let tb: f64 = by[b].iter().sum(); + if tb <= 0.0 || tot <= 0.0 { continue; } + for g in 0..GROUPS { + fit[a][g] = (((by[b][g] + 1.0) / (tb + 5.0)) / ((all[g] + 1.0) / (tot + 5.0))).ln().clamp(-4.0, 2.0); + } + } + Ok((alltime, fit)) +} diff --git a/src/web.rs b/src/web.rs index 1e97408..aee5162 100644 --- a/src/web.rs +++ b/src/web.rs @@ -326,8 +326,14 @@ fn bestweek(st: &State, q: &HashMap) -> Result { let nmf = match q.get("nmf").map(String::as_str) { Some("yes") => Some(true), Some("no") => Some(false), _ => None }; let p = &m.panel; let t0 = p.w() - 1; + // Release weeks between `from` and `to` (default: the next 26), at most 53 weeks, from next week on. + let first = p.weeks[t0] + chrono::Duration::weeks(1); + let date = |k: &str| q.get(k).and_then(|s| chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d").ok()); + let wk = |d: chrono::NaiveDate| ((d - first).num_days().div_euclid(7)).max(0) as usize; + let h0 = date("from").map(wk).unwrap_or(0); + let h1 = date("to").map(wk).unwrap_or(h0 + CALENDAR - 1).max(h0).min(h0 + 52); let groups = [("P3", vec![0usize]), ("P4", (0..C).filter(|&c| crate::model::panel::is_p4(c)).collect()), ("P5", vec![11]), ("P6", vec![12]), ("P8", vec![13])]; - let weeks: Vec = (0..CALENDAR).map(|h| { + let weeks: Vec = (h0..=h1).map(|h| { let start = p.weeks[t0] + chrono::Duration::weeks(h as i64 + 1); let xmas = crate::model::features::christmas(p, t0 + h + 1); match new_song(m, a, h, win, nmf) { diff --git a/src/web/index.html b/src/web/index.html index c4d60f8..1f5ba15 100644 --- a/src/web/index.html +++ b/src/web/index.html @@ -167,6 +167,8 @@ svg text { fill: var(--muted); font-size: 11px; }
Station
+
From
+
To
Added within
@@ -471,7 +473,16 @@ async function loadBest() { if (!bestId) return; const w = $('bw-w').value, nmf = $('bw-nmf').value, ch = $('bw-ch').value; $('bw-out').innerHTML = '

Loading…

'; - const r = await fetch(`${BASE}/api/bestweek?id=${encodeURIComponent(bestId)}&w=${w}&nmf=${nmf}`); + // Dates: from next week, at most 12 months. + const next = windowStart(0); + if (!$('bw-from').value) $('bw-from').value = next; + if ($('bw-from').value < next) $('bw-from').value = next; + const maxTo = new Date($('bw-from').value + 'T00:00:00Z'); maxTo.setUTCFullYear(maxTo.getUTCFullYear() + 1); + if (!$('bw-to').value) { const d = new Date($('bw-from').value + 'T00:00:00Z'); d.setUTCDate(d.getUTCDate() + 7 * 25); $('bw-to').value = d.toISOString().slice(0, 10); } + if ($('bw-to').value > maxTo.toISOString().slice(0, 10)) $('bw-to').value = maxTo.toISOString().slice(0, 10); + if ($('bw-to').value < $('bw-from').value) $('bw-to').value = $('bw-from').value; + $('bw-from').min = next; $('bw-to').min = $('bw-from').value; $('bw-to').max = maxTo.toISOString().slice(0, 10); + const r = await fetch(`${BASE}/api/bestweek?id=${encodeURIComponent(bestId)}&w=${w}&nmf=${nmf}&from=${$('bw-from').value}&to=${$('bw-to').value}`); if (r.status === 503) { $('bw-out').innerHTML = '

The model is still loading on the server; try again in a minute.

'; return; } const b = await r.json(); if (b.error) { $('bw-out').innerHTML = `

${esc(b.error)}

`; return; } @@ -489,10 +500,10 @@ async function loadBest() {
${b.weeks.map(x => { const v = val(x); return `
Week of ${fmtDate(x.week)}
${x.too_soon ? 'too soon' : pct(v)}
${x.christmas ? '🎄' : ''}
`; }).join('')}

Each tile: if ${esc(b.name)} releases a new song that week, the chance DR puts it into rotation within ${w} weeks. ${b.last_release ? `Their last release: ${fmtDate(b.last_release)}; weeks less than 6 weeks after it are "too soon" (the model only learned from releases with a clear gap).` : ''} The new song's NMF Denmark placement: ${nmf === 'auto' ? `as often as their releases were placed (${pct(b.p_nmf)})` : nmf === 'yes' ? 'placed' : 'not placed'}. What moves the numbers from week to week: time since their last release and last DR add, Christmas, and their NMF history; their current airplay is taken as it is now.

` - : `

${esc(b.name)}

No week in the next 26 has an estimate.

`; + : `

${esc(b.name)}

No week in this period has an estimate.

`; } $('bw-q').addEventListener('change', () => { const id = artistByName.get($('bw-q').value); if (id) { bestId = id; loadBest(); } }); -['bw-w', 'bw-nmf', 'bw-ch'].forEach(id => $(id).addEventListener('change', loadBest)); +['bw-w', 'bw-nmf', 'bw-ch', 'bw-from', 'bw-to'].forEach(id => $(id).addEventListener('change', loadBest)); $('q').addEventListener('input', () => { shown = 100; render(); }); $('more').addEventListener('click', () => { shown += 100; render(); }); $('rows').addEventListener('click', e => { const tr = e.target.closest('tr[data-id]'); if (tr) showArtist(tr.dataset.id); });