From e6b080246bf3f7fe24462974d38627518028b064 Mon Sep 17 00:00:00 2001 From: moritsbw-cpu Date: Sun, 15 Nov 2026 11:33:17 +0000 Subject: [PATCH] Synthetic lists: top-N rule (N = typical DR list size), 4-week blocks (weekly samples are noise); long-history features (archive presence 2012-21, DR lists before t); NMF analysis on full artist histories Co-Authored-By: Claude Opus 5.5 --- docs/BACKTEST-ADDS.md | 218 +++++++++++++++++++++++------------------- docs/NMF.md | 33 +++++++ docs/SYNTH.md | 45 +++++++++ src/model/adds.rs | 28 ++++-- src/model/panel.rs | 32 ++++++- src/model/synth.rs | 49 ++++++++-- 6 files changed, 294 insertions(+), 111 deletions(-) create mode 100644 docs/NMF.md create mode 100644 docs/SYNTH.md diff --git a/docs/BACKTEST-ADDS.md b/docs/BACKTEST-ADDS.md index e9bd6cb..b791714 100644 --- a/docs/BACKTEST-ADDS.md +++ b/docs/BACKTEST-ADDS.md @@ -4,139 +4,165 @@ An add = the artist enters rotation (plays ≥ τ) after at least 4 weeks out of ## Window starting h = 0 weeks ahead (17 forecast origins) -Base rates: per channel 0.75% of artist-channels get an add in the window; on any channel 4.2% of artists. +Base rates: per channel 0.36% of artist-channels get an add in the window; on any channel 2.0% of artists. | Model | Log-loss, per channel | PR-AUC, per channel | Log-loss, any channel | PR-AUC, any channel | |---|---|---|---|---| -| rate by in-rotation-now | 0.0440 | 0.012 | 0.1758 | 0.042 | -| rate by last-12-weeks level | 0.0328 | 0.099 | 0.1656 | 0.083 | -| logit | 0.0273 | 0.230 | 0.1563 | 0.156 | +| rate by in-rotation-now | 0.0234 | 0.006 | 0.0994 | 0.023 | +| rate by last-12-weeks level | 0.0162 | 0.097 | 0.0863 | 0.073 | +| logit | 0.0130 | 0.245 | 0.0747 | 0.199 | Skill of the logit (90% block-bootstrap interval over origins): | | vs in-rotation-now | vs last-12-weeks level | |---|---|---| -| per channel | 0.381 (0.361 to 0.396) | 0.168 (0.148 to 0.181) | -| any channel | 0.111 (0.099 to 0.120) | 0.056 (0.040 to 0.068) | +| per channel | 0.444 (0.427 to 0.458) | 0.196 (0.186 to 0.199) | +| any channel | 0.249 (0.230 to 0.265) | 0.134 (0.124 to 0.142) | Calibration, per channel: | Predicted | Rows | Mean predicted | Observed | |---|---|---|---| -| 0.0–0.1 | 760136 | 0.003 | 0.004 | -| 0.1–0.2 | 4598 | 0.148 | 0.180 | -| 0.2–0.3 | 3797 | 0.246 | 0.219 | -| 0.3–0.4 | 1815 | 0.345 | 0.297 | -| 0.4–0.5 | 1080 | 0.444 | 0.359 | -| 0.5–0.6 | 715 | 0.548 | 0.365 | -| 0.6–0.7 | 252 | 0.643 | 0.476 | -| 0.7–0.8 | 149 | 0.742 | 0.470 | -| 0.8–0.9 | 6 | 0.809 | 0.000 | +| 0.0–0.1 | 1669178 | 0.002 | 0.002 | +| 0.1–0.2 | 5763 | 0.143 | 0.193 | +| 0.2–0.3 | 2302 | 0.244 | 0.232 | +| 0.3–0.4 | 1408 | 0.345 | 0.294 | +| 0.4–0.5 | 872 | 0.441 | 0.336 | +| 0.5–0.6 | 583 | 0.547 | 0.453 | +| 0.6–0.7 | 306 | 0.653 | 0.461 | +| 0.7–0.8 | 189 | 0.739 | 0.339 | +| 0.8–0.9 | 152 | 0.839 | 0.658 | +| 0.9–1.0 | 3 | 0.902 | 1.000 | ## Window starting h = 4 weeks ahead (13 forecast origins) -Base rates: per channel 0.77% of artist-channels get an add in the window; on any channel 4.3% of artists. +Base rates: per channel 0.37% of artist-channels get an add in the window; on any channel 2.0% of artists. | Model | Log-loss, per channel | PR-AUC, per channel | Log-loss, any channel | PR-AUC, any channel | |---|---|---|---|---| -| rate by in-rotation-now | 0.0438 | 0.019 | 0.1752 | 0.053 | -| rate by last-12-weeks level | 0.0347 | 0.091 | 0.1689 | 0.078 | -| logit | 0.0326 | 0.139 | 0.1629 | 0.127 | +| rate by in-rotation-now | 0.0232 | 0.014 | 0.0979 | 0.031 | +| rate by last-12-weeks level | 0.0174 | 0.087 | 0.0887 | 0.066 | +| logit | 0.0160 | 0.134 | 0.0805 | 0.142 | Skill of the logit (90% block-bootstrap interval over origins): | | vs in-rotation-now | vs last-12-weeks level | |---|---|---| -| per channel | 0.257 (0.242 to 0.276) | 0.062 (0.043 to 0.077) | -| any channel | 0.070 (0.063 to 0.075) | 0.035 (0.020 to 0.046) | +| per channel | 0.313 (0.305 to 0.323) | 0.081 (0.057 to 0.096) | +| any channel | 0.178 (0.162 to 0.190) | 0.092 (0.084 to 0.098) | Calibration, per channel: | Predicted | Rows | Mean predicted | Observed | |---|---|---|---| -| 0.0–0.1 | 579503 | 0.004 | 0.005 | -| 0.1–0.2 | 6709 | 0.143 | 0.118 | -| 0.2–0.3 | 2781 | 0.243 | 0.193 | -| 0.3–0.4 | 1213 | 0.341 | 0.232 | -| 0.4–0.5 | 375 | 0.442 | 0.291 | -| 0.5–0.6 | 131 | 0.542 | 0.198 | -| 0.6–0.7 | 42 | 0.637 | 0.571 | -| 0.7–0.8 | 18 | 0.721 | 0.667 | +| 0.0–0.1 | 1275342 | 0.002 | 0.002 | +| 0.1–0.2 | 6075 | 0.141 | 0.120 | +| 0.2–0.3 | 2066 | 0.244 | 0.169 | +| 0.3–0.4 | 812 | 0.342 | 0.197 | +| 0.4–0.5 | 509 | 0.447 | 0.287 | +| 0.5–0.6 | 332 | 0.539 | 0.455 | +| 0.6–0.7 | 110 | 0.650 | 0.273 | +| 0.7–0.8 | 36 | 0.737 | 0.278 | +| 0.8–0.9 | 2 | 0.841 | 0.000 | ## Coefficients, window starting now (h = 0), all closed windows | Any-channel feature | Coefficient | |---|---| -| intercept | -1.503 | -| log DR plays t | +0.669 | -| log DR plays 4w | +0.119 | -| log DR plays 12w | -0.428 | -| log max DR plays 12w | +0.140 | -| in rotation anywhere now | -2.146 | -| channels in rotation now (/14) | -2.288 | -| current run anywhere (/12) | -1.783 | -| weeks since in rotation anywhere (/12) | -2.397 | -| Track Top-40 score | +0.248 | -| Airplay Top-20 score | +0.210 | -| chart entry recency 6w | +0.482 | -| chart entry recency 26w | -0.328 | -| log Track Top-40 hits | +0.002 | -| log Airplay Top-20 hits | -0.280 | -| Danish | +0.500 | -| group | -0.468 | -| female | -0.142 | -| career years (/50) | -0.250 | -| debut unknown | +0.129 | +| intercept | -5.107 | +| log DR plays t | +0.979 | +| log DR plays 4w | +0.245 | +| log DR plays 12w | -0.284 | +| log max DR plays 12w | +0.420 | +| in rotation anywhere now | -1.359 | +| station groups in rotation now (/5) | -3.411 | +| current run anywhere (/12) | -1.907 | +| weeks since in rotation anywhere (/12) | -0.348 | +| Track Top-40 score | +0.147 | +| Airplay Top-20 score | -0.633 | +| chart entry recency 6w | +1.325 | +| chart entry recency 26w | -1.443 | +| log Track Top-40 hits | -0.083 | +| log Airplay Top-20 hits | -0.057 | +| Danish | +0.505 | +| group | -0.363 | +| female | -0.169 | +| career years (/50) | +0.108 | +| debut unknown | +0.514 | | Christmas in window | +0.000 | +| discography known | +0.877 | +| release recency 4w | +0.345 | +| release recency 12w | +0.630 | +| log releases last 52w | -0.191 | +| releases in the window | -0.425 | +| releases in window x not in rotation now | +1.334 | +| DR: weeks since last add 13-25 | -0.015 | +| DR: last add 26+ weeks ago | -0.511 | +| DR: no add in the panel yet | -0.998 | +| Airplay Top-20: last new song 13-25 weeks ago | +0.255 | +| Airplay Top-20: 0.5-1.25x own usual gap | -0.106 | +| Airplay Top-20: last new song 2+ years ago | -0.548 | | Per-channel feature | Coefficient | |---|---| -| intercept | -6.792 | -| log plays t | +0.665 | -| log plays t-1 | +0.129 | -| log mean plays 4w | +0.169 | -| log mean plays 12w | -0.662 | -| log max plays 12w | +1.391 | -| log other channels t | +0.257 | -| log other channels 4w | -0.214 | -| log P3 t | +0.174 | -| weeks since played (/12) | -0.733 | -| Track Top-40 score | -0.563 | -| Airplay Top-20 score | -0.416 | -| chart entry recency 6w | +0.160 | -| chart entry recency 26w | +0.288 | -| log Track Top-40 hits | +0.260 | -| log Airplay Top-20 hits | -0.324 | -| Danish | +0.295 | -| group | -0.840 | -| female | +0.007 | -| career years (/50) | -1.415 | -| debut unknown | -0.130 | +| intercept | -8.129 | +| log plays t | +0.772 | +| log plays t-1 | +0.119 | +| log mean plays 4w | +0.121 | +| log mean plays 12w | -0.700 | +| log max plays 12w | +1.336 | +| log other channels t | +0.210 | +| log other channels 4w | -0.129 | +| log P3 t | +0.121 | +| weeks since played (/12) | -1.250 | +| Track Top-40 score | -0.315 | +| Airplay Top-20 score | -0.715 | +| chart entry recency 6w | +0.400 | +| chart entry recency 26w | -0.160 | +| log Track Top-40 hits | +0.273 | +| log Airplay Top-20 hits | -0.269 | +| Danish | +0.222 | +| group | -0.661 | +| female | +0.019 | +| career years (/50) | -1.152 | +| debut unknown | -0.069 | | Christmas target week | +0.000 | -| Danish x P3 | -0.176 | -| Danish x P4 | +0.154 | -| Danish x P6 | -0.213 | -| ch p4kbh | +0.236 | -| ch p4sjaelland | +0.256 | -| ch p4fyn | +0.235 | -| ch p4syd | +0.297 | -| ch p4trekanten | +0.272 | -| ch p4vest | +0.361 | -| ch p4aarhus | +0.297 | -| ch p4nord | +0.186 | -| ch p4bornholm | +0.278 | -| ch p4esbjerg | +0.290 | -| ch p5 | +1.468 | -| ch p6beat | +0.871 | -| ch p8jazz | +1.960 | -| in rotation on channel now | -1.352 | -| in rotation anywhere now | +0.104 | -| channels in rotation now (/14) | -0.674 | -| run on channel = 1 week | +0.944 | -| run = 2 weeks | +0.352 | -| run = 3-5 weeks | -1.570 | -| run >= 6 weeks | -1.078 | -| near threshold (>= half of tau, not in rotation) | +1.389 | -| plays / tau (capped 3, /3) | -0.664 | -| weeks since in rotation on channel (/12) | +0.421 | +| Danish x P3 | +0.010 | +| Danish x P4 | +0.343 | +| Danish x P6 | -0.044 | +| ch p4kbh | +0.293 | +| ch p4sjaelland | +0.344 | +| ch p4fyn | +0.357 | +| ch p4syd | +0.428 | +| ch p4trekanten | +0.408 | +| ch p4vest | +0.465 | +| ch p4aarhus | +0.378 | +| ch p4nord | +0.230 | +| ch p4bornholm | +0.373 | +| ch p4esbjerg | +0.398 | +| ch p5 | +1.665 | +| ch p6beat | +1.017 | +| ch p8jazz | +2.213 | +| in rotation on channel now | -1.017 | +| in rotation anywhere now | +0.188 | +| station groups in rotation now (/5) | -1.490 | +| run on channel = 1 week | +0.936 | +| run = 2 weeks | +0.398 | +| run = 3-5 weeks | -1.398 | +| run >= 6 weeks | -0.954 | +| near threshold (>= half of tau, not in rotation) | +1.429 | +| plays / tau (capped 3, /3) | -0.704 | +| weeks since in rotation on channel (/12) | +0.497 | +| discography known | +0.741 | +| release recency 4w | +0.924 | +| release recency 12w | +0.021 | +| log releases last 52w | -0.185 | +| releases in the window | +0.226 | +| releases in window x not in rotation now | +0.822 | +| DR: weeks since last add 13-25 | +0.070 | +| DR: last add 26+ weeks ago | +0.247 | +| DR: no add in the panel yet | -0.105 | +| Airplay Top-20: last new song 13-25 weeks ago | +0.171 | +| Airplay Top-20: 0.5-1.25x own usual gap | +0.032 | +| Airplay Top-20: last new song 2+ years ago | -0.254 | diff --git a/docs/NMF.md b/docs/NMF.md new file mode 100644 index 0000000..df47761 --- /dev/null +++ b/docs/NMF.md @@ -0,0 +1,33 @@ +# New Music Friday Denmark and DR adds + +Releases by artists in the DR panel (Deezer discographies), in DR weeks 2026-02-09 to 2026-09-07 with 8 weeks of follow-up, whose New Music Friday history was looked up. An add = entering rotation on a DR channel after at least 4 weeks out of rotation everywhere (rotation thresholds from DR's lists). Matched to NMF placements within a week of the release date. NMF Denmark recognised from the track page or, before that's read, by one playlist the size of NMF Denmark for a Danish act (0 events). + +| Group | Releases | DR add within 8 weeks | P3 | P4 | P5 | P6 | P8 | +|---|---|---|---|---|---|---|---| +| NMF Denmark | 1360 | 11.2% | 4% | 5% | 3% | 3% | 1% | +| NMF Denmark, top 10 position | 255 | 18.8% | 12% | 17% | 6% | 4% | 0% | +| NMF elsewhere only | 2300 | 1.9% | 0% | 1% | 0% | 0% | 0% | +| Not on NMF | 5429 | 3.1% | 0% | 0% | 0% | 0% | 2% | +| Danish acts: NMF Denmark | 499 | 14.4% | 5% | 9% | 3% | 4% | 1% | +| Danish acts: not on NMF | 1457 | 5.7% | 1% | 1% | 1% | 0% | 4% | +| Not in rotation at release: NMF Denmark | 1174 | 12.7% | 4% | 4% | 3% | 3% | 1% | +| Not in rotation at release: not on NMF | 5269 | 3.0% | 0% | 0% | 0% | 0% | 2% | + +Weeks from the release to the add, NMF Denmark songs that got one: median 3; wk 0: 19, wk 1: 30, wk 2: 20, wk 3: 27, wk 4: 11, wk 5: 14, wk 6: 13, wk 7: 12, wk 8: 7. + + +Logit for a DR add within 8 weeks of a release (9089 releases): + +| Covariate | Coefficient | Odds ratio (95% CI) | +|---|---|---| +| intercept | -4.134 | 0.02 (0.01–0.02) | +| NMF Denmark | +0.923 | 2.52 (1.95–3.26) | +| NMF elsewhere only | -0.375 | 0.69 (0.48–0.98) | +| log DR plays/week, last 12 weeks | +0.760 | 2.14 (1.94–2.36) | +| in rotation at release | -2.532 | 0.08 (0.04–0.14) | +| Danish act | +0.639 | 1.89 (1.49–2.41) | +| log Airplay Top-20 hits | -0.638 | 0.53 (0.41–0.68) | +| log Track Top-40 hits | -0.056 | 0.95 (0.80–1.12) | + +Observational: NMF picks and radio adds share causes (label push, artist momentum), so the odds ratio is an association after the listed controls, not the effect of being picked. + diff --git a/docs/SYNTH.md b/docs/SYNTH.md new file mode 100644 index 0000000..7d3c55a --- /dev/null +++ b/docs/SYNTH.md @@ -0,0 +1,45 @@ +# Synthetic rotation lists + +## 1. Plays per day by category (exact plays vs DR's lists) + +Weeks used: 2026-11-09 p4esbjerg (bridged), 2026-11-09 p4vest (bridged), 2026-11-09 p4trekanten (bridged), 2026-11-09 p6beat (bridged), 2026-11-09 p5 (bridged), 2026-11-09 p8jazz (bridged) …. + +Synthetic lists only say "on the list or not": DR's list formats have changed over the years, and A/B can't be told apart from plays on every channel. The category medians are for reference. + +| Channel | A median | B median | C median | Not listed, 90th pct | On-list threshold (plays/day) | +|---|---|---|---|---|---| +| p3 | 4.9 | 3.4 | 1.7 | 0.7 | 3.1 | +| p4 | 2.6 | 2.3 | 1.3 | 0.3 | 0.8 | +| p5 | 2.9 | 1.3 | 1.3 | 0.3 | 0.8 | +| p6beat | 4.1 | 2.6 | 1.4 | 0.4 | 1.1 | +| p8jazz | 1.6 | 1.0 | 0.7 | 0.3 | 0.6 | + +In-sample check (song-weeks): listed songs found 373 of 393; songs wrongly put on the list 150. Thresholds are refitted as exact plays and lists for the same weeks accumulate. + +Era-robust rule used for the archive: on the list = among the channel's N most played songs that week, N = the median size of DR's lists (p8jazz 45, p4 42, p3 32, p5 25, p6beat 36). Where the real list is known it finds 373 of 395 listed songs, with 185 songs wrongly on (precision 67%, recall 94%). + +## 2. Archived playlists → synthetic lists + +| Station | Year | 4-week blocks with a synthetic list | Avg songs listed | +|---|---|---|---| +| p3 | 2016 | 5 | 32 | +| p3 | 2017 | 8 | 32 | +| p4 | 2016 | 2 | 42 | +| p4 | 2017 | 1 | 42 | +| p6beat | 2017 | 1 | 36 | + +Lists are per 4-week block (iso_year/iso_week = its first week): a block needs at least 96 archived hours; a song needs 3+ plays and is then ranked against the block's other songs (top N = on the list in that block). Weekly lists from a few sampled days are mostly sampling noise. + +## 3. Airplay Top-20 position → DR plays a week + +Artist's DR plays (all 14 channels) in the chart week, for Airplay Top-20 entries 2025-11-17–2026-11-02: + +| Positions | Entries | Median DR plays | 25th–75th pct | +|---|---|---|---| +| 1–5 | 238 | 294 | 245–327 | +| 6–10 | 243 | 253 | 215–290 | +| 11–15 | 243 | 237 | 188–269 | +| 16–20 | 236 | 196 | 151–233 | + +So the 25-year Airplay Top-20 can stand in for "in heavy rotation nationally" (on the order of the plays above), but it only covers the top 20 songs each week; it can't produce B/C lists or anything below. + diff --git a/src/model/adds.rs b/src/model/adds.rs index f51cb07..24cd810 100644 --- a/src/model/adds.rs +++ b/src/model/adds.rs @@ -52,6 +52,20 @@ pub fn wave_feats(p: &Panel, r: &Rot, a: usize, t: usize) -> [f64; WAVE] { f(known && (13.0..=25.0).contains(&air)), f(ratio_ok), f(known && air >= 104.0)] } +/// Long-history covariates: DR's 2012–2021 archive and DR's real rotation lists before week t. +pub const LONG: usize = 4; +pub const LONG_NAMES: &[&str] = &["log archive plays per 100 sampled hours (2012-21)", "log DR lists with the artist before t (2023-)", + "DR list recency 26w", "DR lists known before t at all"]; + +pub fn long_feats(p: &Panel, a: usize, t: usize) -> [f64; LONG] { + if off("long") { return [0.0; LONG]; } + let end_t = p.weeks[t] + chrono::Duration::days(6); + // A list published on a Friday is known from then. + let before: Vec<&chrono::NaiveDate> = p.listed_on[a].iter().filter(|d| **d <= end_t).collect(); + let since = before.last().map(|d| (end_t - **d).num_days() as f64 / 7.0); + [p.archive_rate[a].ln_1p(), (before.len() as f64).ln_1p(), since.map(|s| (-s / 26.0).exp()).unwrap_or(0.0), (!before.is_empty()) as i32 as f64] +} + /// Release covariates (Deezer discography), appended to both row kinds. pub const REL: usize = 6; pub const REL_NAMES: &[&str] = &["discography known", "release recency 4w", "release recency 12w", "log releases last 52w", @@ -88,7 +102,7 @@ pub const EXTRA: &[&str] = &["in rotation on channel now", "in rotation anywhere "run on channel = 1 week", "run = 2 weeks", "run = 3-5 weeks", "run >= 6 weeks", "near threshold (>= half of tau, not in rotation)", "plays / tau (capped 3, /3)", "weeks since in rotation on channel (/12)"]; -pub const PC: usize = P + 10 + REL + WAVE; +pub const PC: usize = P + 10 + REL + WAVE + LONG; /// Artist-level ("any channel") rows. pub const ANY_NAMES: &[&str] = &[ @@ -98,7 +112,7 @@ pub const ANY_NAMES: &[&str] = &[ "log Track Top-40 hits", "log Airplay Top-20 hits", "Danish", "group", "female", "career years (/50)", "debut unknown", "Christmas in window", ]; -pub const PA: usize = 21 + REL + WAVE; +pub const PA: usize = 21 + REL + WAVE + LONG; /// Positions of the chart covariates (for the what-if): (t40 score, air score, recency 6w, recency 26w). pub const CHART_IDX_CHANNEL: [usize; 4] = [10, 11, 12, 13]; pub const CHART_IDX_ANY: [usize; 4] = [9, 10, 11, 12]; @@ -161,7 +175,8 @@ pub fn row_channel(p: &Panel, r: &Rot, a: usize, c: usize, t: usize, h: usize, r f(weeks == 1), f(weeks == 2), f((3..=5).contains(&weeks)), f(weeks >= 6), f(!on && ratio >= 0.5), ratio.min(3.0) / 3.0, since]); out[P + 10..P + 10 + REL].copy_from_slice(&rel_feats(p, a, t, h, rel, on)); - out[P + 10 + REL..PC].copy_from_slice(&wave_feats(p, r, a, t)); + out[P + 10 + REL..P + 10 + REL + WAVE].copy_from_slice(&wave_feats(p, r, a, t)); + out[P + 10 + REL + WAVE..PC].copy_from_slice(&long_feats(p, a, t)); } pub fn row_any(p: &Panel, r: &Rot, a: usize, t: usize, h: usize, rel: Option, out: &mut [f64]) { @@ -186,7 +201,8 @@ pub fn row_any(p: &Panel, r: &Rot, a: usize, t: usize, h: usize, rel: Option, /// Release dates per artist (Deezer discography: singles, EPs, albums), sorted; None = not looked up / not found. pub releases: Vec>>, + /// Long history: the artist's plays per 100 archived hours on DR (Wayback, 2012–2021), and the + /// dates of DR's real rotation lists (2023–) that had the artist on any channel. + pub archive_rate: Vec, + pub listed_on: Vec>, } impl Panel { @@ -139,7 +143,8 @@ impl Panel { let chart = chart_states(db, &weeks, &names, artists.len())?; let list_tau = super::calibrate::stored(db)?; let releases = load_releases(db, &id_of, artists.len())?; - Ok(Panel { weeks, artists, y, chart, list_tau, releases }) + let (archive_rate, listed_on) = load_long_history(db, &id_of, &names, artists.len())?; + Ok(Panel { weeks, artists, y, chart, list_tau, releases, archive_rate, listed_on }) } } @@ -242,3 +247,28 @@ fn load_releases(db: &Db, id_of: &HashMap, n: usize) -> Result, names: &HashMap>, n: usize) -> Result<(Vec, Vec>)> { + let mut rate = vec![0.0; n]; + let has = |t: &str| -> Result { Ok(db.conn.query_row("SELECT count(*) FROM sqlite_master WHERE name = ?1", [t], |r| r.get::<_, i64>(0))? > 0) }; + if has("archive_plays")? { + let hours: f64 = db.conn.query_row("SELECT count(DISTINCT station || substr(played_at, 1, 13)) FROM archive_plays", [], |r| r.get(0))?; + let mut st = db.conn.prepare("SELECT artist_ids FROM archive_plays WHERE artist_ids != ''")?; + let rows = st.query_map([], |r| r.get::<_, String>(0))?; + for r in rows { + for id in r?.split(',') { if let Some(&a) = id_of.get(id) { rate[a] += 1.0; } } + } + if hours > 0.0 { for v in rate.iter_mut() { *v *= 100.0 / hours; } } + } + let mut listed: Vec> = vec![Vec::new(); n]; + if has("rotation_lists")? { + for r in db.conn.prepare("SELECT DISTINCT list_date, artist FROM rotation_lists")?.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? { + let (d, artist) = r?; + let Ok(d) = NaiveDate::parse_from_str(&d, "%Y-%m-%d") else { continue }; + let list_artist = artist.replace(" [+] ", ", "); + for a in chart_artists(&list_artist, names) { listed[a].push(d); } + } + } + for v in listed.iter_mut() { v.sort(); v.dedup(); } + Ok((rate, listed)) +} diff --git a/src/model/synth.rs b/src/model/synth.rs index c69dc99..8ade30a 100644 --- a/src/model/synth.rs +++ b/src/model/synth.rs @@ -150,6 +150,33 @@ pub fn run(db: &Db, p: &Panel) -> Result { let false_pos: usize = hit[3][..3].iter().sum(); writeln!(md, "\nIn-sample check (song-weeks): listed songs found {listed_found} of {listed_total}; songs wrongly put on the list {false_pos}. Thresholds are refitted as exact plays and lists for the same weeks accumulate.\n")?; + // Typical list size per channel (DR's real lists: A/B/C or one list), the era-robust rule: + // on the list = among the channel's N most played songs that week. + let mut list_size: HashMap = HashMap::new(); + for lc in ["p3", "p4", "p5", "p6beat", "p8jazz"] { + let mut v: Vec = db.conn.prepare("SELECT count(*) FROM rotation_lists WHERE channel = ?1 AND category IN ('A', 'B', 'C', 'LIST') GROUP BY list_date")? + .query_map([lc], |r| r.get(0))?.collect::>()?; + v.sort(); + if !v.is_empty() { list_size.insert(lc.to_string(), v[v.len() / 2] as usize); } + } + // Check the rule where the real list is known: top N by plays vs the list. + let (mut tp, mut fp, mut fnn) = (0usize, 0usize, 0usize); + for (week, by_station) in &plays { + let days = days_seen.get(week).map(|s| s.len()).unwrap_or(0); + if days < 5 { continue; } + for (station, songs) in by_station { + let Some(lc) = list_channel(station) else { continue }; + let (Some((list, _)), Some(&n)) = (list_for(*week, lc), list_size.get(lc)) else { continue }; + let mut ranked: Vec<(&String, &f64)> = songs.iter().collect(); + ranked.sort_by(|a, b| b.1.total_cmp(a.1)); + let top: std::collections::HashSet<&String> = ranked.iter().take(n).map(|x| x.0).collect(); + for s in &top { if list.contains_key(*s) { tp += 1 } else { fp += 1 } } + fnn += list.keys().filter(|s| !top.contains(s)).count(); + } + } + writeln!(md, "Era-robust rule used for the archive: on the list = among the channel's N most played songs that week, N = the median size of DR's lists ({}). Where the real list is known it finds {tp} of {} listed songs, with {fp} songs wrongly on (precision {:.0}%, recall {:.0}%).\n", + list_size.iter().map(|(k, v)| format!("{k} {v}")).collect::>().join(", "), tp + fnn, 100.0 * tp as f64 / (tp + fp).max(1) as f64, 100.0 * tp as f64 / (tp + fnn).max(1) as f64)?; + // ---- 2. apply to the archive let has_archive: i64 = db.conn.query_row("SELECT count(*) FROM sqlite_master WHERE name = 'archive_plays'", [], |r| r.get(0))?; writeln!(md, "## 2. Archived playlists → synthetic lists\n")?; @@ -163,7 +190,12 @@ pub fn run(db: &Db, p: &Panel) -> Result { .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, String>(3)?)))? { let (st, at, a, t) = r?; let Ok(d) = NaiveDate::parse_from_str(&at[..10], "%Y-%m-%d") else { continue }; - let w = weeks.entry((st, d.iso_week().year(), d.iso_week().week())).or_insert_with(|| Wk { hours: HashMap::new(), songs: HashMap::new() }); + // 4-week blocks (from Monday 3 Jan 2000): a few sampled days a week are too noisy for a + // weekly list (two 36-hour samples of one week's P3 differ by ~9 of the top 32 songs), + // pooled over 4 weeks the list holds; songs stay on the list ~7+ weeks anyway. + let epoch = NaiveDate::from_ymd_opt(2000, 1, 3).unwrap(); + let block = epoch + Duration::weeks(((d - epoch).num_days() / 7 / 4) * 4); + let w = weeks.entry((st, block.iso_week().year(), block.iso_week().week())).or_insert_with(|| Wk { hours: HashMap::new(), songs: HashMap::new() }); w.hours.entry(at[..10].to_string()).or_default().insert(at[11..13].to_string()); let e = w.songs.entry(song(&a, &t)).or_insert((0.0, a.clone(), t.clone())); e.0 += 1.0; @@ -173,17 +205,18 @@ pub fn run(db: &Db, p: &Panel) -> Result { let mut summary: BTreeMap<(String, i32), (usize, usize)> = BTreeMap::new(); // (station, year) -> (weeks, songs listed) for ((st, y, w), wk) in &weeks { let Some(lc) = list_channel(st) else { continue }; - let Some(th) = thresholds.get(lc) else { continue }; + if !thresholds.contains_key(lc) { continue; } // Only daytime-complete days count fully; scale plays by covered hours / 24. let hours: usize = wk.hours.values().map(|h| h.len()).sum(); let days = wk.hours.values().filter(|h| h.len() >= 12).count(); - if hours < 36 { continue; } // less than a day and a half: too thin to tell a C song from chance + if hours < 96 { continue; } // under 4 days' worth in the block: too thin (two 72-hour samples of one week still differ by ~5 of 32) let day_equiv = hours as f64 / 24.0; + let Some(&size) = list_size.get(lc) else { continue }; + let mut ranked: Vec<(&String, &(f64, String, String))> = wk.songs.iter().filter(|(_, v)| v.0 >= 3.0).collect(); + ranked.sort_by(|a, b| b.1 .0.total_cmp(&a.1 .0)); let mut n = 0; - for (s, (cnt, a, t)) in &wk.songs { - if *cnt < 3.0 { continue; } // a song needs 3+ plays in the sample to count + for (s, (cnt, a, t)) in ranked.into_iter().take(size) { let ppd = cnt / day_equiv; - if ppd < th[2] { continue; } tx.execute("INSERT OR REPLACE INTO synthetic_rotation VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, 'wayback')", rusqlite::params![lc, y, w, s, a, t, ppd, "listed", days as i64, hours as i64])?; n += 1; @@ -192,9 +225,9 @@ pub fn run(db: &Db, p: &Panel) -> Result { e.0 += 1; e.1 += n; } tx.commit()?; - writeln!(md, "| Station | Year | Weeks with a synthetic list | Avg songs listed per week |\n|---|---|---|---|")?; + writeln!(md, "| Station | Year | 4-week blocks with a synthetic list | Avg songs listed |\n|---|---|---|---|")?; for ((st, y), (wks, n)) in &summary { writeln!(md, "| {st} | {y} | {wks} | {:.0} |", *n as f64 / *wks as f64)?; } - writeln!(md, "\nA week needs at least 36 archived hours and a song at least 3 plays in them; plays are scaled to a full day by the hours covered.\n")?; + writeln!(md, "\nLists are per 4-week block (iso_year/iso_week = its first week): a block needs at least 96 archived hours; a song needs 3+ plays and is then ranked against the block's other songs (top N = on the list in that block). Weekly lists from a few sampled days are mostly sampling noise.\n")?; } // ---- 3. IFPI Airplay Top-20 position vs DR plays