From 94abfa02551e620aa22589a5b8c0d11417296be0 Mon Sep 17 00:00:00 2001 From: moritsbw-cpu Date: Mon, 16 Nov 2026 00:22:55 +0000 Subject: [PATCH] Old DR artist pages from the Wayback Machine (53 weeks of plays per capture, 2019-2024) into dr_artist_history; leak fixes (weeks since last add counts only adds up to t-1; training windows close a week before the origin) and a leak test (model leakcheck) that rebuilds every feature from data cut at the origin; backtests split pre-release (NMF from history) and post-release (NMF known); site shows the new-song backtest Co-Authored-By: Claude Opus 5.5 --- docs/BACKTEST-ADDS.md | 283 +++++++++++++++++++++------------------- docs/BACKTEST-EVENTS.md | 91 ++++++------- src/main.rs | 10 +- src/model/adds.rs | 67 ++++++---- src/model/leakcheck.rs | 46 +++++++ src/model/mod.rs | 2 + src/model/panel.rs | 22 ++++ src/oldpages.rs | 194 +++++++++++++++++++++++++++ src/web/index.html | 20 ++- 9 files changed, 512 insertions(+), 223 deletions(-) create mode 100644 src/model/leakcheck.rs create mode 100644 src/oldpages.rs diff --git a/docs/BACKTEST-ADDS.md b/docs/BACKTEST-ADDS.md index 869a802..0484bf3 100644 --- a/docs/BACKTEST-ADDS.md +++ b/docs/BACKTEST-ADDS.md @@ -4,181 +4,190 @@ An add = the artist enters rotation (plays ≥ τ) after at least 4 weeks out of ## Window starting h = 0 weeks ahead (17 forecast origins) -Base rates: per channel 0.32% of artist-channels get an add in the window; on any channel 1.8% of artists. +Base rates: per channel 0.43% of artist-channels get an add in the window; on any channel 2.4% of artists. | Model | Log-loss, per channel | PR-AUC, per channel | Log-loss, any channel | PR-AUC, any channel | |---|---|---|---|---| -| rate by in-rotation-now | 0.0214 | 0.006 | 0.0913 | 0.021 | -| rate by last-12-weeks level | 0.0147 | 0.093 | 0.0781 | 0.072 | -| logit | 0.0119 | 0.229 | 0.0679 | 0.204 | +| rate by in-rotation-now | 0.0255 | 0.032 | 0.1058 | 0.067 | +| rate by last-12-weeks level | 0.0177 | 0.147 | 0.0909 | 0.133 | +| logit (pre-release: NMF from history) | 0.0147 | 0.258 | 0.0800 | 0.251 | +| logit (post-release: NMF known) | 0.0145 | 0.266 | 0.0799 | 0.259 | Skill of the logit (90% block-bootstrap interval over origins): | | vs in-rotation-now | vs last-12-weeks level | |---|---|---| -| per channel | 0.441 (0.427 to 0.451) | 0.187 (0.174 to 0.191) | -| any channel | 0.256 (0.238 to 0.275) | 0.130 (0.120 to 0.141) | +| per channel | 0.424 (0.413 to 0.437) | 0.171 (0.168 to 0.178) | +| any channel | 0.244 (0.234 to 0.257) | 0.120 (0.115 to 0.127) | Calibration, per channel: | Predicted | Rows | Mean predicted | Observed | |---|---|---|---| -| 0.0–0.1 | 1876920 | 0.001 | 0.002 | -| 0.1–0.2 | 6259 | 0.142 | 0.211 | -| 0.2–0.3 | 2324 | 0.243 | 0.272 | -| 0.3–0.4 | 933 | 0.342 | 0.279 | -| 0.4–0.5 | 710 | 0.450 | 0.304 | -| 0.5–0.6 | 613 | 0.550 | 0.303 | -| 0.6–0.7 | 445 | 0.647 | 0.353 | -| 0.7–0.8 | 283 | 0.748 | 0.502 | -| 0.8–0.9 | 237 | 0.844 | 0.422 | -| 0.9–1.0 | 44 | 0.921 | 0.545 | +| 0.0–0.1 | 1868823 | 0.002 | 0.002 | +| 0.1–0.2 | 9096 | 0.142 | 0.160 | +| 0.2–0.3 | 3991 | 0.247 | 0.230 | +| 0.3–0.4 | 2858 | 0.344 | 0.215 | +| 0.4–0.5 | 1615 | 0.448 | 0.310 | +| 0.5–0.6 | 1081 | 0.544 | 0.457 | +| 0.6–0.7 | 728 | 0.646 | 0.475 | +| 0.7–0.8 | 429 | 0.749 | 0.441 | +| 0.8–0.9 | 104 | 0.834 | 0.346 | +| 0.9–1.0 | 43 | 0.917 | 0.930 | ## Window starting h = 4 weeks ahead (13 forecast origins) -Base rates: per channel 0.33% of artist-channels get an add in the window; on any channel 1.8% of artists. +Base rates: per channel 0.44% of artist-channels get an add in the window; on any channel 2.4% of artists. | Model | Log-loss, per channel | PR-AUC, per channel | Log-loss, any channel | PR-AUC, any channel | |---|---|---|---|---| -| rate by in-rotation-now | 0.0213 | 0.014 | 0.0896 | 0.031 | -| rate by last-12-weeks level | 0.0158 | 0.084 | 0.0801 | 0.066 | -| logit | 0.0147 | 0.134 | 0.0744 | 0.143 | +| rate by in-rotation-now | 0.0263 | 0.032 | 0.1051 | 0.083 | +| rate by last-12-weeks level | 0.0193 | 0.114 | 0.0937 | 0.136 | +| logit (pre-release: NMF from history) | 0.0176 | 0.166 | 0.0857 | 0.208 | +| logit (post-release: NMF known) | 0.0172 | 0.175 | 0.0856 | 0.218 | Skill of the logit (90% block-bootstrap interval over origins): | | vs in-rotation-now | vs last-12-weeks level | |---|---|---| -| per channel | 0.310 (0.302 to 0.316) | 0.070 (0.042 to 0.087) | -| any channel | 0.170 (0.152 to 0.186) | 0.071 (0.064 to 0.080) | +| per channel | 0.330 (0.315 to 0.337) | 0.091 (0.065 to 0.106) | +| any channel | 0.185 (0.174 to 0.194) | 0.086 (0.081 to 0.090) | Calibration, per channel: | Predicted | Rows | Mean predicted | Observed | |---|---|---|---| -| 0.0–0.1 | 1434699 | 0.002 | 0.002 | -| 0.1–0.2 | 5169 | 0.142 | 0.147 | -| 0.2–0.3 | 2108 | 0.243 | 0.164 | -| 0.3–0.4 | 980 | 0.344 | 0.198 | -| 0.4–0.5 | 621 | 0.446 | 0.203 | -| 0.5–0.6 | 404 | 0.544 | 0.223 | -| 0.6–0.7 | 255 | 0.644 | 0.475 | -| 0.7–0.8 | 74 | 0.751 | 0.203 | -| 0.8–0.9 | 42 | 0.837 | 0.310 | +| 0.0–0.1 | 1428760 | 0.002 | 0.003 | +| 0.1–0.2 | 7389 | 0.141 | 0.129 | +| 0.2–0.3 | 3960 | 0.247 | 0.186 | +| 0.3–0.4 | 2024 | 0.344 | 0.199 | +| 0.4–0.5 | 1134 | 0.444 | 0.283 | +| 0.5–0.6 | 604 | 0.546 | 0.224 | +| 0.6–0.7 | 394 | 0.643 | 0.459 | +| 0.7–0.8 | 83 | 0.732 | 0.277 | +| 0.8–0.9 | 3 | 0.833 | 1.000 | +| 0.9–1.0 | 1 | 0.907 | 1.000 | ## Coefficients, window starting now (h = 0), all closed windows | Any-channel feature | Coefficient | |---|---| -| intercept | -4.804 | -| log DR plays t | +0.988 | -| log DR plays 4w | +0.335 | -| log DR plays 12w | -0.407 | -| log max DR plays 12w | +0.696 | -| in rotation anywhere now | -1.287 | -| station groups in rotation now (/5) | -3.848 | -| current run anywhere (/12) | -1.865 | -| weeks since in rotation anywhere (/12) | -0.272 | -| Track Top-40 score | +0.074 | -| Airplay Top-20 score | -0.687 | -| chart entry recency 6w | +1.656 | -| chart entry recency 26w | -1.565 | -| log Track Top-40 hits | -0.209 | -| log Airplay Top-20 hits | -0.197 | -| Danish | +0.499 | -| group | -0.341 | -| female | -0.140 | -| career years (/50) | +0.297 | -| debut unknown | +0.622 | +| intercept | -4.213 | +| log DR plays t | +0.883 | +| log DR plays 4w | +0.253 | +| log DR plays 12w | -0.263 | +| log max DR plays 12w | +0.594 | +| in rotation anywhere now | -1.148 | +| station groups in rotation now (/5) | +0.297 | +| current run anywhere (/12) | -1.212 | +| weeks since in rotation anywhere (/12) | -0.749 | +| Track Top-40 score | +0.427 | +| Airplay Top-20 score | -1.332 | +| chart entry recency 6w | +1.210 | +| chart entry recency 26w | -0.653 | +| log Track Top-40 hits | -0.200 | +| log Airplay Top-20 hits | -0.069 | +| Danish | +0.462 | +| group | -0.240 | +| female | -0.102 | +| career years (/50) | +0.210 | +| debut unknown | +0.497 | | Christmas in window | +0.000 | -| discography known | -0.354 | -| release recency 4w | +0.199 | -| release recency 12w | +0.894 | -| log releases last 52w | -0.200 | -| releases in the window | -1.019 | -| releases in window x not in rotation now | +1.573 | -| DR: weeks since last add 13-25 | +0.099 | -| DR: last add 26+ weeks ago | -0.438 | -| DR: no add in the panel yet | -0.948 | -| Airplay Top-20: last new song 13-25 weeks ago | +0.299 | -| Airplay Top-20: 0.5-1.25x own usual gap | -0.018 | +| discography known | -0.325 | +| release recency 4w | +0.440 | +| release recency 12w | +0.670 | +| log releases last 52w | -0.188 | +| releases in the window | -0.218 | +| releases in window x not in rotation now | +0.854 | +| DR: weeks since last add 13-25 | +0.095 | +| DR: last add 26+ weeks ago | -0.642 | +| DR: no add in the panel yet | -0.847 | +| Airplay Top-20: last new song 13-25 weeks ago | +0.279 | +| Airplay Top-20: 0.5-1.25x own usual gap | +0.231 | | Airplay Top-20: last new song 2+ years ago | -0.529 | -| log archive plays per 100 sampled hours (2012-21) | +0.366 | -| log DR lists with the artist before t (2023-) | +0.332 | -| DR list recency 26w | -0.621 | -| DR lists known before t at all | -0.166 | -| NMF Denmark in the window | +0.985 | -| NMF Denmark top 10 in the window | +0.444 | -| log NMF Denmark placements last 52w | -0.079 | -| share of releases on NMF Denmark (2 years) | -0.623 | +| log archive plays per 100 sampled hours (2012-21) | +0.288 | +| log DR lists with the artist before t (2023-) | +0.209 | +| DR list recency 26w | -0.929 | +| DR lists known before t at all | +0.040 | +| NMF Denmark in the window | +0.941 | +| NMF Denmark top 10 in the window | +0.416 | +| log NMF Denmark placements last 52w | -0.071 | +| share of releases on NMF Denmark (2 years) | -0.415 | +| season index of the window (log, IFPI 2002-24) | +0.403 | | Per-channel feature | Coefficient | |---|---| -| intercept | -7.632 | -| log plays t | +0.821 | -| log plays t-1 | +0.119 | -| log mean plays 4w | -0.008 | -| log mean plays 12w | -0.531 | -| log max plays 12w | +1.318 | -| log other channels t | +0.235 | -| log other channels 4w | -0.087 | -| log P3 t | +0.037 | -| weeks since played (/12) | -1.762 | -| Track Top-40 score | -0.273 | -| Airplay Top-20 score | -0.757 | -| chart entry recency 6w | +0.747 | -| chart entry recency 26w | -0.459 | -| log Track Top-40 hits | +0.226 | -| log Airplay Top-20 hits | -0.351 | -| Danish | +0.235 | -| group | -0.650 | -| female | +0.012 | -| career years (/50) | -1.128 | -| debut unknown | -0.056 | +| intercept | -7.101 | +| log plays t | +1.122 | +| log plays t-1 | +0.142 | +| log mean plays 4w | +0.113 | +| log mean plays 12w | -0.862 | +| log max plays 12w | +1.014 | +| log other channels t | +0.332 | +| log other channels 4w | -0.100 | +| log P3 t | +0.085 | +| weeks since played (/12) | -0.437 | +| Track Top-40 score | -0.127 | +| Airplay Top-20 score | -1.373 | +| chart entry recency 6w | +0.625 | +| chart entry recency 26w | -0.512 | +| log Track Top-40 hits | +0.257 | +| log Airplay Top-20 hits | -0.395 | +| Danish | +0.162 | +| group | -0.629 | +| female | +0.127 | +| career years (/50) | -1.500 | +| debut unknown | -0.289 | | Christmas target week | +0.000 | -| Danish x P3 | -0.051 | -| Danish x P4 | +0.302 | -| Danish x P6 | +0.004 | -| ch p4kbh | +0.695 | -| ch p4sjaelland | +0.514 | -| ch p4fyn | +0.499 | -| ch p4syd | +0.544 | -| ch p4trekanten | +0.581 | -| ch p4vest | +0.611 | -| ch p4aarhus | +0.526 | -| ch p4nord | +0.587 | -| ch p4bornholm | +0.536 | -| ch p4esbjerg | +0.555 | -| ch p5 | +1.820 | -| ch p6beat | +1.118 | -| ch p8jazz | +2.402 | -| in rotation on channel now | -0.952 | -| in rotation anywhere now | +0.118 | -| station groups in rotation now (/5) | -1.127 | -| run on channel = 1 week | +0.893 | -| run = 2 weeks | +0.413 | -| run = 3-5 weeks | -1.287 | -| run >= 6 weeks | -0.971 | -| near threshold (>= half of tau, not in rotation) | +1.421 | -| plays / tau (capped 3, /3) | -0.967 | -| weeks since in rotation on channel (/12) | +0.425 | -| discography known | -0.042 | -| release recency 4w | +0.840 | -| release recency 12w | +0.180 | -| log releases last 52w | -0.059 | -| releases in the window | -0.478 | -| releases in window x not in rotation now | +1.007 | -| DR: weeks since last add 13-25 | +0.007 | -| DR: last add 26+ weeks ago | +0.310 | -| DR: no add in the panel yet | -0.072 | -| Airplay Top-20: last new song 13-25 weeks ago | +0.084 | -| Airplay Top-20: 0.5-1.25x own usual gap | +0.103 | -| Airplay Top-20: last new song 2+ years ago | -0.240 | -| log archive plays per 100 sampled hours (2012-21) | +0.179 | -| log DR lists with the artist before t (2023-) | +0.175 | -| DR list recency 26w | -0.254 | -| DR lists known before t at all | -0.234 | -| NMF Denmark in the window | +0.853 | -| NMF Denmark top 10 in the window | +0.479 | -| log NMF Denmark placements last 52w | -0.122 | -| share of releases on NMF Denmark (2 years) | -0.189 | +| Danish x P3 | +0.072 | +| Danish x P4 | +0.085 | +| Danish x P6 | +0.154 | +| ch p4kbh | +1.100 | +| ch p4sjaelland | +0.968 | +| ch p4fyn | +0.948 | +| ch p4syd | +0.976 | +| ch p4trekanten | +0.992 | +| ch p4vest | +1.078 | +| ch p4aarhus | +0.934 | +| ch p4nord | +1.002 | +| ch p4bornholm | +0.986 | +| ch p4esbjerg | +0.968 | +| ch p5 | +2.081 | +| ch p6beat | +1.295 | +| ch p8jazz | +2.330 | +| in rotation on channel now | +0.358 | +| in rotation anywhere now | -0.338 | +| station groups in rotation now (/5) | +0.043 | +| run on channel = 1 week | +0.663 | +| run = 2 weeks | +0.441 | +| run = 3-5 weeks | -0.055 | +| run >= 6 weeks | -0.691 | +| near threshold (>= half of tau, not in rotation) | +1.376 | +| plays / tau (capped 3, /3) | -3.407 | +| weeks since in rotation on channel (/12) | -0.300 | +| discography known | -0.069 | +| release recency 4w | +1.033 | +| release recency 12w | -0.008 | +| log releases last 52w | -0.003 | +| releases in the window | -0.143 | +| releases in window x not in rotation now | +0.782 | +| DR: weeks since last add 13-25 | +0.107 | +| DR: last add 26+ weeks ago | -0.085 | +| DR: no add in the panel yet | -0.486 | +| Airplay Top-20: last new song 13-25 weeks ago | +0.194 | +| Airplay Top-20: 0.5-1.25x own usual gap | -0.009 | +| Airplay Top-20: last new song 2+ years ago | -0.417 | +| log archive plays per 100 sampled hours (2012-21) | +0.348 | +| log DR lists with the artist before t (2023-) | +0.028 | +| DR list recency 26w | -0.444 | +| DR lists known before t at all | +0.055 | +| NMF Denmark in the window | +0.805 | +| NMF Denmark top 10 in the window | +0.579 | +| log NMF Denmark placements last 52w | -0.077 | +| share of releases on NMF Denmark (2 years) | +0.166 | +| season index of the window (log, IFPI 2002-24) | +1.240 | +| share of the artist's DR plays on this station (before t) | +0.963 | +| log DR plays on this station (before t) | +0.049 | +| never played on this station (before t) | -2.888 | +| genre fit for this station | +0.108 | diff --git a/docs/BACKTEST-EVENTS.md b/docs/BACKTEST-EVENTS.md index 6a4c833..e920d94 100644 --- a/docs/BACKTEST-EVENTS.md +++ b/docs/BACKTEST-EVENTS.md @@ -4,53 +4,54 @@ Events: an artist in the panel releases something (Deezer) in week t0+1, with no | Model | Log-loss, any channel | PR-AUC, any channel | Log-loss, per channel | PR-AUC, per channel | |---|---|---|---|---| -| rate by in-rotation-now | 0.2073 | 0.118 | 0.0666 | 0.052 | -| rate by last-12-weeks level | 0.1831 | 0.201 | 0.0492 | 0.190 | -| event model | 0.1803 | 0.258 | 0.0436 | 0.320 | +| rate by in-rotation-now | 0.2073 | 0.115 | 0.0666 | 0.059 | +| rate by last-12-weeks level | 0.1838 | 0.215 | 0.0491 | 0.202 | +| event model (pre-release: NMF from history) | 0.1854 | 0.248 | 0.0445 | 0.314 | +| event model (post-release: NMF known) | 0.1814 | 0.280 | 0.0422 | 0.325 | -Skill of the event model, any channel: 0.130 (0.085 to 0.187) vs in-rotation-now, 0.015 (-0.036 to 0.061) vs last-12-weeks level. +Skill of the event model, any channel: 0.106 (0.081 to 0.178) vs in-rotation-now, -0.008 (-0.051 to 0.068) vs last-12-weeks level. | Any-channel feature (all events) | Coefficient | |---|---| -| intercept | -3.676 | -| log DR plays t | +0.461 | -| log DR plays 4w | +0.667 | -| log DR plays 12w | -0.467 | -| log max DR plays 12w | +0.260 | -| in rotation anywhere now | -0.464 | -| station groups in rotation now (/5) | -0.219 | -| current run anywhere (/12) | -1.217 | -| weeks since in rotation anywhere (/12) | -0.542 | -| Track Top-40 score | -0.339 | -| Airplay Top-20 score | -0.125 | -| chart entry recency 6w | +0.509 | -| chart entry recency 26w | +0.377 | -| log Track Top-40 hits | -0.310 | -| log Airplay Top-20 hits | -0.180 | -| Danish | +0.640 | -| group | +0.038 | -| female | -0.081 | -| career years (/50) | +0.505 | -| debut unknown | +0.552 | +| intercept | -3.665 | +| log DR plays t | +0.557 | +| log DR plays 4w | +0.507 | +| log DR plays 12w | -0.363 | +| log max DR plays 12w | +0.249 | +| in rotation anywhere now | -0.476 | +| station groups in rotation now (/5) | -0.252 | +| current run anywhere (/12) | -1.197 | +| weeks since in rotation anywhere (/12) | -0.514 | +| Track Top-40 score | -0.282 | +| Airplay Top-20 score | -0.313 | +| chart entry recency 6w | +0.713 | +| chart entry recency 26w | +0.477 | +| log Track Top-40 hits | -0.303 | +| log Airplay Top-20 hits | -0.118 | +| Danish | +0.580 | +| group | +0.043 | +| female | -0.064 | +| career years (/50) | +0.433 | +| debut unknown | +0.530 | | Christmas in window | +0.000 | -| discography known | -0.000 | -| release recency 4w | +0.167 | -| release recency 12w | -0.863 | -| log releases last 52w | -0.208 | -| releases in the window | -0.000 | -| releases in window x not in rotation now | +0.464 | -| DR: weeks since last add 13-25 | -0.027 | -| DR: last add 26+ weeks ago | +0.411 | -| DR: no add in the panel yet | -0.679 | -| Airplay Top-20: last new song 13-25 weeks ago | +0.478 | -| Airplay Top-20: 0.5-1.25x own usual gap | -0.177 | -| Airplay Top-20: last new song 2+ years ago | -0.035 | -| log archive plays per 100 sampled hours (2012-21) | +0.583 | -| log DR lists with the artist before t (2023-) | +0.192 | -| DR list recency 26w | -0.900 | -| DR lists known before t at all | +0.491 | -| NMF Denmark in the window | +0.886 | -| NMF Denmark top 10 in the window | +0.810 | -| log NMF Denmark placements last 52w | -0.024 | -| share of releases on NMF Denmark (2 years) | -0.468 | -| season index of the window (log, IFPI 2002-24) | +0.414 | +| discography known | +0.000 | +| release recency 4w | +0.243 | +| release recency 12w | -0.814 | +| log releases last 52w | -0.224 | +| releases in the window | +0.000 | +| releases in window x not in rotation now | +0.476 | +| DR: weeks since last add 13-25 | -0.054 | +| DR: last add 26+ weeks ago | +0.497 | +| DR: no add in the panel yet | -0.684 | +| Airplay Top-20: last new song 13-25 weeks ago | +0.463 | +| Airplay Top-20: 0.5-1.25x own usual gap | -0.144 | +| Airplay Top-20: last new song 2+ years ago | -0.107 | +| log archive plays per 100 sampled hours (2012-21) | +0.532 | +| log DR lists with the artist before t (2023-) | +0.108 | +| DR list recency 26w | -0.955 | +| DR lists known before t at all | +0.572 | +| NMF Denmark in the window | +0.866 | +| NMF Denmark top 10 in the window | +0.788 | +| log NMF Denmark placements last 52w | -0.047 | +| share of releases on NMF Denmark (2 years) | -0.354 | +| season index of the window (log, IFPI 2002-24) | +0.372 | diff --git a/src/main.rs b/src/main.rs index c878e58..53d84c5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,6 @@ //! airplay: collects what Danish radio stations play, for modelling when artists get played again. //! -//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive] [--backfill] +//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive|oldpages] [--backfill] //! airplay export nmf-denmark [--out data/nmf_denmark.csv] //! airplay stats [--db data/airplay.db] //! airplay check [--db data/airplay.db] gaps, duplicates, DR cross-check, chart completeness @@ -22,6 +22,7 @@ mod hitlisten; mod http; mod model; mod musicbrainz; +mod oldpages; mod playlists; mod rotations; mod spotify; @@ -61,7 +62,7 @@ fn main() -> Result<()> { rotations::store_all(&db, &dir, args.iter().any(|a| a == "--reparse")) } _ => { - eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive] [--backfill] + eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf|nmf-artists|nmf-charts|nmf-tracks|archive|oldpages] [--backfill] //! airplay export nmf-denmark [--out data/nmf_denmark.csv]\n airplay stats [--db FILE]\n airplay check [--db FILE]\n airplay model panel|backtest|rank\n airplay serve [--addr HOST:PORT] [--prefix /airplay]"); std::process::exit(2); } @@ -140,6 +141,11 @@ fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> { let r = playlists::nmf_artists(db, &sfc_http, if backfill { usize::MAX } else { 800 }, &mut log); step("new music friday per artist (superfridaychart)", r.map(|n| format!("{n} artists looked up")), &mut log); } + if only.as_deref() == Some("oldpages") { + // Wayback copies of DR's old artist pages: 53 weeks of plays each (only on request; long, once). + let r = oldpages::collect(db, &http::Http::new(1500)?, &mut log); + step("old dr artist pages (wayback)", r.map(|n| format!("{n} pages read")), &mut log); + } if only.as_deref() == Some("archive") { // Wayback Machine copies of old DR playlists (only on request: about an hour, once). let r = archive::collect(db, &http::Http::new(1500)?, &mut log); diff --git a/src/model/adds.rs b/src/model/adds.rs index 8399dbb..89ab931 100644 --- a/src/model/adds.rs +++ b/src/model/adds.rs @@ -42,7 +42,9 @@ pub const WAVE_NAMES: &[&str] = &["DR: weeks since last add 13-25", "DR: last ad /// and the IFPI wave position. pub fn wave_feats(p: &Panel, r: &Rot, a: usize, t: usize, shift: usize) -> [f64; WAVE] { if off("wave") { return [0.0; WAVE]; } - let since = (0..=t).find(|&i| r.add_any(a, t - i)).map(|s| s + shift); + // An add in week w is only confirmed in week w + 1 (the surge rule reads the following week), + // so as of week t only adds up to t - 1 are known. + let since = (1..=t).find(|&i| r.add_any(a, t - i)).map(|s| s + shift); let ch = p.chart(a, t); let f = |b: bool| b as i32 as f64; let air = ch.air_since as f64 + shift as f64; @@ -374,7 +376,9 @@ fn platt(ab: (f64, f64), p: f64) -> f64 { glm::sigmoid(ab.0 + ab.1 * logit(p)).c fn origins(h: usize, last_week: usize) -> std::ops::RangeInclusive { origins_w(h, last_week, W) } fn origins_w(h: usize, last_week: usize, win: usize) -> std::ops::RangeInclusive { - let hi = last_week.saturating_sub(h + win); + // The label's last week needs the following week too (surge rule): the window must close at + // least one week before `last_week`. + let hi = last_week.saturating_sub(h + win + 1); LAGS - 1..=hi } @@ -394,7 +398,7 @@ impl AddModel { fn fit_with(p: &Panel, r: &Rot, h: usize, last_week: usize, events: bool, win: usize) -> AddModel { let mut m = Self::fit_raw(p, r, h, last_week, events, win); - let hi = last_week.saturating_sub(h + win); + let hi = last_week.saturating_sub(h + win + 1); if hi + 1 >= CAL_ORIGINS + LAGS { let c0 = hi + 1 - CAL_ORIGINS; // the inner model may only use windows that closed before the first calibration origin @@ -516,11 +520,11 @@ pub fn backtest(p: &Panel, tau: &[u32; C], horizons: &[usize], start: usize, md: let last_origin = p.w() - 1 - h - W; if last_origin < start { continue; } // [model][week] log-loss sums; models: base(rot now), base(popularity bins), logit - let names = ["rate by in-rotation-now", "rate by last-12-weeks level", "logit"]; - let mut wk_c: Vec<[f64; 3]> = Vec::new(); - let mut wk_a: Vec<[f64; 3]> = Vec::new(); - let mut pairs_c: [Vec<(f64, bool)>; 3] = Default::default(); - let mut pairs_a: [Vec<(f64, bool)>; 3] = Default::default(); + let names = ["rate by in-rotation-now", "rate by last-12-weeks level", "logit (pre-release: NMF from history)", "logit (post-release: NMF known)"]; + let mut wk_c: Vec<[f64; 4]> = Vec::new(); + let mut wk_a: Vec<[f64; 4]> = Vec::new(); + let mut pairs_c: [Vec<(f64, bool)>; 4] = Default::default(); + let mut pairs_a: [Vec<(f64, bool)>; 4] = Default::default(); let mut calib = [[0.0f64; 3]; 10]; let (mut nc, mut na, mut hits_c, mut hits_a) = (0.0, 0.0, 0.0, 0.0); for t0 in start..=last_origin { @@ -548,25 +552,31 @@ pub fn backtest(p: &Panel, tau: &[u32; C], horizons: &[usize], start: usize, md: let b = lvl(tot); any_lvl[b][0] += y; any_lvl[b][1] += 1.0; } } - let (mut lc, mut la) = ([0.0; 3], [0.0; 3]); + let (mut lc, mut la) = ([0.0; 4], [0.0; 4]); for a in 0..p.a() { - let (pc, pa) = m.predict(p, &r, a, t0, Scenario::ACTUAL, None); + // Pre-release: the window's NMF placement isn't known yet; blend by the artist's NMF rate. + let q = p_nmf(p, a, t0); + let (pcn, pan) = m.predict(p, &r, a, t0, Scenario { nmf: Some(true), ..Scenario::ACTUAL }, None); + let (pcr, par) = m.predict(p, &r, a, t0, Scenario { nmf: Some(false), ..Scenario::ACTUAL }, None); + let pc: [f64; C] = std::array::from_fn(|c| q * pcn[c] + (1.0 - q) * pcr[c]); + let pa = q * pan + (1.0 - q) * par; + let (pcx, pax) = m.predict(p, &r, a, t0, Scenario::ACTUAL, None); let mut tot = 0.0; for c in 0..C { let y = r.add_in_window(a, c, t0, h); let on = r.on(a, c, t0) as usize; let m12 = (0..LAGS).map(|i| p.y(a, c, t0 - i)).sum::() / LAGS as f64; tot += m12; - let preds = [by_rot[on][0] / by_rot[on][1], by_lvl[lvl(m12)][0] / by_lvl[lvl(m12)][1], pc[c]]; - for k in 0..3 { lc[k] += ll(preds[k], y); pairs_c[k].push((preds[k], y)); } + let preds = [by_rot[on][0] / by_rot[on][1], by_lvl[lvl(m12)][0] / by_lvl[lvl(m12)][1], pc[c], pcx[c]]; + for k in 0..4 { lc[k] += ll(preds[k], y); pairs_c[k].push((preds[k], y)); } let d = ((pc[c] * 10.0) as usize).min(9); calib[d][0] += pc[c]; calib[d][1] += y as i32 as f64; calib[d][2] += 1.0; nc += 1.0; hits_c += y as i32 as f64; } let y = r.add_any_in_window(a, t0, h); let on = r.any(a, t0) as usize; - let preds = [any_rot[on][0] / any_rot[on][1], any_lvl[lvl(tot)][0] / any_lvl[lvl(tot)][1], pa]; - for k in 0..3 { la[k] += ll(preds[k], y); pairs_a[k].push((preds[k], y)); } + let preds = [any_rot[on][0] / any_rot[on][1], any_lvl[lvl(tot)][0] / any_lvl[lvl(tot)][1], pa, pax]; + for k in 0..4 { la[k] += ll(preds[k], y); pairs_a[k].push((preds[k], y)); } na += 1.0; hits_a += y as i32 as f64; } wk_c.push(lc); wk_a.push(la); @@ -576,7 +586,7 @@ pub fn backtest(p: &Panel, tau: &[u32; C], horizons: &[usize], start: usize, md: writeln!(md, "## Window starting h = {h} weeks ahead ({nwk} forecast origins)\n")?; writeln!(md, "Base rates: per channel {:.2}% of artist-channels get an add in the window; on any channel {:.1}% of artists.\n", 100.0 * hits_c / nc, 100.0 * hits_a / na)?; writeln!(md, "| Model | Log-loss, per channel | PR-AUC, per channel | Log-loss, any channel | PR-AUC, any channel |\n|---|---|---|---|---|")?; - for k in 0..3 { + for k in 0..4 { let (sc, sa) = (wk_c.iter().map(|w| w[k]).sum::() / nc, wk_a.iter().map(|w| w[k]).sum::() / na); let (ac, aa) = (average_precision(pairs_c[k].clone()), average_precision(pairs_a[k].clone())); for (metric, v) in [("add_logloss", sc), ("add_prauc", ac), ("addany_logloss", sa), ("addany_prauc", aa)] { @@ -584,7 +594,7 @@ pub fn backtest(p: &Panel, tau: &[u32; C], horizons: &[usize], start: usize, md: } writeln!(md, "| {} | {sc:.4} | {ac:.3} | {sa:.4} | {aa:.3} |", names[k])?; } - let col = |v: &Vec<[f64; 3]>, k: usize| v.iter().map(|w| w[k]).collect::>(); + let col = |v: &Vec<[f64; 4]>, k: usize| v.iter().map(|w| w[k]).collect::>(); let (s1, s2) = (skill(&col(&wk_c, 2), &col(&wk_c, 0)), skill(&col(&wk_c, 2), &col(&wk_c, 1))); let (s3, s4) = (skill(&col(&wk_a, 2), &col(&wk_a, 0)), skill(&col(&wk_a, 2), &col(&wk_a, 1))); rows.push(("logit".into(), h, "add_skill_vs_level".into(), s2.0)); @@ -616,11 +626,11 @@ pub fn backtest(p: &Panel, tau: &[u32; C], horizons: &[usize], start: usize, md: pub fn backtest_events(p: &Panel, tau: &[u32; C], start: usize, md: &mut String) -> Result> { let r = Rot::new(p, tau); let mut rows = Vec::new(); - let names = ["rate by in-rotation-now", "rate by last-12-weeks level", "event model"]; - let mut wk_any: Vec<[f64; 3]> = Vec::new(); - let mut wk_ch: Vec<[f64; 3]> = Vec::new(); - let mut pairs_any: [Vec<(f64, bool)>; 3] = Default::default(); - let mut pairs_ch: [Vec<(f64, bool)>; 3] = Default::default(); + let names = ["rate by in-rotation-now", "rate by last-12-weeks level", "event model (pre-release: NMF from history)", "event model (post-release: NMF known)"]; + let mut wk_any: Vec<[f64; 4]> = Vec::new(); + let mut wk_ch: Vec<[f64; 4]> = Vec::new(); + let mut pairs_any: [Vec<(f64, bool)>; 4] = Default::default(); + let mut pairs_ch: [Vec<(f64, bool)>; 4] = Default::default(); let (mut n_ev, mut n_hit) = (0usize, 0usize); let lvl = |m: f64| ((m.ln_1p() / 0.5) as usize).min(7); let level = |a: usize, t: usize| (0..LAGS).map(|i| p.group_total(a, t - i, None)).sum::() / LAGS as f64; @@ -646,7 +656,7 @@ pub fn backtest_events(p: &Panel, tau: &[u32; C], start: usize, md: &mut String) } } } - let (mut la, mut lc) = ([0.0; 3], [0.0; 3]); + let (mut la, mut lc) = ([0.0; 4], [0.0; 4]); let mut any_events = false; for a in 0..p.a() { if !clean_release(p, a, t0 + 1) { continue; } @@ -655,18 +665,19 @@ pub fn backtest_events(p: &Panel, tau: &[u32; C], start: usize, md: &mut String) let (pcn, pan) = m.predict(p, &r, a, t0, Scenario::new(true, true), None); let (pcr, par) = m.predict(p, &r, a, t0, Scenario::new(true, false), None); let pa = q * pan + (1.0 - q) * par; + let (pcx, pax) = m.predict(p, &r, a, t0, Scenario::ACTUAL, None); let y = r.add_any_in_window(a, t0, 0); let on = r.any(a, t0) as usize; - let preds = [by_rot[on][0] / by_rot[on][1], by_lvl[lvl(level(a, t0))][0] / by_lvl[lvl(level(a, t0))][1], pa]; - for k in 0..3 { la[k] += ll(preds[k], y); pairs_any[k].push((preds[k], y)); } + let preds = [by_rot[on][0] / by_rot[on][1], by_lvl[lvl(level(a, t0))][0] / by_lvl[lvl(level(a, t0))][1], pa, pax]; + for k in 0..4 { la[k] += ll(preds[k], y); pairs_any[k].push((preds[k], y)); } n_ev += 1; n_hit += y as usize; for c in 0..C { let pc = q * pcn[c] + (1.0 - q) * pcr[c]; let yc = r.add_in_window(a, c, t0, 0); let onc = r.on(a, c, t0) as usize; let bc = lvl((0..LAGS).map(|i| p.y(a, c, t0 - i)).sum::() / LAGS as f64); - let pr = [cr[onc][0] / cr[onc][1], cl[bc][0] / cl[bc][1], pc]; - for k in 0..3 { lc[k] += ll(pr[k], yc); pairs_ch[k].push((pr[k], yc)); } + let pr = [cr[onc][0] / cr[onc][1], cl[bc][0] / cl[bc][1], pc, pcx[c]]; + for k in 0..4 { lc[k] += ll(pr[k], yc); pairs_ch[k].push((pr[k], yc)); } } } if any_events { wk_any.push(la); wk_ch.push(lc); } @@ -677,13 +688,13 @@ pub fn backtest_events(p: &Panel, tau: &[u32; C], start: usize, md: &mut String) 100.0 * n_hit as f64 / n_ev.max(1) as f64)?; writeln!(md, "| Model | Log-loss, any channel | PR-AUC, any channel | Log-loss, per channel | PR-AUC, per channel |\n|---|---|---|---|---|")?; let nch = (n_ev * C).max(1) as f64; - for k in 0..3 { + for k in 0..4 { let (a, c) = (wk_any.iter().map(|w| w[k]).sum::() / n_ev.max(1) as f64, wk_ch.iter().map(|w| w[k]).sum::() / nch); let (pa, pc) = (average_precision(pairs_any[k].clone()), average_precision(pairs_ch[k].clone())); writeln!(md, "| {} | {a:.4} | {pa:.3} | {c:.4} | {pc:.3} |", names[k])?; for (metric, v) in [("event_logloss_any", a), ("event_prauc_any", pa), ("event_logloss_ch", c), ("event_prauc_ch", pc)] { rows.push((names[k].to_string(), 0, metric.to_string(), v)); } } - let col = |v: &Vec<[f64; 3]>, k: usize| v.iter().map(|w| w[k]).collect::>(); + let col = |v: &Vec<[f64; 4]>, k: usize| v.iter().map(|w| w[k]).collect::>(); let (s1, s2) = (skill(&col(&wk_any, 2), &col(&wk_any, 0)), skill(&col(&wk_any, 2), &col(&wk_any, 1))); rows.push(("event model".into(), 0, "event_skill_any_vs_level".into(), s2.0)); writeln!(md, "\nSkill of the event model, any channel: {:.3} ({:.3} to {:.3}) vs in-rotation-now, {:.3} ({:.3} to {:.3}) vs last-12-weeks level.\n", s1.0, s1.1, s1.2, s2.0, s2.1, s2.2)?; diff --git a/src/model/leakcheck.rs b/src/model/leakcheck.rs new file mode 100644 index 0000000..6b2fa74 --- /dev/null +++ b/src/model/leakcheck.rs @@ -0,0 +1,46 @@ +//! Leak test: every feature must be computable from what was known at the end of the origin week. +//! Rebuilds the feature rows from a copy of the data with everything after week t removed and +//! reports any feature that changes (`airplay model leakcheck`). + +use super::adds::{row_any, row_channel, Rot, Scenario, ANY_NAMES, AFF_NAMES, EXTRA, LONG_NAMES, NMF_NAMES, PA, PC, REL_NAMES, SEASON_NAMES, WAVE_NAMES}; +use super::common::thresholds; +use super::features::{self, LAGS}; +use super::panel::{Panel, C}; +use anyhow::Result; +use std::fmt::Write as _; + +pub fn run(p: &Panel) -> Result { + let tau = thresholds(p, p.w()); + let r_full = Rot::new(p, &tau); + let names_c: Vec<&str> = features::NAMES.iter().chain(EXTRA).chain(REL_NAMES).chain(WAVE_NAMES).chain(LONG_NAMES).chain(NMF_NAMES).chain(SEASON_NAMES).chain(AFF_NAMES).copied().collect(); + let names_a: Vec<&str> = ANY_NAMES.iter().chain(REL_NAMES).chain(WAVE_NAMES).chain(LONG_NAMES).chain(NMF_NAMES).chain(SEASON_NAMES).copied().collect(); + let mut bad_c = vec![0usize; PC]; + let mut bad_a = vec![0usize; PA]; + let mut rows = 0; + // A fixed scenario (release, no NMF): the window variables are inputs, not data. + let sc = Scenario::new(true, false); + for t in [LAGS + 3, 20, 28, 36, p.w() - 10] { + let q = p.truncated(t); + let r_trunc = Rot::new(&q, &tau); + let (mut x1, mut x2) = (vec![0.0; PC], vec![0.0; PC]); + let (mut a1, mut a2) = (vec![0.0; PA], vec![0.0; PA]); + for a in (0..p.a()).step_by(5) { + for c in 0..C { + row_channel(p, &r_full, a, c, t, 0, sc, &mut x1); + row_channel(&q, &r_trunc, a, c, t, 0, sc, &mut x2); + for k in 0..PC { if (x1[k] - x2[k]).abs() > 1e-9 { bad_c[k] += 1; } } + rows += 1; + } + row_any(p, &r_full, a, t, 0, sc, &mut a1); + row_any(&q, &r_trunc, a, t, 0, sc, &mut a2); + for k in 0..PA { if (a1[k] - a2[k]).abs() > 1e-9 { bad_a[k] += 1; } } + } + } + let mut md = String::new(); + writeln!(md, "# Leak test\n\n{rows} per-channel rows at 5 origins: features rebuilt from data cut at the origin week.\n")?; + let leaks: Vec = bad_c.iter().enumerate().filter(|(_, n)| **n > 0).map(|(k, n)| format!("per-channel \"{}\": {n} rows differ", names_c.get(k).unwrap_or(&"?"))) + .chain(bad_a.iter().enumerate().filter(|(_, n)| **n > 0).map(|(k, n)| format!("any-channel \"{}\": {n} rows differ", names_a.get(k).unwrap_or(&"?")))).collect(); + if leaks.is_empty() { writeln!(md, "No feature uses data from after the origin.")?; } + else { writeln!(md, "Features that change when the future is removed (leaks):\n")?; for l in leaks { writeln!(md, "- {l}")?; } } + Ok(md) +} diff --git a/src/model/mod.rs b/src/model/mod.rs index cdf8eca..e03cdcc 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -20,6 +20,7 @@ pub mod features; pub mod glm; pub mod ifpi; pub mod kalman; +pub mod leakcheck; pub mod nmf; pub mod panel; pub mod rank; @@ -84,6 +85,7 @@ pub fn run(db: &Db, args: &[String]) -> Result<()> { if let Some(out) = arg(args, "--out") { std::fs::write(out, &md)?; } print!("{md}"); } + Some("leakcheck") => { print!("{}", leakcheck::run(&p)?); } Some("addcheck") => { if let Some(r) = arg(args, "--rule") { let _ = adds::ADD_RULE.set(r); } print!("{}", addcheck::run(db, &p)?); diff --git a/src/model/panel.rs b/src/model/panel.rs index a6e3a2e..e42f0fd 100644 --- a/src/model/panel.rs +++ b/src/model/panel.rs @@ -38,6 +38,7 @@ impl Panel { } } +#[derive(Clone)] pub struct Artist { pub id: String, pub name: String, @@ -65,6 +66,7 @@ pub struct Chart { pub air_gap: u16, } +#[derive(Clone)] pub struct Panel { /// Monday of each complete week, oldest first. pub weeks: Vec, @@ -100,6 +102,26 @@ impl Panel { #[inline] pub fn chart(&self, a: usize, w: usize) -> Chart { self.chart[a * self.weeks.len() + w] } + /// The panel as it was known at the end of week t: plays after t removed (and taken out of the + /// all-time counts), releases, NMF placements and lists after t dropped. For the leak test. + pub fn truncated(&self, t: usize) -> Panel { + let mut q = self.clone(); + let nw = self.w(); + let end_t = self.weeks[t] + Duration::days(6); + for a in 0..self.a() { + for g in 0..GROUPS { + let cs: Vec = (0..C).filter(|&c| group(c) == g).collect(); + let after: f64 = (t + 1..nw).map(|w| cs.iter().map(|&c| self.y(a, c, w)).sum::() / cs.len() as f64).sum(); + q.alltime[a][g] = (self.alltime[a][g] - after).max(0.0); + } + for c in 0..C { for w in t + 1..nw { q.y[(a * C + c) * nw + w] = 0.0; } } + if let Some(r) = q.releases[a].as_mut() { r.retain(|d| *d <= end_t); } + q.nmf_dk[a].retain(|(d, _)| *d <= end_t); + q.listed_on[a].retain(|d| *d <= end_t); + } + q + } + pub fn load(db: &Db) -> Result { // Weeks: every Monday in the artist pages whose week had ended before the earliest page was read. let first_read: String = db.conn.query_row("SELECT min(fetched_at) FROM dr_artists", [], |r| r.get(0))?; diff --git a/src/oldpages.rs b/src/oldpages.rs new file mode 100644 index 0000000..2e1e665 --- /dev/null +++ b/src/oldpages.rs @@ -0,0 +1,194 @@ +//! DR's old artist pages (dr.dk/musik/kunstner/{slug}/{id}, about 2019–2024) from the Wayback +//! Machine. Each page embeds the artist's last 53 weeks of DR plays (all channels together) plus +//! profile and top tracks, so several captures per artist rebuild years of weekly history. +//! +//! airplay collect --only oldpages +//! +//! Captures are listed from the CDX index (one a month per page at most); per artist the oldest is +//! taken, then the next at least 44 weeks later, and the latest, so the 53-week windows tile the +//! history with some overlap. Later captures win where windows overlap. + +use crate::{db::Db, http::Http}; +use anyhow::Result; +use chrono::{Datelike, Duration, NaiveDate, Utc}; +use regex::Regex; +use rusqlite::params; +use serde_json::Value; +use std::collections::BTreeMap; + +pub fn create_tables(db: &Db) -> Result<()> { + db.conn.execute_batch( + "CREATE TABLE IF NOT EXISTS old_artist_captures ( + artist_id TEXT NOT NULL, snapshot TEXT NOT NULL, url TEXT NOT NULL, + chosen INTEGER NOT NULL DEFAULT 0, fetched_at TEXT, weeks INTEGER, + PRIMARY KEY (artist_id, snapshot) + ); + CREATE TABLE IF NOT EXISTS old_cdx_progress (key TEXT PRIMARY KEY, value TEXT); + CREATE TABLE IF NOT EXISTS dr_artist_history ( + artist_id TEXT NOT NULL, week_start TEXT NOT NULL, -- Monday of the week + plays INTEGER NOT NULL, -- all DR channels together + capture TEXT NOT NULL, -- Wayback timestamp it came from + PRIMARY KEY (artist_id, week_start) + ); + CREATE TABLE IF NOT EXISTS old_artist_meta ( + artist_id TEXT PRIMARY KEY, name TEXT, country TEXT, description TEXT, gender TEXT, artist_type TEXT, + capture TEXT NOT NULL, top_tracks TEXT + );", + )?; + Ok(()) +} + +/// JavaScript `unescape`: %XX is a Latin-1 code point, %uXXXX a UTF-16 unit. +pub fn js_unescape(s: &str) -> String { + let b = s.as_bytes(); + let mut units: Vec = Vec::with_capacity(b.len()); + let mut i = 0; + let hex = |x: &[u8]| std::str::from_utf8(x).ok().and_then(|h| u16::from_str_radix(h, 16).ok()); + while i < b.len() { + if b[i] == b'%' { + if i + 6 <= b.len() && b[i + 1] == b'u' { + if let Some(v) = hex(&b[i + 2..i + 6]) { units.push(v); i += 6; continue; } + } + if i + 3 <= b.len() { + if let Some(v) = hex(&b[i + 1..i + 3]) { units.push(v); i += 3; continue; } + } + } + units.push(b[i] as u16); + i += 1; + } + String::from_utf16_lossy(&units) +} + +/// The embedded page data, if any. +pub fn page_data(html: &str) -> Option { + let start = html.find("JSON.parse(unescape('")? + "JSON.parse(unescape('".len(); + let end = start + html[start..].find("'))")?; + serde_json::from_str(&js_unescape(&html[start..end])).ok() +} + +fn monday(d: NaiveDate) -> NaiveDate { d - Duration::days(d.weekday().num_days_from_monday() as i64) } + +/// Lists every capture of the old artist pages into old_artist_captures (resumable). +fn list_captures(db: &Db, http: &Http, log: &mut Vec) -> Result { + let id_re = Regex::new(r"/musik/kunstner/[^/?#]+/(\d+)")?; + let mut resume: Option = db.conn.query_row("SELECT value FROM old_cdx_progress WHERE key = 'resume'", [], |r| r.get(0)).ok(); + let done: bool = db.conn.query_row("SELECT count(*) FROM old_cdx_progress WHERE key = 'complete'", [], |r| r.get::<_, i64>(0))? > 0; + if done { return Ok(0); } + let mut n = 0; + loop { + let mut url = "https://web.archive.org/cdx/search/cdx?url=www.dr.dk/musik/kunstner/*&output=json&fl=timestamp,original&filter=statuscode:200&collapse=timestamp:6&limit=20000&showResumeKey=true".to_string(); + if let Some(k) = &resume { url.push_str(&format!("&resumeKey={}", k)); } + let mut body = None; + for attempt in 0..10 { + match http.get(&url) { + Ok(Some(b)) if b.trim_start().starts_with('[') => { body = Some(b); break; } + Ok(_) | Err(_) => { if attempt == 9 { log.push("old pages: CDX listing failed".into()); } } + } + std::thread::sleep(std::time::Duration::from_secs(30)); + } + let Some(body) = body else { return Ok(n) }; + let rows: Vec> = serde_json::from_str(&body).unwrap_or_default(); + let mut next: Option = None; + let tx = db.conn.unchecked_transaction()?; + for (k, row) in rows.iter().enumerate().skip(1) { + match row.len() { + 0 => { if let Some(r) = rows.get(k + 1).and_then(|r| r.first()) { next = Some(r.clone()); } break; } + _ if row.len() >= 2 => { + if let Some(c) = id_re.captures(&row[1]) { + tx.execute("INSERT OR IGNORE INTO old_artist_captures (artist_id, snapshot, url) VALUES (?1, ?2, ?3)", params![&c[1], &row[0], &row[1]])?; + n += 1; + } + } + _ => {} + } + } + match &next { + Some(k) => { tx.execute("INSERT OR REPLACE INTO old_cdx_progress VALUES ('resume', ?1)", [k])?; } + None => { tx.execute("INSERT OR REPLACE INTO old_cdx_progress VALUES ('complete', '1')", [])?; } + } + tx.commit()?; + eprintln!("old pages: {n} captures listed"); + match next { Some(k) => resume = Some(urlencode(&k)), None => break } + } + Ok(n) +} + +fn urlencode(s: &str) -> String { + s.bytes().map(|b| match b { b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => (b as char).to_string(), _ => format!("%{b:02X}") }).collect() +} + +/// Per artist: the oldest capture, then each next one at least 44 weeks later, and the latest. +fn choose(db: &Db) -> Result<()> { + let mut by: BTreeMap> = BTreeMap::new(); + for r in db.conn.prepare("SELECT artist_id, snapshot FROM old_artist_captures ORDER BY artist_id, snapshot")?.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? { + let (a, s) = r?; + by.entry(a).or_default().push(s); + } + let date = |s: &str| NaiveDate::parse_from_str(&s[..8], "%Y%m%d").ok(); + let tx = db.conn.unchecked_transaction()?; + for (a, snaps) in by { + let mut last: Option = None; + for s in &snaps { + let Some(d) = date(s) else { continue }; + if last.is_none_or(|l| (d - l).num_weeks() >= 44) { + tx.execute("UPDATE old_artist_captures SET chosen = 1 WHERE artist_id = ?1 AND snapshot = ?2", params![a, s])?; + last = Some(d); + } + } + if let Some(s) = snaps.last() { tx.execute("UPDATE old_artist_captures SET chosen = 1 WHERE artist_id = ?1 AND snapshot = ?2", params![a, s])?; } + } + tx.commit()?; + Ok(()) +} + +pub fn collect(db: &Db, http: &Http, log: &mut Vec) -> Result { + create_tables(db)?; + list_captures(db, http, log)?; + choose(db)?; + // Artists in the pool first, then the rest; oldest captures first within each. + let todo: Vec<(String, String, String)> = db.conn.prepare( + "SELECT c.artist_id, c.snapshot, c.url FROM old_artist_captures c LEFT JOIN dr_artists a ON a.artist_id = c.artist_id + WHERE c.chosen = 1 AND c.fetched_at IS NULL ORDER BY a.artist_id IS NULL, c.artist_id, c.snapshot")? + .query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?.collect::>()?; + eprintln!("old pages: {} captures to read", todo.len()); + let mut pages = 0; + for (a, snap, url) in todo { + let html = match http.get(&format!("https://web.archive.org/web/{snap}id_/{url}")) { + Ok(h) => h.unwrap_or_default(), + Err(e) => { log.push(format!("old page {url}: {e}")); continue; } + }; + if html.is_empty() || html.contains("Internet Archive: Temporarily Offline") { std::thread::sleep(std::time::Duration::from_secs(60)); continue; } + let data = page_data(&html); + let mut n = 0; + let tx = db.conn.unchecked_transaction()?; + if let Some(v) = &data { + let m = &v["musicArtist"]; + for w in m["weeks"].as_array().into_iter().flatten() { + let (Some(d), Some(p)) = (w["date"].as_str().and_then(|s| NaiveDate::parse_from_str(s, "%Y-%m-%d").ok()), w["plays"].as_i64()) else { continue }; + // Later captures win where windows overlap. + tx.execute("INSERT INTO dr_artist_history VALUES (?1, ?2, ?3, ?4) + ON CONFLICT (artist_id, week_start) DO UPDATE SET plays = excluded.plays, capture = excluded.capture WHERE excluded.capture > capture", + params![a, monday(d).to_string(), p, snap])?; + n += 1; + } + let tops: Vec = m["topTracks"].as_array().into_iter().flatten().map(|t| serde_json::json!({"title": t["track"]["title"], "plays": t["track"]["playCount"]})).collect(); + tx.execute("INSERT INTO old_artist_meta VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) + ON CONFLICT (artist_id) DO UPDATE SET name = excluded.name, country = excluded.country, description = excluded.description, + gender = excluded.gender, artist_type = excluded.artist_type, capture = excluded.capture, top_tracks = excluded.top_tracks WHERE excluded.capture > capture", + params![a, m["name"].as_str(), m["country"].as_str(), m["description"].as_str(), m["gender"].as_str(), m["artistType"].as_str(), snap, Value::Array(tops).to_string()])?; + } + tx.execute("UPDATE old_artist_captures SET fetched_at = ?1, weeks = ?2 WHERE artist_id = ?3 AND snapshot = ?4", params![Utc::now().to_rfc3339(), n as i64, a, snap])?; + tx.commit()?; + pages += 1; + if pages % 200 == 0 { eprintln!("old pages: {pages} read"); } + } + Ok(pages) +} + +#[cfg(test)] +mod tests { + #[test] + fn unescape_latin1_and_utf16() { + assert_eq!(super::js_unescape("%C1sd%EDs%20%u00F0x"), "Ásdís ðx"); + } +} diff --git a/src/web/index.html b/src/web/index.html index 13ac91d..44e5b4d 100644 --- a/src/web/index.html +++ b/src/web/index.html @@ -186,8 +186,7 @@ svg text { fill: var(--muted); font-size: 11px; }

DR publishes plays per artist, not per song, so "new add" can't yet tell a new song from an old one coming back. In the one week of song-level plays we have, only 3–12% of artists in rotation on a station had two songs in rotation at once, so "drops out first, then gets added" is the usual pattern. The weekly rotation lists (song level) will sharpen this.

-

Backtest: new adds

-

Walk-forward: for each week from week 27, the models are fitted only on what was known at the time and forecast 1, 4 and 8 weeks ahead. "Adds" are rows where the artist was not in rotation at the time of the forecast, which is the hard and interesting part. Skill above 0 means better than the baseline.

+

Backtest

20 years of IFPI charts

@@ -432,18 +431,17 @@ function whatIfUI(a, h) { // ---------- model tab ---------- function renderModel() { const sc = meta.backtest || []; - const get = (m, h, k) => sc.find(x => x.model === m && x.h === h && x.metric === k)?.value; - const hs = [...new Set(sc.filter(x => x.metric.startsWith('add')).map(x => x.h))].sort((a, b) => a - b); - const ms = [['rate by in-rotation-now', 'Rate by "in rotation now"'], ['rate by last-12-weeks level', 'Rate by last-12-weeks level'], ['logit', 'Model']]; + const get = (m, k) => sc.find(x => x.model === m && x.metric === k)?.value; + const ms = [['rate by in-rotation-now', 'Rate by "in rotation now"'], ['rate by last-12-weeks level', 'Rate by last-12-weeks airplay'], + ['event model (pre-release: NMF from history)', 'Model, before release (as on this site)'], ['event model (post-release: NMF known)', 'Model, NMF placement known']]; const f = (v, d) => v == null || isNaN(v) ? '—' : v.toFixed(d); - $('cards').innerHTML = hs.map(h => { - const sk = get('logit', h, 'addany_skill_vs_level'), skc = get('logit', h, 'add_skill_vs_level'); - return `

Window starting ${h ? h + ' weeks ahead' : 'now'}

+ const sk = get('event model', 'event_skill_any_vs_level'); + $('cards').innerHTML = sc.some(x => x.metric.startsWith('event_')) ? `

If an artist releases a new song, does DR add it within 8 weeks?

- ${ms.map(([m, l]) => ``).join('')} + ${ms.map(([m, l]) => ``).join('')}
Any station
log-loss
Any station
PR-AUC
Per station
PR-AUC
${l}${f(get(m, h, 'addany_logloss'), 3)}${f(get(m, h, 'addany_prauc'), 3)}${f(get(m, h, 'add_prauc'), 3)}
${l}${f(get(m, 'event_logloss_any'), 3)}${f(get(m, 'event_prauc_any'), 3)}${f(get(m, 'event_prauc_ch'), 3)}
-

Skill vs the last-12-weeks baseline: ${sk > 0 ? '+' : ''}${f(sk, 3)} any station, ${skc > 0 ? '+' : ''}${f(skc, 3)} per station. Lower log-loss and higher PR-AUC are better.

`; - }).join('') || '

No add backtest stored yet (`airplay model adds`).

'; +

Walk-forward over the past year: each week's releases scored by a model fitted only on releases whose outcome was already known. Skill of the before-release model vs last-12-weeks airplay: ${sk > 0 ? '+' : ''}${f(sk, 3)} (any station). Lower log-loss and higher PR-AUC are better; a leak test confirms no feature uses data from after the forecast week.

` + : '

No backtest stored yet (`airplay model events`).

'; } // ---------- wiring ----------