From 33a3c3f2e5ad10abbc39a9bac8924e564ebe703b Mon Sep 17 00:00:00 2001 From: moritsbw-cpu Date: Sun, 15 Nov 2026 03:05:05 +0000 Subject: [PATCH] Daily job parses new rotation lists and refits thresholds before ranking; fix backfill wait loop (pgrep -f matched its parent shell); NMF runs inside backfill.sh Co-Authored-By: Claude Opus 5.5 --- deploy/airplay-collect.service | 4 +- docs/BACKTEST.md | 185 ++++++++++++++++----------------- scripts/after-backfill-nmf.sh | 6 -- scripts/backfill.sh | 3 +- src/model/mod.rs | 5 +- 5 files changed, 100 insertions(+), 103 deletions(-) delete mode 100755 scripts/after-backfill-nmf.sh diff --git a/deploy/airplay-collect.service b/deploy/airplay-collect.service index c2f6f6f..a738b06 100644 --- a/deploy/airplay-collect.service +++ b/deploy/airplay-collect.service @@ -8,7 +8,9 @@ Wants=network-online.target Type=oneshot WorkingDirectory=/root/dev/Airplay-Predictor ExecStart=/root/dev/Airplay-Predictor/target/release/airplay collect --db /root/dev/Airplay-Predictor/data/airplay.db -# Fresh forecasts for the playground after every collection. +# After collecting: parse a new rotation list, refit the thresholds to the lists, fresh forecasts for the site. +ExecStartPost=/root/dev/Airplay-Predictor/target/release/airplay rotations --db /root/dev/Airplay-Predictor/data/airplay.db +ExecStartPost=/root/dev/Airplay-Predictor/target/release/airplay model calibrate --db /root/dev/Airplay-Predictor/data/airplay.db ExecStartPost=/root/dev/Airplay-Predictor/target/release/airplay model rank --db /root/dev/Airplay-Predictor/data/airplay.db --top 0 Nice=10 IOSchedulingClass=idle diff --git a/docs/BACKTEST.md b/docs/BACKTEST.md index 3d5a4ef..2907825 100644 --- a/docs/BACKTEST.md +++ b/docs/BACKTEST.md @@ -1,148 +1,147 @@ # Backtest: DR artist × channel × week -Panel: 2329 artists × 14 channels × 51 complete weeks (2025-11-17 to 2026-11-02). Forecast origins from week 27; regressions refitted every 1 origin(s). +Panel: 3246 artists × 14 channels × 51 complete weeks (2025-11-17 to 2026-11-02). Forecast origins from week 27; regressions refitted every 2 origin(s). -"In rotation" = at least τ plays in the week: p3 7, p4kbh 5, p4sjaelland 5, p4fyn 5, p4syd 5, p4trekanten 5, p4vest 5, p4aarhus 5, p4nord 5, p4bornholm 5, p4esbjerg 5, p5 4, p6beat 5, p8jazz 3. +"In rotation" = at least τ plays in the week: p3 23, p4kbh 15, p4sjaelland 14, p4fyn 14, p4syd 14, p4trekanten 14, p4vest 13, p4aarhus 14, p4nord 15, p4bornholm 14, p4esbjerg 14, p5 10, p6beat 16, p8jazz 7. ## h = 1 week(s) ahead (24 forecast weeks) | Model | Log-loss, all | Log-loss, adds | Brier, adds | PR-AUC, adds | MAE plays | Spearman | NDCG@50 | |---|---|---|---|---|---|---|---| -| persistence | 0.0683 | 0.0488 | 0.0085 | 0.013 | 0.28 | 0.829 | 0.967 | -| popularity | 0.0470 | 0.0287 | 0.0080 | 0.256 | 0.35 | 0.864 | 0.888 | -| kalman | 0.0385 | 0.0256 | 0.0065 | 0.377 | 0.32 | 0.874 | 0.957 | -| nb | 0.0389 | 0.0260 | 0.0066 | 0.355 | 0.28 | 0.859 | 0.965 | -| logit | 0.0363 | 0.0241 | 0.0064 | 0.367 | 0.28 | 0.859 | 0.965 | +| persistence | 0.0217 | 0.0159 | 0.0023 | 0.003 | 0.27 | 0.826 | 0.967 | +| popularity | 0.0216 | 0.0110 | 0.0028 | 0.170 | 0.31 | 0.864 | 0.886 | +| kalman | 0.0156 | 0.0088 | 0.0019 | 0.316 | 0.30 | 0.872 | 0.951 | +| nb | 0.0131 | 0.0080 | 0.0017 | 0.330 | 0.27 | 0.856 | 0.960 | +| logit | 0.0113 | 0.0066 | 0.0017 | 0.359 | 0.27 | 0.856 | 0.960 | Skill on adds (1 − log-loss ÷ baseline log-loss; 90% block-bootstrap interval over weeks): | Model | vs persistence | vs popularity | |---|---|---| -| kalman | 0.475 (0.451 to 0.496) | 0.109 (0.063 to 0.141) | -| nb | 0.466 (0.445 to 0.486) | 0.094 (0.051 to 0.123) | -| logit | 0.505 (0.489 to 0.520) | 0.160 (0.129 to 0.184) | +| kalman | 0.447 (0.376 to 0.510) | 0.204 (0.136 to 0.266) | +| nb | 0.497 (0.439 to 0.550) | 0.274 (0.204 to 0.337) | +| logit | 0.582 (0.554 to 0.614) | 0.398 (0.368 to 0.436) | Calibration of the NB's add probabilities (rows not in rotation at the origin): | Predicted | Rows | Mean predicted | Observed | |---|---|---|---| -| 0.0–0.1 | 716676 | 0.003 | 0.002 | -| 0.1–0.2 | 15028 | 0.143 | 0.088 | -| 0.2–0.3 | 6884 | 0.244 | 0.219 | -| 0.3–0.4 | 3000 | 0.343 | 0.391 | -| 0.4–0.5 | 1355 | 0.441 | 0.578 | -| 0.5–0.6 | 521 | 0.540 | 0.551 | -| 0.6–0.7 | 197 | 0.650 | 0.482 | -| 0.7–0.8 | 108 | 0.748 | 0.509 | -| 0.8–0.9 | 95 | 0.847 | 0.211 | -| 0.9–1.0 | 31 | 0.941 | 0.258 | +| 0.0–0.1 | 1067535 | 0.000 | 0.001 | +| 0.1–0.2 | 2694 | 0.146 | 0.129 | +| 0.2–0.3 | 1559 | 0.247 | 0.246 | +| 0.3–0.4 | 1113 | 0.347 | 0.340 | +| 0.4–0.5 | 697 | 0.449 | 0.420 | +| 0.5–0.6 | 444 | 0.543 | 0.482 | +| 0.6–0.7 | 212 | 0.649 | 0.448 | +| 0.7–0.8 | 96 | 0.740 | 0.417 | +| 0.8–0.9 | 63 | 0.840 | 0.508 | +| 0.9–1.0 | 19 | 0.950 | 0.421 | ## h = 4 week(s) ahead (21 forecast weeks) | Model | Log-loss, all | Log-loss, adds | Brier, adds | PR-AUC, adds | MAE plays | Spearman | NDCG@50 | |---|---|---|---|---|---|---|---| -| persistence | 0.0773 | 0.0560 | 0.0100 | 0.014 | 0.36 | 0.810 | 0.908 | -| popularity | 0.0563 | 0.0350 | 0.0091 | 0.246 | 0.41 | 0.841 | 0.834 | -| kalman | 0.0536 | 0.0346 | 0.0081 | 0.347 | 0.43 | 0.849 | 0.895 | -| nb | 0.0543 | 0.0371 | 0.0081 | 0.319 | 0.37 | 0.835 | 0.909 | -| logit | 0.0484 | 0.0319 | 0.0079 | 0.325 | 0.37 | 0.835 | 0.909 | +| persistence | 0.0318 | 0.0242 | 0.0037 | 0.005 | 0.33 | 0.811 | 0.907 | +| popularity | 0.0284 | 0.0168 | 0.0039 | 0.145 | 0.36 | 0.845 | 0.829 | +| kalman | 0.0283 | 0.0196 | 0.0033 | 0.227 | 0.39 | 0.850 | 0.887 | +| nb | 0.0257 | 0.0186 | 0.0031 | 0.251 | 0.34 | 0.835 | 0.898 | +| logit | 0.0198 | 0.0127 | 0.0029 | 0.298 | 0.34 | 0.835 | 0.898 | Skill on adds (1 − log-loss ÷ baseline log-loss; 90% block-bootstrap interval over weeks): | Model | vs persistence | vs popularity | |---|---|---| -| kalman | 0.383 (0.333 to 0.430) | 0.012 (-0.045 to 0.048) | -| nb | 0.337 (0.254 to 0.407) | -0.061 (-0.165 to 0.010) | -| logit | 0.430 (0.398 to 0.462) | 0.088 (0.056 to 0.102) | +| kalman | 0.191 (0.046 to 0.325) | -0.170 (-0.329 to -0.038) | +| nb | 0.235 (0.093 to 0.362) | -0.107 (-0.283 to 0.030) | +| logit | 0.477 (0.420 to 0.536) | 0.244 (0.175 to 0.293) | Calibration of the NB's add probabilities (rows not in rotation at the origin): | Predicted | Rows | Mean predicted | Observed | |---|---|---|---| -| 0.0–0.1 | 622067 | 0.003 | 0.002 | -| 0.1–0.2 | 15752 | 0.144 | 0.083 | -| 0.2–0.3 | 7411 | 0.245 | 0.203 | -| 0.3–0.4 | 3454 | 0.343 | 0.359 | -| 0.4–0.5 | 1395 | 0.442 | 0.463 | -| 0.5–0.6 | 487 | 0.545 | 0.591 | -| 0.6–0.7 | 212 | 0.646 | 0.538 | -| 0.7–0.8 | 125 | 0.747 | 0.208 | -| 0.8–0.9 | 76 | 0.844 | 0.579 | -| 0.9–1.0 | 21 | 0.933 | 0.667 | +| 0.0–0.1 | 931278 | 0.001 | 0.002 | +| 0.1–0.2 | 3415 | 0.143 | 0.094 | +| 0.2–0.3 | 2051 | 0.247 | 0.207 | +| 0.3–0.4 | 1566 | 0.348 | 0.287 | +| 0.4–0.5 | 1010 | 0.444 | 0.413 | +| 0.5–0.6 | 528 | 0.545 | 0.443 | +| 0.6–0.7 | 235 | 0.641 | 0.306 | +| 0.7–0.8 | 94 | 0.748 | 0.521 | +| 0.8–0.9 | 35 | 0.839 | 0.457 | +| 0.9–1.0 | 3 | 0.927 | 0.000 | ## h = 8 week(s) ahead (17 forecast weeks) | Model | Log-loss, all | Log-loss, adds | Brier, adds | PR-AUC, adds | MAE plays | Spearman | NDCG@50 | |---|---|---|---|---|---|---|---| -| persistence | 0.0874 | 0.0648 | 0.0119 | 0.017 | 0.45 | 0.779 | 0.837 | -| popularity | 0.0679 | 0.0445 | 0.0107 | 0.236 | 0.46 | 0.815 | 0.784 | -| kalman | 0.0703 | 0.0473 | 0.0105 | 0.293 | 0.57 | 0.815 | 0.825 | -| nb | 0.0774 | 0.0570 | 0.0103 | 0.270 | 0.48 | 0.797 | 0.828 | -| logit | 0.0634 | 0.0432 | 0.0101 | 0.262 | 0.48 | 0.797 | 0.828 | +| persistence | 0.0433 | 0.0344 | 0.0055 | 0.007 | 0.40 | 0.783 | 0.833 | +| popularity | 0.0359 | 0.0241 | 0.0052 | 0.152 | 0.40 | 0.823 | 0.777 | +| kalman | 0.0427 | 0.0330 | 0.0050 | 0.195 | 0.51 | 0.821 | 0.816 | +| nb | 0.0418 | 0.0330 | 0.0048 | 0.233 | 0.43 | 0.804 | 0.812 | +| logit | 0.0301 | 0.0208 | 0.0046 | 0.280 | 0.43 | 0.804 | 0.812 | Skill on adds (1 − log-loss ÷ baseline log-loss; 90% block-bootstrap interval over weeks): | Model | vs persistence | vs popularity | |---|---|---| -| kalman | 0.271 (0.219 to 0.305) | -0.064 (-0.100 to -0.046) | -| nb | 0.120 (-0.012 to 0.209) | -0.283 (-0.418 to -0.196) | -| logit | 0.334 (0.289 to 0.366) | 0.028 (-0.001 to 0.043) | +| kalman | 0.040 (-0.078 to 0.167) | -0.367 (-0.490 to -0.223) | +| nb | 0.042 (-0.097 to 0.159) | -0.365 (-0.522 to -0.228) | +| logit | 0.394 (0.348 to 0.433) | 0.137 (0.089 to 0.173) | Calibration of the NB's add probabilities (rows not in rotation at the origin): | Predicted | Rows | Mean predicted | Observed | |---|---|---|---| -| 0.0–0.1 | 497574 | 0.004 | 0.004 | -| 0.1–0.2 | 15374 | 0.143 | 0.071 | -| 0.2–0.3 | 7616 | 0.244 | 0.160 | -| 0.3–0.4 | 4057 | 0.343 | 0.263 | -| 0.4–0.5 | 1630 | 0.441 | 0.426 | -| 0.5–0.6 | 493 | 0.544 | 0.519 | -| 0.6–0.7 | 264 | 0.637 | 0.523 | -| 0.7–0.8 | 79 | 0.751 | 0.456 | -| 0.8–0.9 | 63 | 0.837 | 0.476 | -| 0.9–1.0 | 15 | 0.922 | 0.733 | +| 0.0–0.1 | 751167 | 0.001 | 0.003 | +| 0.1–0.2 | 4090 | 0.145 | 0.084 | +| 0.2–0.3 | 2546 | 0.248 | 0.209 | +| 0.3–0.4 | 1673 | 0.348 | 0.289 | +| 0.4–0.5 | 976 | 0.443 | 0.416 | +| 0.5–0.6 | 429 | 0.544 | 0.445 | +| 0.6–0.7 | 208 | 0.646 | 0.409 | +| 0.7–0.8 | 76 | 0.740 | 0.434 | +| 0.8–0.9 | 25 | 0.828 | 0.880 | -## NB coefficients, h = 4, all weeks (alpha = 0.116) +## NB coefficients, h = 4, all weeks (alpha = 0.134) | Feature | Coefficient | |---|---| -| intercept | -0.694 | -| log plays t | +0.380 | +| intercept | -0.787 | +| log plays t | +0.313 | | log plays t-1 | +0.140 | -| log mean plays 4w | +0.447 | -| log mean plays 12w | -0.190 | -| log max plays 12w | +0.420 | -| log other channels t | -0.024 | -| log other channels 4w | +0.045 | -| log P3 t | +0.020 | -| weeks since played (/12) | -3.147 | -| Track Top-40 score | -0.018 | -| Airplay Top-20 score | -0.397 | -| chart entry recency 6w | +0.425 | -| chart entry recency 26w | -0.275 | -| log Track Top-40 hits | +0.029 | -| log Airplay Top-20 hits | +0.019 | -| Danish | -0.013 | -| group | -0.091 | -| female | +0.022 | -| career years (/50) | -0.269 | -| debut unknown | -0.080 | +| log mean plays 4w | +0.480 | +| log mean plays 12w | -0.057 | +| log max plays 12w | +0.368 | +| log other channels t | -0.018 | +| log other channels 4w | +0.044 | +| log P3 t | +0.014 | +| weeks since played (/12) | -3.164 | +| Track Top-40 score | -0.021 | +| Airplay Top-20 score | -0.418 | +| chart entry recency 6w | +0.539 | +| chart entry recency 26w | -0.383 | +| log Track Top-40 hits | +0.032 | +| log Airplay Top-20 hits | +0.005 | +| Danish | -0.007 | +| group | -0.053 | +| female | +0.021 | +| career years (/50) | -0.229 | +| debut unknown | -0.059 | | Christmas target week | +0.000 | -| Danish x P3 | +0.007 | -| Danish x P4 | +0.039 | -| Danish x P6 | +0.020 | -| ch p4kbh | -0.043 | -| ch p4sjaelland | -0.039 | -| ch p4fyn | -0.046 | -| ch p4syd | -0.040 | -| ch p4trekanten | -0.044 | -| ch p4vest | -0.041 | -| ch p4aarhus | -0.040 | -| ch p4nord | -0.037 | -| ch p4bornholm | -0.043 | -| ch p4esbjerg | -0.041 | -| ch p5 | +0.092 | -| ch p6beat | +0.044 | -| ch p8jazz | +0.085 | +| Danish x P3 | -0.001 | +| Danish x P4 | +0.020 | +| Danish x P6 | +0.007 | +| ch p4kbh | -0.013 | +| ch p4sjaelland | -0.009 | +| ch p4fyn | -0.015 | +| ch p4syd | -0.010 | +| ch p4trekanten | -0.012 | +| ch p4vest | -0.010 | +| ch p4aarhus | -0.010 | +| ch p4nord | -0.007 | +| ch p4bornholm | -0.013 | +| ch p4esbjerg | -0.011 | +| ch p5 | +0.118 | +| ch p6beat | +0.066 | +| ch p8jazz | +0.103 | diff --git a/scripts/after-backfill-nmf.sh b/scripts/after-backfill-nmf.sh deleted file mode 100755 index 44d5556..0000000 --- a/scripts/after-backfill-nmf.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -# One-off: New Music Friday history once the running backfill has the ISRCs. -cd "$(dirname "$0")/.." || exit 1 -while pgrep -f "scripts/backfill.sh" >/dev/null; do sleep 120; done -./target/release/airplay collect --only nmf --backfill -./target/release/airplay stats diff --git a/scripts/backfill.sh b/scripts/backfill.sh index 9976907..84144ac 100755 --- a/scripts/backfill.sh +++ b/scripts/backfill.sh @@ -4,7 +4,8 @@ cd "$(dirname "$0")/.." || exit 1 bin=./target/release/airplay # Wait for collectors already running (a backfill started by hand, the daily timer). -while pgrep -f "release/airplay collect" >/dev/null; do sleep 60; done +# (match the process name, not -f: a parent shell whose command line mentions the binary would match forever) +while pgrep -x airplay -a | grep -q " collect"; do sleep 60; done for source in dr hitlisten deezer musicbrainz spotify nmf playlists; do echo "== $(date -u +%FT%TZ) $source" $bin collect --only "$source" --backfill diff --git a/src/model/mod.rs b/src/model/mod.rs index fe0e97e..31f00c6 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -3,7 +3,7 @@ //! //! airplay model panel what the panel holds //! airplay model backtest [--h 1,4,8] [--refit 1] [--out docs/BACKTEST.md] -//! airplay model calibrate rotation thresholds from DR's lists -> docs/CALIBRATION.md +//! airplay model calibrate [--out docs/CALIBRATION.md] rotation thresholds from DR's lists //! airplay model ifpi the 20-year IFPI test bed -> docs/IFPI.md //! airplay model adds [--h 0,4] [--out docs/BACKTEST-ADDS.md] backtest of the new-add model //! airplay model rank [--h 4] [--max-h 12] [--top 50] @@ -64,7 +64,8 @@ pub fn run(db: &Db, args: &[String]) -> Result<()> { } Some("calibrate") => { let md = calibrate::run(db, &p)?; - std::fs::write("docs/CALIBRATION.md", &md)?; + // The daily job runs this too; only write the report into the repo when asked. + if let Some(out) = arg(args, "--out") { std::fs::write(out, &md)?; } print!("{md}"); } Some("adds") => {