From 868d3cb96d302af7ba026838fef2e96f3fecbfe0 Mon Sep 17 00:00:00 2001 From: moritsbw-cpu Date: Thu, 19 Nov 2026 22:21:35 +0000 Subject: [PATCH] Add-model backtests report ROC-AUC too Co-Authored-By: Claude Opus 5.5 --- src/model/adds.rs | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/model/adds.rs b/src/model/adds.rs index bd82a20..6eebc0d 100644 --- a/src/model/adds.rs +++ b/src/model/adds.rs @@ -587,6 +587,22 @@ pub fn set_chart(x: &mut [f64], idx: [usize; 4], entry: Option>, t40 fn ll(p: f64, y: bool) -> f64 { -(if y { p } else { 1.0 - p }).clamp(1e-9, 1.0).ln() } +/// ROC-AUC (Mann-Whitney): the chance a random add is ranked above a random non-add; ties count half. +fn roc_auc(mut v: Vec<(f64, bool)>) -> f64 { + v.sort_by(|a, b| a.0.total_cmp(&b.0)); + let (mut i, mut rank_sum, n_pos) = (0, 0.0, v.iter().filter(|x| x.1).count() as f64); + let n_neg = v.len() as f64 - n_pos; + while i < v.len() { + let mut j = i; + while j < v.len() && v[j].0 == v[i].0 { j += 1; } + let avg = (i + j + 1) as f64 / 2.0; // ranks i+1 ..= j + rank_sum += v[i..j].iter().filter(|x| x.1).count() as f64 * avg; + i = j; + } + if n_pos == 0.0 || n_neg == 0.0 { return f64::NAN; } + (rank_sum - n_pos * (n_pos + 1.0) / 2.0) / (n_pos * n_neg) +} + fn average_precision(mut v: Vec<(f64, bool)>) -> f64 { v.sort_by(|a, b| b.0.total_cmp(&a.0)); let (mut hits, mut sum) = (0.0, 0.0); @@ -639,6 +655,7 @@ impl Scores { fn col(&self, k: usize) -> Vec { self.wk.iter().map(|w| w[k]).collect() } fn logloss(&self, k: usize) -> f64 { self.col(k).iter().sum::() / self.n.max(1.0) } fn prauc(&self, k: usize) -> f64 { average_precision(self.pairs[k].clone()) } + fn rocauc(&self, k: usize) -> f64 { roc_auc(self.pairs[k].clone()) } fn calib_table(&self, md: &mut String, ks: &[usize]) -> Result<()> { write!(md, "| Predicted |")?; for &k in ks { write!(md, " {}: rows / mean predicted / observed |", self.names[k])?; } @@ -672,11 +689,12 @@ const SCORED: [&str; 8] = ["rate by in-rotation-now", "rate by last-12-weeks lev "model + pooled Platt (pre-release)", "model + pooled isotonic (pre-release)", "model, NMF known (post-release)"]; fn table(md: &mut String, s_c: &Scores, s_a: &Scores, rows: &mut Vec<(String, usize, String, f64)>, h: usize, prefix: &str) -> Result<()> { - writeln!(md, "| Model | Log-loss, per channel | PR-AUC, per channel | Log-loss, any channel | PR-AUC, any channel |\n|---|---|---|---|---|")?; + writeln!(md, "| Model | Log-loss, per channel | PR-AUC, per channel | ROC-AUC, per channel | Log-loss, any channel | PR-AUC, any channel | ROC-AUC, any channel |\n|---|---|---|---|---|---|---|")?; for k in 0..SCORED.len() { let (lc, pc, la, pa) = (s_c.logloss(k), s_c.prauc(k), s_a.logloss(k), s_a.prauc(k)); - writeln!(md, "| {} | {lc:.4} | {pc:.3} | {la:.4} | {pa:.3} |", SCORED[k])?; - for (m, v) in [("logloss_ch", lc), ("prauc_ch", pc), ("logloss_any", la), ("prauc_any", pa)] { + let (rc, ra) = (s_c.rocauc(k), s_a.rocauc(k)); + writeln!(md, "| {} | {lc:.4} | {pc:.3} | {rc:.3} | {la:.4} | {pa:.3} | {ra:.3} |", SCORED[k])?; + for (m, v) in [("logloss_ch", lc), ("prauc_ch", pc), ("rocauc_ch", rc), ("logloss_any", la), ("prauc_any", pa), ("rocauc_any", ra)] { rows.push((SCORED[k].to_string(), h, format!("{prefix}{m}"), v)); } }