Add-model backtests report ROC-AUC too

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-19 22:21:35 +00:00
co-authored by Claude Opus 5.5
parent d7eeeafda9
commit 868d3cb96d
+21 -3
View File
@@ -587,6 +587,22 @@ pub fn set_chart(x: &mut [f64], idx: [usize; 4], entry: Option<Option<f64>>, t40
fn ll(p: f64, y: bool) -> f64 { -(if y { p } else { 1.0 - p }).clamp(1e-9, 1.0).ln() } fn ll(p: f64, y: bool) -> f64 { -(if y { p } else { 1.0 - p }).clamp(1e-9, 1.0).ln() }
/// ROC-AUC (Mann-Whitney): the chance a random add is ranked above a random non-add; ties count half.
fn roc_auc(mut v: Vec<(f64, bool)>) -> f64 {
v.sort_by(|a, b| a.0.total_cmp(&b.0));
let (mut i, mut rank_sum, n_pos) = (0, 0.0, v.iter().filter(|x| x.1).count() as f64);
let n_neg = v.len() as f64 - n_pos;
while i < v.len() {
let mut j = i;
while j < v.len() && v[j].0 == v[i].0 { j += 1; }
let avg = (i + j + 1) as f64 / 2.0; // ranks i+1 ..= j
rank_sum += v[i..j].iter().filter(|x| x.1).count() as f64 * avg;
i = j;
}
if n_pos == 0.0 || n_neg == 0.0 { return f64::NAN; }
(rank_sum - n_pos * (n_pos + 1.0) / 2.0) / (n_pos * n_neg)
}
fn average_precision(mut v: Vec<(f64, bool)>) -> f64 { fn average_precision(mut v: Vec<(f64, bool)>) -> f64 {
v.sort_by(|a, b| b.0.total_cmp(&a.0)); v.sort_by(|a, b| b.0.total_cmp(&a.0));
let (mut hits, mut sum) = (0.0, 0.0); let (mut hits, mut sum) = (0.0, 0.0);
@@ -639,6 +655,7 @@ impl Scores {
fn col(&self, k: usize) -> Vec<f64> { self.wk.iter().map(|w| w[k]).collect() } fn col(&self, k: usize) -> Vec<f64> { self.wk.iter().map(|w| w[k]).collect() }
fn logloss(&self, k: usize) -> f64 { self.col(k).iter().sum::<f64>() / self.n.max(1.0) } fn logloss(&self, k: usize) -> f64 { self.col(k).iter().sum::<f64>() / self.n.max(1.0) }
fn prauc(&self, k: usize) -> f64 { average_precision(self.pairs[k].clone()) } fn prauc(&self, k: usize) -> f64 { average_precision(self.pairs[k].clone()) }
fn rocauc(&self, k: usize) -> f64 { roc_auc(self.pairs[k].clone()) }
fn calib_table(&self, md: &mut String, ks: &[usize]) -> Result<()> { fn calib_table(&self, md: &mut String, ks: &[usize]) -> Result<()> {
write!(md, "| Predicted |")?; write!(md, "| Predicted |")?;
for &k in ks { write!(md, " {}: rows / mean predicted / observed |", self.names[k])?; } for &k in ks { write!(md, " {}: rows / mean predicted / observed |", self.names[k])?; }
@@ -672,11 +689,12 @@ const SCORED: [&str; 8] = ["rate by in-rotation-now", "rate by last-12-weeks lev
"model + pooled Platt (pre-release)", "model + pooled isotonic (pre-release)", "model, NMF known (post-release)"]; "model + pooled Platt (pre-release)", "model + pooled isotonic (pre-release)", "model, NMF known (post-release)"];
fn table(md: &mut String, s_c: &Scores, s_a: &Scores, rows: &mut Vec<(String, usize, String, f64)>, h: usize, prefix: &str) -> Result<()> { fn table(md: &mut String, s_c: &Scores, s_a: &Scores, rows: &mut Vec<(String, usize, String, f64)>, h: usize, prefix: &str) -> Result<()> {
writeln!(md, "| Model | Log-loss, per channel | PR-AUC, per channel | Log-loss, any channel | PR-AUC, any channel |\n|---|---|---|---|---|")?; writeln!(md, "| Model | Log-loss, per channel | PR-AUC, per channel | ROC-AUC, per channel | Log-loss, any channel | PR-AUC, any channel | ROC-AUC, any channel |\n|---|---|---|---|---|---|---|")?;
for k in 0..SCORED.len() { for k in 0..SCORED.len() {
let (lc, pc, la, pa) = (s_c.logloss(k), s_c.prauc(k), s_a.logloss(k), s_a.prauc(k)); let (lc, pc, la, pa) = (s_c.logloss(k), s_c.prauc(k), s_a.logloss(k), s_a.prauc(k));
writeln!(md, "| {} | {lc:.4} | {pc:.3} | {la:.4} | {pa:.3} |", SCORED[k])?; let (rc, ra) = (s_c.rocauc(k), s_a.rocauc(k));
for (m, v) in [("logloss_ch", lc), ("prauc_ch", pc), ("logloss_any", la), ("prauc_any", pa)] { writeln!(md, "| {} | {lc:.4} | {pc:.3} | {rc:.3} | {la:.4} | {pa:.3} | {ra:.3} |", SCORED[k])?;
for (m, v) in [("logloss_ch", lc), ("prauc_ch", pc), ("rocauc_ch", rc), ("logloss_any", la), ("prauc_any", pa), ("rocauc_any", ra)] {
rows.push((SCORED[k].to_string(), h, format!("{prefix}{m}"), v)); rows.push((SCORED[k].to_string(), h, format!("{prefix}{m}"), v));
} }
} }