Add-model backtests report ROC-AUC too
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+21
-3
@@ -587,6 +587,22 @@ pub fn set_chart(x: &mut [f64], idx: [usize; 4], entry: Option<Option<f64>>, t40
|
|||||||
|
|
||||||
fn ll(p: f64, y: bool) -> f64 { -(if y { p } else { 1.0 - p }).clamp(1e-9, 1.0).ln() }
|
fn ll(p: f64, y: bool) -> f64 { -(if y { p } else { 1.0 - p }).clamp(1e-9, 1.0).ln() }
|
||||||
|
|
||||||
|
/// ROC-AUC (Mann-Whitney): the chance a random add is ranked above a random non-add; ties count half.
|
||||||
|
fn roc_auc(mut v: Vec<(f64, bool)>) -> f64 {
|
||||||
|
v.sort_by(|a, b| a.0.total_cmp(&b.0));
|
||||||
|
let (mut i, mut rank_sum, n_pos) = (0, 0.0, v.iter().filter(|x| x.1).count() as f64);
|
||||||
|
let n_neg = v.len() as f64 - n_pos;
|
||||||
|
while i < v.len() {
|
||||||
|
let mut j = i;
|
||||||
|
while j < v.len() && v[j].0 == v[i].0 { j += 1; }
|
||||||
|
let avg = (i + j + 1) as f64 / 2.0; // ranks i+1 ..= j
|
||||||
|
rank_sum += v[i..j].iter().filter(|x| x.1).count() as f64 * avg;
|
||||||
|
i = j;
|
||||||
|
}
|
||||||
|
if n_pos == 0.0 || n_neg == 0.0 { return f64::NAN; }
|
||||||
|
(rank_sum - n_pos * (n_pos + 1.0) / 2.0) / (n_pos * n_neg)
|
||||||
|
}
|
||||||
|
|
||||||
fn average_precision(mut v: Vec<(f64, bool)>) -> f64 {
|
fn average_precision(mut v: Vec<(f64, bool)>) -> f64 {
|
||||||
v.sort_by(|a, b| b.0.total_cmp(&a.0));
|
v.sort_by(|a, b| b.0.total_cmp(&a.0));
|
||||||
let (mut hits, mut sum) = (0.0, 0.0);
|
let (mut hits, mut sum) = (0.0, 0.0);
|
||||||
@@ -639,6 +655,7 @@ impl Scores {
|
|||||||
fn col(&self, k: usize) -> Vec<f64> { self.wk.iter().map(|w| w[k]).collect() }
|
fn col(&self, k: usize) -> Vec<f64> { self.wk.iter().map(|w| w[k]).collect() }
|
||||||
fn logloss(&self, k: usize) -> f64 { self.col(k).iter().sum::<f64>() / self.n.max(1.0) }
|
fn logloss(&self, k: usize) -> f64 { self.col(k).iter().sum::<f64>() / self.n.max(1.0) }
|
||||||
fn prauc(&self, k: usize) -> f64 { average_precision(self.pairs[k].clone()) }
|
fn prauc(&self, k: usize) -> f64 { average_precision(self.pairs[k].clone()) }
|
||||||
|
fn rocauc(&self, k: usize) -> f64 { roc_auc(self.pairs[k].clone()) }
|
||||||
fn calib_table(&self, md: &mut String, ks: &[usize]) -> Result<()> {
|
fn calib_table(&self, md: &mut String, ks: &[usize]) -> Result<()> {
|
||||||
write!(md, "| Predicted |")?;
|
write!(md, "| Predicted |")?;
|
||||||
for &k in ks { write!(md, " {}: rows / mean predicted / observed |", self.names[k])?; }
|
for &k in ks { write!(md, " {}: rows / mean predicted / observed |", self.names[k])?; }
|
||||||
@@ -672,11 +689,12 @@ const SCORED: [&str; 8] = ["rate by in-rotation-now", "rate by last-12-weeks lev
|
|||||||
"model + pooled Platt (pre-release)", "model + pooled isotonic (pre-release)", "model, NMF known (post-release)"];
|
"model + pooled Platt (pre-release)", "model + pooled isotonic (pre-release)", "model, NMF known (post-release)"];
|
||||||
|
|
||||||
fn table(md: &mut String, s_c: &Scores, s_a: &Scores, rows: &mut Vec<(String, usize, String, f64)>, h: usize, prefix: &str) -> Result<()> {
|
fn table(md: &mut String, s_c: &Scores, s_a: &Scores, rows: &mut Vec<(String, usize, String, f64)>, h: usize, prefix: &str) -> Result<()> {
|
||||||
writeln!(md, "| Model | Log-loss, per channel | PR-AUC, per channel | Log-loss, any channel | PR-AUC, any channel |\n|---|---|---|---|---|")?;
|
writeln!(md, "| Model | Log-loss, per channel | PR-AUC, per channel | ROC-AUC, per channel | Log-loss, any channel | PR-AUC, any channel | ROC-AUC, any channel |\n|---|---|---|---|---|---|---|")?;
|
||||||
for k in 0..SCORED.len() {
|
for k in 0..SCORED.len() {
|
||||||
let (lc, pc, la, pa) = (s_c.logloss(k), s_c.prauc(k), s_a.logloss(k), s_a.prauc(k));
|
let (lc, pc, la, pa) = (s_c.logloss(k), s_c.prauc(k), s_a.logloss(k), s_a.prauc(k));
|
||||||
writeln!(md, "| {} | {lc:.4} | {pc:.3} | {la:.4} | {pa:.3} |", SCORED[k])?;
|
let (rc, ra) = (s_c.rocauc(k), s_a.rocauc(k));
|
||||||
for (m, v) in [("logloss_ch", lc), ("prauc_ch", pc), ("logloss_any", la), ("prauc_any", pa)] {
|
writeln!(md, "| {} | {lc:.4} | {pc:.3} | {rc:.3} | {la:.4} | {pa:.3} | {ra:.3} |", SCORED[k])?;
|
||||||
|
for (m, v) in [("logloss_ch", lc), ("prauc_ch", pc), ("rocauc_ch", rc), ("logloss_any", la), ("prauc_any", pa), ("rocauc_any", ra)] {
|
||||||
rows.push((SCORED[k].to_string(), h, format!("{prefix}{m}"), v));
|
rows.push((SCORED[k].to_string(), h, format!("{prefix}{m}"), v));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user