New-add model (add = entering rotation after 4+ weeks out; per channel and any DR channel, 8-week window), site ranks by it; DR rotation-list PDF parser (glyph positions, channel by play profile) and Wayback Machine backfill: 29 weekly lists 2023-2026

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-15 02:50:13 +00:00
co-authored by Claude Opus 5.5
parent 387502ebfa
commit c698925814
9 changed files with 1122 additions and 89 deletions
+62 -47
View File
@@ -10,6 +10,7 @@
//! Rankings are read from the forecasts table (`airplay model rank`); the what-if models are
//! fitted in the background at start and again whenever a newer forecast run appears.
use crate::model::adds::{set_chart, AddModel, Rot, CHART_IDX_CHANNEL};
use crate::model::common::{thresholds, Direct, Rng};
use crate::model::features::{self, P};
use crate::model::glm;
@@ -30,6 +31,8 @@ struct Models {
panel: Panel,
tau: [u32; C],
direct: Vec<Direct>, // index h - 1
adds: Vec<AddModel>, // index h
rot: Rot,
index: HashMap<String, usize>,
}
@@ -100,9 +103,11 @@ fn fit_models(db_path: &str, made_at: String) -> Result<Models> {
let db = crate::db::Db::open(db_path)?;
let panel = Panel::load(&db)?;
let tau = thresholds(&panel, panel.w());
let direct = (1..=MAX_H).map(|h| Direct::fit(&panel, h, panel.w() - 1, &tau, true)).collect();
let direct = (1..=MAX_H).map(|h| Direct::fit(&panel, h, panel.w() - 1, &tau, false)).collect();
let rot = Rot::new(&panel, &tau);
let adds = (0..=MAX_H).map(|h| AddModel::fit(&panel, &rot, h, panel.w() - 1)).collect();
let index = panel.artists.iter().enumerate().map(|(i, a)| (a.id.clone(), i)).collect();
Ok(Models { made_at, panel, tau, direct, index })
Ok(Models { made_at, panel, tau, direct, adds, rot, index })
}
fn query(url: &str) -> HashMap<String, String> {
@@ -143,7 +148,7 @@ fn handle(st: &State, method: &Method, url: &str) -> Result<Reply> {
"" => Ok((301, "text/plain", Arc::new(String::new()))),
"/" | "/index.html" => Ok((200, "text/html; charset=utf-8", Arc::new(PAGE.to_string()))),
"/api/meta" => meta(st),
"/api/rank" => rank(st, q.get("h").and_then(|h| h.parse().ok()).unwrap_or(4).clamp(1, MAX_H)),
"/api/rank" => rank(st, q.get("h").and_then(|h| h.parse().ok()).unwrap_or(0).min(MAX_H)),
"/api/artist" => artist(st, q.get("id").map(String::as_str).unwrap_or("")),
"/api/whatif" => whatif(st, &q),
_ => Ok((404, "text/plain", Arc::new("not found".into()))),
@@ -169,6 +174,21 @@ fn meta(st: &State) -> Result<Reply> {
"channels": CHANNELS, "tau": tau, "whatif_ready": models.is_some(), "backtest_run": run, "backtest": scores}))
}
/// Puts one forecast row into an artist's JSON: airplay ('rotation', 'any'), new adds ('add',
/// 'add_any') and the origin week ('now').
fn fold(e: &mut Value, target: &str, ch: &str, prob: Option<f64>, mean: Option<f64>, lo: Option<f64>, hi: Option<f64>) {
let r3 = |v: Option<f64>| json!(v.map(|p| (p * 1000.0).round() / 1000.0));
let r1 = |v: Option<f64>| json!(v.map(|m| (m * 10.0).round() / 10.0));
match target {
"any" => { e["p_any"] = r3(prob); e["mean"] = r1(mean); e["lo"] = json!(lo); e["hi"] = json!(hi); }
"rotation" => { e["p"][ch] = r3(prob); e["plays"][ch] = r1(mean); }
"add_any" => { e["add_any"] = r3(prob); }
"add" => { e["add"][ch] = r3(prob); }
"now" => { e["now"][ch] = json!({"on": prob.unwrap_or(0.0) > 0.5, "plays": mean}); }
_ => {}
}
}
fn rank(st: &State, h: usize) -> Result<Reply> {
let c = ro(&st.db_path)?;
let Some(made) = latest_run(&c)? else { return ok(json!({"rows": []})) };
@@ -180,26 +200,24 @@ fn rank(st: &State, h: usize) -> Result<Reply> {
names.insert(id, (n, country.as_deref() == Some("Danmark"), kind.is_some_and(|k| k != "person")));
}
let mut rows: HashMap<String, Value> = HashMap::new();
let mut st_ = c.prepare("SELECT key, target, prob, mean, lo, hi FROM forecasts WHERE made_at = ?1 AND model_version = ?2 AND horizon = ?3")?;
let it = st_.query_map(params![made, VERSION, h as i64], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, Option<f64>>(2)?,
r.get::<_, Option<f64>>(3)?, r.get::<_, Option<f64>>(4)?, r.get::<_, Option<f64>>(5)?)))?;
let mut st_ = c.prepare("SELECT key, target, prob, mean, lo, hi, target_week FROM forecasts WHERE made_at = ?1 AND model_version = ?2
AND ((target IN ('rotation', 'any') AND horizon = ?3) OR (target IN ('add', 'add_any') AND horizon = ?4) OR target = 'now')")?;
let it = st_.query_map(params![made, VERSION, h.max(1) as i64, h as i64], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, Option<f64>>(2)?,
r.get::<_, Option<f64>>(3)?, r.get::<_, Option<f64>>(4)?, r.get::<_, Option<f64>>(5)?, r.get::<_, String>(6)?)))?;
let mut window = String::new();
for r in it {
let (key, target, prob, mean, lo, hi) = r?;
let (key, target, prob, mean, lo, hi, week) = r?;
let Some((id, ch)) = key.split_once(':') else { continue };
let e = rows.entry(id.to_string()).or_insert_with(|| {
let (n, dk, group) = names.get(id).cloned().unwrap_or_default();
json!({"id": id, "name": n, "danish": dk, "group": group, "p": {}, "plays": {}})
json!({"id": id, "name": n, "danish": dk, "group": group, "p": {}, "plays": {}, "add": {}, "now": {}})
});
if target == "any" {
e["p_any"] = json!(prob); e["mean"] = json!(mean); e["lo"] = json!(lo); e["hi"] = json!(hi);
} else {
e["p"][ch] = json!(prob.map(|p| (p * 1000.0).round() / 1000.0));
e["plays"][ch] = json!(mean.map(|m| (m * 10.0).round() / 10.0));
}
if target == "add_any" && window.is_empty() { window = week; }
fold(e, &target, ch, prob, mean, lo, hi);
}
let mut out: Vec<Value> = rows.into_values().collect();
out.sort_by(|a, b| b["mean"].as_f64().unwrap_or(0.0).total_cmp(&a["mean"].as_f64().unwrap_or(0.0)));
let body = Arc::new(json!({"made_at": made, "h": h, "rows": out}).to_string());
out.sort_by(|a, b| b["add_any"].as_f64().unwrap_or(0.0).total_cmp(&a["add_any"].as_f64().unwrap_or(0.0)));
let body = Arc::new(json!({"made_at": made, "h": h, "window_start": window, "rows": out}).to_string());
let mut cache = st.rank_cache.write().unwrap();
cache.retain(|k, _| k.0 == made);
cache.insert((made, h), body.clone());
@@ -232,17 +250,21 @@ fn artist(st: &State, id: &str) -> Result<Reply> {
}
let mut fc: Vec<Value> = Vec::new();
let mut now_out = Value::Null;
if let Some(made) = latest_run(&c)? {
let mut byh: HashMap<i64, Value> = HashMap::new();
for r in c.prepare("SELECT horizon, target_week, key, prob, mean, lo, hi FROM forecasts WHERE made_at = ?1 AND model_version = ?2 AND key LIKE ?3")?
let mut now = json!({});
for r in c.prepare("SELECT horizon, target_week, key, prob, mean, lo, hi, target FROM forecasts WHERE made_at = ?1 AND model_version = ?2 AND key LIKE ?3")?
.query_map(params![made, VERSION, format!("{id}:%")], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?,
r.get::<_, Option<f64>>(3)?, r.get::<_, Option<f64>>(4)?, r.get::<_, Option<f64>>(5)?, r.get::<_, Option<f64>>(6)?)))? {
let (h, week, key, prob, mean, lo, hi) = r?;
let e = byh.entry(h).or_insert_with(|| json!({"h": h, "week": week, "p": {}, "plays": {}}));
r.get::<_, Option<f64>>(3)?, r.get::<_, Option<f64>>(4)?, r.get::<_, Option<f64>>(5)?, r.get::<_, Option<f64>>(6)?, r.get::<_, String>(7)?)))? {
let (h, week, key, prob, mean, lo, hi, target) = r?;
let ch = key.split_once(':').map(|x| x.1).unwrap_or("");
if ch == "any" { e["p_any"] = json!(prob); e["mean"] = json!(mean); e["lo"] = json!(lo); e["hi"] = json!(hi); }
else { e["p"][ch] = json!(prob); e["plays"][ch] = json!(mean); }
if target == "now" { fold(&mut now, "now", ch, prob, mean, lo, hi); continue; }
let e = byh.entry(h).or_insert_with(|| json!({"h": h, "p": {}, "plays": {}, "add": {}}));
if target.starts_with("add") { e["window_start"] = json!(week); } else { e["week"] = json!(week); }
fold(e, &target, ch, prob, mean, lo, hi);
}
now_out = now["now"].clone();
let mut v: Vec<(i64, Value)> = byh.into_iter().collect();
v.sort_by_key(|x| x.0);
fc = v.into_iter().map(|x| x.1).collect();
@@ -267,7 +289,7 @@ fn artist(st: &State, id: &str) -> Result<Reply> {
ok(json!({
"id": id, "name": name, "country": a["country"], "gender": a["gender"], "description": a["description"],
"debut": a["lifespan"]["beginYear"], "type": a["type"], "image": image, "top_tracks": top,
"weeks": weeks, "plays": plays, "forecasts": fc, "charts": charts,
"weeks": weeks, "plays": plays, "forecasts": fc, "now": now_out, "charts": charts,
}))
}
@@ -275,59 +297,52 @@ fn artist(st: &State, id: &str) -> Result<Reply> {
/// entry weeks since a new chart entry (a release that charted), "none" = never
/// t40 position on the latest Track Top-40 (0 = not on it)
/// air position on the latest Airplay Top-20 (0 = not on it)
/// h = 0..12: the add window starts h weeks after the latest week; plays are for week max(h, 1).
fn whatif(st: &State, q: &HashMap<String, String>) -> Result<Reply> {
let guard = st.models.read().unwrap();
let Some(m) = guard.as_ref() else { return Ok((503, "application/json", Arc::new(json!({"loading": true}).to_string()))) };
let Some(&a) = m.index.get(q.get("id").map(String::as_str).unwrap_or("")) else {
return Ok((404, "application/json", Arc::new(json!({"error": "artist not in the model"}).to_string())));
};
let h: usize = q.get("h").and_then(|v| v.parse().ok()).unwrap_or(4).clamp(1, MAX_H);
let d = &m.direct[h - 1];
let h: usize = q.get("h").and_then(|v| v.parse().ok()).unwrap_or(0).min(MAX_H);
let d = &m.direct[h.max(1) - 1];
let am = &m.adds[h];
let p = &m.panel;
let t0 = p.w() - 1;
let chart = p.chart(a, t0);
let num = |k: &str| q.get(k).filter(|v| !v.is_empty()).and_then(|v| v.parse::<f64>().ok());
let entry: Option<Option<f64>> = q.get("entry").filter(|v| !v.is_empty()).map(|v| if v == "none" { None } else { v.parse().ok() });
let t40 = num("t40");
let air = num("air");
let (t40, air) = (num("t40"), num("air"));
let over = move |x: &mut [f64], idx: [usize; 4]| set_chart(x, idx, entry, t40, air);
let run = |overrides: bool| -> Value {
let f: Option<&dyn Fn(&mut [f64], [usize; 4])> = if overrides { Some(&over) } else { None };
let (add_c, add_any) = am.predict(p, &m.rot, a, t0, f);
let mut x = vec![0.0; P];
let (mut mu, mut p_nb, mut p_lg) = ([0.0; C], [0.0; C], [0.0; C]);
let mut mu = [0.0; C];
for c in 0..C {
features::row(p, a, c, t0, t0 + h, &mut x);
if overrides {
if let Some(e) = entry {
let (r6, r26) = match e { Some(w) => ((-w / 6.0).exp(), (-w / 26.0).exp()), None => (0.0, 0.0) };
x[12] = r6; x[13] = r26;
}
if let Some(r) = t40 { x[10] = if r >= 1.0 && r <= 40.0 { (41.0 - r) / 40.0 } else { 0.0 }; }
if let Some(r) = air { x[11] = if r >= 1.0 && r <= 20.0 { (21.0 - r) / 20.0 } else { 0.0 }; }
}
features::row(p, a, c, t0, t0 + h.max(1), &mut x);
if overrides { over(&mut x, CHART_IDX_CHANNEL); }
mu[c] = glm::dot(&d.nb, &x).clamp(-30.0, 12.0).exp();
p_nb[c] = glm::nb_tail(mu[c], d.alpha, m.tau[c]);
p_lg[c] = glm::sigmoid(glm::dot(&d.logit, &x));
}
let sims = 4000;
let shape = 1.0 / d.alpha;
let mut rng = Rng(0xa1 ^ a as u64);
let (mut any, mut tot) = (0usize, Vec::with_capacity(sims));
let mut tot = Vec::with_capacity(sims);
for _ in 0..sims {
let g = rng.gamma(shape) / shape;
let (mut s, mut hit) = (0.0, false);
for c in 0..C { let y = rng.poisson(mu[c] * g); s += y; hit |= y >= m.tau[c] as f64; }
any += hit as usize; tot.push(s);
tot.push((0..C).map(|c| rng.poisson(mu[c] * g)).sum::<f64>());
}
tot.sort_by(f64::total_cmp);
let ch = |v: &[f64; C]| CHANNELS.iter().zip(v).map(|(c, p)| (c.to_string(), json!(p))).collect::<serde_json::Map<_, _>>();
json!({"p_any": any as f64 / sims as f64, "mean": mu.iter().sum::<f64>(), "lo": tot[sims / 10], "hi": tot[sims * 9 / 10],
"p": ch(&p_nb), "p_logit": ch(&p_lg), "plays": ch(&mu)})
json!({"add_any": add_any, "add": ch(&add_c), "mean": mu.iter().sum::<f64>(), "lo": tot[sims / 10], "hi": tot[sims * 9 / 10], "plays": ch(&mu)})
};
let now: serde_json::Map<String, Value> = (0..C).filter(|&c| m.rot.on(a, c, t0)).map(|c| (CHANNELS[c].to_string(), json!(p.y(a, c, t0)))).collect();
ok(json!({
"h": h, "week": (p.weeks[t0] + chrono::Duration::weeks(h as i64)).to_string(),
"h": h, "window_start": (p.weeks[t0] + chrono::Duration::weeks(h as i64 + 1)).to_string(),
"current": {"t40": chart.t40_rank, "air": chart.air_rank,
"entry": if chart.weeks_since_entry == u16::MAX { Value::Null } else { json!(chart.weeks_since_entry) },
"t40_hits": chart.t40_hits, "air_hits": chart.air_hits},
"t40_hits": chart.t40_hits, "air_hits": chart.air_hits, "in_rotation": now},
"base": run(false), "scenario": run(true),
}))
}