Playground site: airplay serve (ranking, artist history and forecast, what-if on chart situation, backtest summary) at idiotmaxxing.com/airplay via a Traefik file route; daily collection refreshes the forecasts

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-15 02:21:41 +00:00
co-authored by Claude Opus 5.5
parent eededc7571
commit 387502ebfa
9 changed files with 806 additions and 2 deletions
+333
View File
@@ -0,0 +1,333 @@
//! The playground site: `airplay serve [--addr 127.0.0.1:8790] [--prefix /airplay]`.
//!
//! {prefix}/ the page (src/web/index.html)
//! {prefix}/api/meta latest forecast run, horizons, channels, backtest scores
//! {prefix}/api/rank?h=4 every artist's forecast for one horizon
//! {prefix}/api/artist?id= one artist: profile, 51 weeks of plays per channel, forecasts, chart runs
//! {prefix}/api/whatif?id=&h=&entry=&t40=&air=
//! re-run the model for one artist with a changed chart situation
//!
//! Rankings are read from the forecasts table (`airplay model rank`); the what-if models are
//! fitted in the background at start and again whenever a newer forecast run appears.
use crate::model::common::{thresholds, Direct, Rng};
use crate::model::features::{self, P};
use crate::model::glm;
use crate::model::panel::{Panel, C, CHANNELS};
use crate::model::rank::VERSION;
use anyhow::Result;
use rusqlite::{params, Connection, OpenFlags, OptionalExtension};
use serde_json::{json, Value};
use std::collections::HashMap;
use std::sync::{Arc, RwLock};
use tiny_http::{Header, Method, Response, Server};
const PAGE: &str = include_str!("web/index.html");
const MAX_H: usize = 12;
struct Models {
made_at: String,
panel: Panel,
tau: [u32; C],
direct: Vec<Direct>, // index h - 1
index: HashMap<String, usize>,
}
struct State {
db_path: String,
prefix: String,
models: RwLock<Option<Arc<Models>>>,
rank_cache: RwLock<HashMap<(String, usize), Arc<String>>>,
}
fn ro(path: &str) -> Result<Connection> {
let c = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX)?;
c.busy_timeout(std::time::Duration::from_secs(10))?;
Ok(c)
}
fn latest_run(c: &Connection) -> Result<Option<String>> {
Ok(c.query_row("SELECT max(made_at) FROM forecasts WHERE model_version = ?1", [VERSION], |r| r.get(0)).optional()?.flatten())
}
pub fn serve(db_path: &str, args: &[String]) -> Result<()> {
let arg = |n: &str| args.iter().position(|a| a == n).and_then(|i| args.get(i + 1)).cloned();
let addr = arg("--addr").unwrap_or_else(|| "127.0.0.1:8790".into());
let prefix = arg("--prefix").unwrap_or_else(|| "/airplay".into()).trim_end_matches('/').to_string();
let state = Arc::new(State { db_path: db_path.into(), prefix, models: RwLock::new(None), rank_cache: RwLock::new(HashMap::new()) });
// Keep the what-if models in step with the latest forecast run.
{
let state = state.clone();
std::thread::spawn(move || loop {
let current = state.models.read().unwrap().as_ref().map(|m| m.made_at.clone());
let latest = ro(&state.db_path).ok().and_then(|c| latest_run(&c).ok().flatten());
if latest.is_some() && latest != current {
match fit_models(&state.db_path, latest.clone().unwrap()) {
Ok(m) => { *state.models.write().unwrap() = Some(Arc::new(m)); eprintln!("what-if models ready ({})", latest.unwrap()); }
Err(e) => eprintln!("what-if models: {e}"),
}
}
std::thread::sleep(std::time::Duration::from_secs(600));
});
}
let server = Arc::new(Server::http(&addr).map_err(|e| anyhow::anyhow!("{addr}: {e}"))?);
eprintln!("serving on http://{addr}{}/", state.prefix);
let workers: Vec<_> = (0..4).map(|_| {
let (server, state) = (server.clone(), state.clone());
std::thread::spawn(move || {
for req in server.incoming_requests() {
let (status, ctype, body) = match handle(&state, req.method(), req.url()) {
Ok(r) => r,
Err(e) => (500, "application/json", Arc::new(json!({"error": e.to_string()}).to_string())),
};
let mut resp = Response::from_data(body.as_bytes().to_vec()).with_status_code(status)
.with_header(Header::from_bytes("Content-Type", ctype).unwrap())
.with_header(Header::from_bytes("X-Robots-Tag", "noindex").unwrap());
if status == 301 {
resp = resp.with_header(Header::from_bytes("Location", format!("{}/", state.prefix)).unwrap());
}
let _ = req.respond(resp);
}
})
}).collect();
for w in workers { let _ = w.join(); }
Ok(())
}
fn fit_models(db_path: &str, made_at: String) -> Result<Models> {
let db = crate::db::Db::open(db_path)?;
let panel = Panel::load(&db)?;
let tau = thresholds(&panel, panel.w());
let direct = (1..=MAX_H).map(|h| Direct::fit(&panel, h, panel.w() - 1, &tau, true)).collect();
let index = panel.artists.iter().enumerate().map(|(i, a)| (a.id.clone(), i)).collect();
Ok(Models { made_at, panel, tau, direct, index })
}
fn query(url: &str) -> HashMap<String, String> {
let q = url.split_once('?').map(|x| x.1).unwrap_or("");
q.split('&').filter_map(|kv| {
let (k, v) = kv.split_once('=').unwrap_or((kv, ""));
Some((decode(k), decode(v)))
}).collect()
}
fn decode(s: &str) -> String {
let b = s.as_bytes();
let mut out = Vec::with_capacity(b.len());
let mut i = 0;
while i < b.len() {
match b[i] {
b'+' => out.push(b' '),
b'%' if i + 2 < b.len() => {
if let Ok(v) = u8::from_str_radix(std::str::from_utf8(&b[i + 1..i + 3]).unwrap_or("zz"), 16) { out.push(v); i += 2; } else { out.push(b'%'); }
}
c => out.push(c),
}
i += 1;
}
String::from_utf8_lossy(&out).into_owned()
}
type Reply = (u16, &'static str, Arc<String>);
fn ok(v: Value) -> Result<Reply> { Ok((200, "application/json", Arc::new(v.to_string()))) }
fn handle(st: &State, method: &Method, url: &str) -> Result<Reply> {
if *method != Method::Get && *method != Method::Head { return Ok((405, "text/plain", Arc::new("GET only".into()))); }
let path = url.split('?').next().unwrap_or("");
let Some(rest) = path.strip_prefix(&st.prefix) else { return Ok((404, "text/plain", Arc::new("not found".into()))) };
let q = query(url);
match rest {
"" => Ok((301, "text/plain", Arc::new(String::new()))),
"/" | "/index.html" => Ok((200, "text/html; charset=utf-8", Arc::new(PAGE.to_string()))),
"/api/meta" => meta(st),
"/api/rank" => rank(st, q.get("h").and_then(|h| h.parse().ok()).unwrap_or(4).clamp(1, MAX_H)),
"/api/artist" => artist(st, q.get("id").map(String::as_str).unwrap_or("")),
"/api/whatif" => whatif(st, &q),
_ => Ok((404, "text/plain", Arc::new("not found".into()))),
}
}
fn meta(st: &State) -> Result<Reply> {
let c = ro(&st.db_path)?;
let Some(made) = latest_run(&c)? else { return ok(json!({"ready": false})) };
let (origin,): (String,) = c.query_row("SELECT origin_week FROM forecasts WHERE made_at = ?1 LIMIT 1", [&made], |r| Ok((r.get(0)?,)))?;
let horizons: Vec<Value> = c.prepare("SELECT DISTINCT horizon, target_week FROM forecasts WHERE made_at = ?1 ORDER BY horizon")?
.query_map([&made], |r| Ok(json!({"h": r.get::<_, i64>(0)?, "week": r.get::<_, String>(1)?})))?.collect::<Result<_, _>>()?;
let run: Option<String> = c.query_row("SELECT max(run_at) FROM backtest_scores", [], |r| r.get(0)).optional()?.flatten();
let scores: Vec<Value> = match &run {
Some(run) => c.prepare("SELECT model, horizon, metric, value FROM backtest_scores WHERE run_at = ?1")?
.query_map([run], |r| Ok(json!({"model": r.get::<_, String>(0)?, "h": r.get::<_, i64>(1)?, "metric": r.get::<_, String>(2)?, "value": r.get::<_, Option<f64>>(3)?})))?
.collect::<Result<_, _>>()?,
None => Vec::new(),
};
let models = st.models.read().unwrap();
let tau = models.as_ref().map(|m| m.tau.to_vec());
ok(json!({"ready": true, "made_at": made, "origin_week": origin, "model": VERSION, "horizons": horizons,
"channels": CHANNELS, "tau": tau, "whatif_ready": models.is_some(), "backtest_run": run, "backtest": scores}))
}
fn rank(st: &State, h: usize) -> Result<Reply> {
let c = ro(&st.db_path)?;
let Some(made) = latest_run(&c)? else { return ok(json!({"rows": []})) };
if let Some(hit) = st.rank_cache.read().unwrap().get(&(made.clone(), h)) { return Ok((200, "application/json", hit.clone())); }
let mut names: HashMap<String, (String, bool, bool)> = HashMap::new();
for r in c.prepare("SELECT artist_id, name, json_extract(raw, '$.artist.country'), json_extract(raw, '$.artist.type') FROM dr_artists")?
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, Option<String>>(2)?, r.get::<_, Option<String>>(3)?)))? {
let (id, n, country, kind) = r?;
names.insert(id, (n, country.as_deref() == Some("Danmark"), kind.is_some_and(|k| k != "person")));
}
let mut rows: HashMap<String, Value> = HashMap::new();
let mut st_ = c.prepare("SELECT key, target, prob, mean, lo, hi FROM forecasts WHERE made_at = ?1 AND model_version = ?2 AND horizon = ?3")?;
let it = st_.query_map(params![made, VERSION, h as i64], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, Option<f64>>(2)?,
r.get::<_, Option<f64>>(3)?, r.get::<_, Option<f64>>(4)?, r.get::<_, Option<f64>>(5)?)))?;
for r in it {
let (key, target, prob, mean, lo, hi) = r?;
let Some((id, ch)) = key.split_once(':') else { continue };
let e = rows.entry(id.to_string()).or_insert_with(|| {
let (n, dk, group) = names.get(id).cloned().unwrap_or_default();
json!({"id": id, "name": n, "danish": dk, "group": group, "p": {}, "plays": {}})
});
if target == "any" {
e["p_any"] = json!(prob); e["mean"] = json!(mean); e["lo"] = json!(lo); e["hi"] = json!(hi);
} else {
e["p"][ch] = json!(prob.map(|p| (p * 1000.0).round() / 1000.0));
e["plays"][ch] = json!(mean.map(|m| (m * 10.0).round() / 10.0));
}
}
let mut out: Vec<Value> = rows.into_values().collect();
out.sort_by(|a, b| b["mean"].as_f64().unwrap_or(0.0).total_cmp(&a["mean"].as_f64().unwrap_or(0.0)));
let body = Arc::new(json!({"made_at": made, "h": h, "rows": out}).to_string());
let mut cache = st.rank_cache.write().unwrap();
cache.retain(|k, _| k.0 == made);
cache.insert((made, h), body.clone());
Ok((200, "application/json", body))
}
fn artist(st: &State, id: &str) -> Result<Reply> {
let c = ro(&st.db_path)?;
let Some((name, raw)) = c.query_row("SELECT name, raw FROM dr_artists WHERE artist_id = ?1", [id], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?))).optional()?
else { return Ok((404, "application/json", Arc::new(json!({"error": "unknown artist"}).to_string()))) };
let v: Value = serde_json::from_str(&raw).unwrap_or(Value::Null);
let a = &v["artist"];
let image = a["images"][0]["sizes"].as_array().and_then(|s| s.iter().find(|x| x["size"] == "medium")).and_then(|x| x["source"].as_str()).map(str::to_string);
let top: Vec<Value> = v["mostPlayed"].as_array().into_iter().flatten().take(8)
.map(|t| json!({"title": t["title"], "total": t["total"]})).collect();
// Weekly plays per channel, in the panel's weeks when the models are loaded.
// Complete weeks only (as in the panel): ended before the first artist page was read.
let mut weeks: Vec<String> = c.prepare(
"SELECT DISTINCT week_start FROM dr_artist_weeks
WHERE date(week_start, '+7 days') <= (SELECT substr(min(fetched_at), 1, 10) FROM dr_artists) ORDER BY week_start")?
.query_map([], |r| r.get(0))?.collect::<Result<_, _>>()?;
if let Some(m) = st.models.read().unwrap().as_ref() { weeks = m.panel.weeks.iter().map(|d| d.to_string()).collect(); }
let widx: HashMap<&str, usize> = weeks.iter().enumerate().map(|(i, w)| (w.as_str(), i)).collect();
let mut plays: HashMap<&str, Vec<i64>> = CHANNELS.iter().map(|c| (*c, vec![0; weeks.len()])).collect();
for r in c.prepare("SELECT week_start, channel, plays FROM dr_artist_weeks WHERE artist_id = ?1")?
.query_map([id], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, i64>(2)?)))? {
let (w, ch, n) = r?;
if let (Some(&i), Some(v)) = (widx.get(w.as_str()), plays.get_mut(ch.as_str())) { v[i] = n; }
}
let mut fc: Vec<Value> = Vec::new();
if let Some(made) = latest_run(&c)? {
let mut byh: HashMap<i64, Value> = HashMap::new();
for r in c.prepare("SELECT horizon, target_week, key, prob, mean, lo, hi FROM forecasts WHERE made_at = ?1 AND model_version = ?2 AND key LIKE ?3")?
.query_map(params![made, VERSION, format!("{id}:%")], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?,
r.get::<_, Option<f64>>(3)?, r.get::<_, Option<f64>>(4)?, r.get::<_, Option<f64>>(5)?, r.get::<_, Option<f64>>(6)?)))? {
let (h, week, key, prob, mean, lo, hi) = r?;
let e = byh.entry(h).or_insert_with(|| json!({"h": h, "week": week, "p": {}, "plays": {}}));
let ch = key.split_once(':').map(|x| x.1).unwrap_or("");
if ch == "any" { e["p_any"] = json!(prob); e["mean"] = json!(mean); e["lo"] = json!(lo); e["hi"] = json!(hi); }
else { e["p"][ch] = json!(prob); e["plays"][ch] = json!(mean); }
}
let mut v: Vec<(i64, Value)> = byh.into_iter().collect();
v.sort_by_key(|x| x.0);
fc = v.into_iter().map(|x| x.1).collect();
}
// Chart runs: rows whose artist field names this artist (same matching as the panel, roughly).
let key = crate::model::panel::key(&name);
let mut charts: Vec<Value> = Vec::new();
if key.chars().count() >= 3 {
let like = format!("%{}%", name.to_uppercase());
for r in c.prepare("SELECT chart, year, week, position, artist, title FROM hitlisten_charts
WHERE chart IN ('t40', 'airplay') AND upper(artist) LIKE ?1 ORDER BY year DESC, week DESC LIMIT 400")?
.query_map([like], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?, r.get::<_, i64>(2)?, r.get::<_, i64>(3)?, r.get::<_, String>(4)?, r.get::<_, String>(5)?)))? {
let (chart, y, w, pos, art, title) = r?;
let names: Vec<String> = art.to_lowercase().split(&[',', '&'][..]).flat_map(|p| p.split(" feat").map(str::to_string).collect::<Vec<_>>())
.flat_map(|p| p.split(" x ").map(crate::model::panel::key).collect::<Vec<_>>()).collect();
if !names.iter().any(|n| *n == key) && crate::model::panel::key(&art) != key { continue; }
charts.push(json!({"chart": chart, "year": y, "week": w, "position": pos, "title": title}));
}
}
ok(json!({
"id": id, "name": name, "country": a["country"], "gender": a["gender"], "description": a["description"],
"debut": a["lifespan"]["beginYear"], "type": a["type"], "image": image, "top_tracks": top,
"weeks": weeks, "plays": plays, "forecasts": fc, "charts": charts,
}))
}
/// One artist's forecast under a changed chart situation. Overrides (all optional):
/// entry weeks since a new chart entry (a release that charted), "none" = never
/// t40 position on the latest Track Top-40 (0 = not on it)
/// air position on the latest Airplay Top-20 (0 = not on it)
fn whatif(st: &State, q: &HashMap<String, String>) -> Result<Reply> {
let guard = st.models.read().unwrap();
let Some(m) = guard.as_ref() else { return Ok((503, "application/json", Arc::new(json!({"loading": true}).to_string()))) };
let Some(&a) = m.index.get(q.get("id").map(String::as_str).unwrap_or("")) else {
return Ok((404, "application/json", Arc::new(json!({"error": "artist not in the model"}).to_string())));
};
let h: usize = q.get("h").and_then(|v| v.parse().ok()).unwrap_or(4).clamp(1, MAX_H);
let d = &m.direct[h - 1];
let p = &m.panel;
let t0 = p.w() - 1;
let chart = p.chart(a, t0);
let num = |k: &str| q.get(k).filter(|v| !v.is_empty()).and_then(|v| v.parse::<f64>().ok());
let entry: Option<Option<f64>> = q.get("entry").filter(|v| !v.is_empty()).map(|v| if v == "none" { None } else { v.parse().ok() });
let t40 = num("t40");
let air = num("air");
let run = |overrides: bool| -> Value {
let mut x = vec![0.0; P];
let (mut mu, mut p_nb, mut p_lg) = ([0.0; C], [0.0; C], [0.0; C]);
for c in 0..C {
features::row(p, a, c, t0, t0 + h, &mut x);
if overrides {
if let Some(e) = entry {
let (r6, r26) = match e { Some(w) => ((-w / 6.0).exp(), (-w / 26.0).exp()), None => (0.0, 0.0) };
x[12] = r6; x[13] = r26;
}
if let Some(r) = t40 { x[10] = if r >= 1.0 && r <= 40.0 { (41.0 - r) / 40.0 } else { 0.0 }; }
if let Some(r) = air { x[11] = if r >= 1.0 && r <= 20.0 { (21.0 - r) / 20.0 } else { 0.0 }; }
}
mu[c] = glm::dot(&d.nb, &x).clamp(-30.0, 12.0).exp();
p_nb[c] = glm::nb_tail(mu[c], d.alpha, m.tau[c]);
p_lg[c] = glm::sigmoid(glm::dot(&d.logit, &x));
}
let sims = 4000;
let shape = 1.0 / d.alpha;
let mut rng = Rng(0xa1 ^ a as u64);
let (mut any, mut tot) = (0usize, Vec::with_capacity(sims));
for _ in 0..sims {
let g = rng.gamma(shape) / shape;
let (mut s, mut hit) = (0.0, false);
for c in 0..C { let y = rng.poisson(mu[c] * g); s += y; hit |= y >= m.tau[c] as f64; }
any += hit as usize; tot.push(s);
}
tot.sort_by(f64::total_cmp);
let ch = |v: &[f64; C]| CHANNELS.iter().zip(v).map(|(c, p)| (c.to_string(), json!(p))).collect::<serde_json::Map<_, _>>();
json!({"p_any": any as f64 / sims as f64, "mean": mu.iter().sum::<f64>(), "lo": tot[sims / 10], "hi": tot[sims * 9 / 10],
"p": ch(&p_nb), "p_logit": ch(&p_lg), "plays": ch(&mu)})
};
ok(json!({
"h": h, "week": (p.weeks[t0] + chrono::Duration::weeks(h as i64)).to_string(),
"current": {"t40": chart.t40_rank, "air": chart.air_rank,
"entry": if chart.weeks_since_entry == u16::MAX { Value::Null } else { json!(chart.weeks_since_entry) },
"t40_hits": chart.t40_hits, "air_hits": chart.air_hits},
"base": run(false), "scenario": run(true),
}))
}