Playground site: airplay serve (ranking, artist history and forecast, what-if on chart situation, backtest summary) at idiotmaxxing.com/airplay via a Traefik file route; daily collection refreshes the forecasts
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+333
@@ -0,0 +1,333 @@
|
||||
//! The playground site: `airplay serve [--addr 127.0.0.1:8790] [--prefix /airplay]`.
|
||||
//!
|
||||
//! {prefix}/ the page (src/web/index.html)
|
||||
//! {prefix}/api/meta latest forecast run, horizons, channels, backtest scores
|
||||
//! {prefix}/api/rank?h=4 every artist's forecast for one horizon
|
||||
//! {prefix}/api/artist?id= one artist: profile, 51 weeks of plays per channel, forecasts, chart runs
|
||||
//! {prefix}/api/whatif?id=&h=&entry=&t40=&air=
|
||||
//! re-run the model for one artist with a changed chart situation
|
||||
//!
|
||||
//! Rankings are read from the forecasts table (`airplay model rank`); the what-if models are
|
||||
//! fitted in the background at start and again whenever a newer forecast run appears.
|
||||
|
||||
use crate::model::common::{thresholds, Direct, Rng};
|
||||
use crate::model::features::{self, P};
|
||||
use crate::model::glm;
|
||||
use crate::model::panel::{Panel, C, CHANNELS};
|
||||
use crate::model::rank::VERSION;
|
||||
use anyhow::Result;
|
||||
use rusqlite::{params, Connection, OpenFlags, OptionalExtension};
|
||||
use serde_json::{json, Value};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, RwLock};
|
||||
use tiny_http::{Header, Method, Response, Server};
|
||||
|
||||
const PAGE: &str = include_str!("web/index.html");
|
||||
const MAX_H: usize = 12;
|
||||
|
||||
struct Models {
|
||||
made_at: String,
|
||||
panel: Panel,
|
||||
tau: [u32; C],
|
||||
direct: Vec<Direct>, // index h - 1
|
||||
index: HashMap<String, usize>,
|
||||
}
|
||||
|
||||
struct State {
|
||||
db_path: String,
|
||||
prefix: String,
|
||||
models: RwLock<Option<Arc<Models>>>,
|
||||
rank_cache: RwLock<HashMap<(String, usize), Arc<String>>>,
|
||||
}
|
||||
|
||||
fn ro(path: &str) -> Result<Connection> {
|
||||
let c = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX)?;
|
||||
c.busy_timeout(std::time::Duration::from_secs(10))?;
|
||||
Ok(c)
|
||||
}
|
||||
|
||||
fn latest_run(c: &Connection) -> Result<Option<String>> {
|
||||
Ok(c.query_row("SELECT max(made_at) FROM forecasts WHERE model_version = ?1", [VERSION], |r| r.get(0)).optional()?.flatten())
|
||||
}
|
||||
|
||||
pub fn serve(db_path: &str, args: &[String]) -> Result<()> {
|
||||
let arg = |n: &str| args.iter().position(|a| a == n).and_then(|i| args.get(i + 1)).cloned();
|
||||
let addr = arg("--addr").unwrap_or_else(|| "127.0.0.1:8790".into());
|
||||
let prefix = arg("--prefix").unwrap_or_else(|| "/airplay".into()).trim_end_matches('/').to_string();
|
||||
let state = Arc::new(State { db_path: db_path.into(), prefix, models: RwLock::new(None), rank_cache: RwLock::new(HashMap::new()) });
|
||||
|
||||
// Keep the what-if models in step with the latest forecast run.
|
||||
{
|
||||
let state = state.clone();
|
||||
std::thread::spawn(move || loop {
|
||||
let current = state.models.read().unwrap().as_ref().map(|m| m.made_at.clone());
|
||||
let latest = ro(&state.db_path).ok().and_then(|c| latest_run(&c).ok().flatten());
|
||||
if latest.is_some() && latest != current {
|
||||
match fit_models(&state.db_path, latest.clone().unwrap()) {
|
||||
Ok(m) => { *state.models.write().unwrap() = Some(Arc::new(m)); eprintln!("what-if models ready ({})", latest.unwrap()); }
|
||||
Err(e) => eprintln!("what-if models: {e}"),
|
||||
}
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_secs(600));
|
||||
});
|
||||
}
|
||||
|
||||
let server = Arc::new(Server::http(&addr).map_err(|e| anyhow::anyhow!("{addr}: {e}"))?);
|
||||
eprintln!("serving on http://{addr}{}/", state.prefix);
|
||||
let workers: Vec<_> = (0..4).map(|_| {
|
||||
let (server, state) = (server.clone(), state.clone());
|
||||
std::thread::spawn(move || {
|
||||
for req in server.incoming_requests() {
|
||||
let (status, ctype, body) = match handle(&state, req.method(), req.url()) {
|
||||
Ok(r) => r,
|
||||
Err(e) => (500, "application/json", Arc::new(json!({"error": e.to_string()}).to_string())),
|
||||
};
|
||||
let mut resp = Response::from_data(body.as_bytes().to_vec()).with_status_code(status)
|
||||
.with_header(Header::from_bytes("Content-Type", ctype).unwrap())
|
||||
.with_header(Header::from_bytes("X-Robots-Tag", "noindex").unwrap());
|
||||
if status == 301 {
|
||||
resp = resp.with_header(Header::from_bytes("Location", format!("{}/", state.prefix)).unwrap());
|
||||
}
|
||||
let _ = req.respond(resp);
|
||||
}
|
||||
})
|
||||
}).collect();
|
||||
for w in workers { let _ = w.join(); }
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn fit_models(db_path: &str, made_at: String) -> Result<Models> {
|
||||
let db = crate::db::Db::open(db_path)?;
|
||||
let panel = Panel::load(&db)?;
|
||||
let tau = thresholds(&panel, panel.w());
|
||||
let direct = (1..=MAX_H).map(|h| Direct::fit(&panel, h, panel.w() - 1, &tau, true)).collect();
|
||||
let index = panel.artists.iter().enumerate().map(|(i, a)| (a.id.clone(), i)).collect();
|
||||
Ok(Models { made_at, panel, tau, direct, index })
|
||||
}
|
||||
|
||||
fn query(url: &str) -> HashMap<String, String> {
|
||||
let q = url.split_once('?').map(|x| x.1).unwrap_or("");
|
||||
q.split('&').filter_map(|kv| {
|
||||
let (k, v) = kv.split_once('=').unwrap_or((kv, ""));
|
||||
Some((decode(k), decode(v)))
|
||||
}).collect()
|
||||
}
|
||||
|
||||
fn decode(s: &str) -> String {
|
||||
let b = s.as_bytes();
|
||||
let mut out = Vec::with_capacity(b.len());
|
||||
let mut i = 0;
|
||||
while i < b.len() {
|
||||
match b[i] {
|
||||
b'+' => out.push(b' '),
|
||||
b'%' if i + 2 < b.len() => {
|
||||
if let Ok(v) = u8::from_str_radix(std::str::from_utf8(&b[i + 1..i + 3]).unwrap_or("zz"), 16) { out.push(v); i += 2; } else { out.push(b'%'); }
|
||||
}
|
||||
c => out.push(c),
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
String::from_utf8_lossy(&out).into_owned()
|
||||
}
|
||||
|
||||
type Reply = (u16, &'static str, Arc<String>);
|
||||
|
||||
fn ok(v: Value) -> Result<Reply> { Ok((200, "application/json", Arc::new(v.to_string()))) }
|
||||
|
||||
fn handle(st: &State, method: &Method, url: &str) -> Result<Reply> {
|
||||
if *method != Method::Get && *method != Method::Head { return Ok((405, "text/plain", Arc::new("GET only".into()))); }
|
||||
let path = url.split('?').next().unwrap_or("");
|
||||
let Some(rest) = path.strip_prefix(&st.prefix) else { return Ok((404, "text/plain", Arc::new("not found".into()))) };
|
||||
let q = query(url);
|
||||
match rest {
|
||||
"" => Ok((301, "text/plain", Arc::new(String::new()))),
|
||||
"/" | "/index.html" => Ok((200, "text/html; charset=utf-8", Arc::new(PAGE.to_string()))),
|
||||
"/api/meta" => meta(st),
|
||||
"/api/rank" => rank(st, q.get("h").and_then(|h| h.parse().ok()).unwrap_or(4).clamp(1, MAX_H)),
|
||||
"/api/artist" => artist(st, q.get("id").map(String::as_str).unwrap_or("")),
|
||||
"/api/whatif" => whatif(st, &q),
|
||||
_ => Ok((404, "text/plain", Arc::new("not found".into()))),
|
||||
}
|
||||
}
|
||||
|
||||
fn meta(st: &State) -> Result<Reply> {
|
||||
let c = ro(&st.db_path)?;
|
||||
let Some(made) = latest_run(&c)? else { return ok(json!({"ready": false})) };
|
||||
let (origin,): (String,) = c.query_row("SELECT origin_week FROM forecasts WHERE made_at = ?1 LIMIT 1", [&made], |r| Ok((r.get(0)?,)))?;
|
||||
let horizons: Vec<Value> = c.prepare("SELECT DISTINCT horizon, target_week FROM forecasts WHERE made_at = ?1 ORDER BY horizon")?
|
||||
.query_map([&made], |r| Ok(json!({"h": r.get::<_, i64>(0)?, "week": r.get::<_, String>(1)?})))?.collect::<Result<_, _>>()?;
|
||||
let run: Option<String> = c.query_row("SELECT max(run_at) FROM backtest_scores", [], |r| r.get(0)).optional()?.flatten();
|
||||
let scores: Vec<Value> = match &run {
|
||||
Some(run) => c.prepare("SELECT model, horizon, metric, value FROM backtest_scores WHERE run_at = ?1")?
|
||||
.query_map([run], |r| Ok(json!({"model": r.get::<_, String>(0)?, "h": r.get::<_, i64>(1)?, "metric": r.get::<_, String>(2)?, "value": r.get::<_, Option<f64>>(3)?})))?
|
||||
.collect::<Result<_, _>>()?,
|
||||
None => Vec::new(),
|
||||
};
|
||||
let models = st.models.read().unwrap();
|
||||
let tau = models.as_ref().map(|m| m.tau.to_vec());
|
||||
ok(json!({"ready": true, "made_at": made, "origin_week": origin, "model": VERSION, "horizons": horizons,
|
||||
"channels": CHANNELS, "tau": tau, "whatif_ready": models.is_some(), "backtest_run": run, "backtest": scores}))
|
||||
}
|
||||
|
||||
fn rank(st: &State, h: usize) -> Result<Reply> {
|
||||
let c = ro(&st.db_path)?;
|
||||
let Some(made) = latest_run(&c)? else { return ok(json!({"rows": []})) };
|
||||
if let Some(hit) = st.rank_cache.read().unwrap().get(&(made.clone(), h)) { return Ok((200, "application/json", hit.clone())); }
|
||||
let mut names: HashMap<String, (String, bool, bool)> = HashMap::new();
|
||||
for r in c.prepare("SELECT artist_id, name, json_extract(raw, '$.artist.country'), json_extract(raw, '$.artist.type') FROM dr_artists")?
|
||||
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, Option<String>>(2)?, r.get::<_, Option<String>>(3)?)))? {
|
||||
let (id, n, country, kind) = r?;
|
||||
names.insert(id, (n, country.as_deref() == Some("Danmark"), kind.is_some_and(|k| k != "person")));
|
||||
}
|
||||
let mut rows: HashMap<String, Value> = HashMap::new();
|
||||
let mut st_ = c.prepare("SELECT key, target, prob, mean, lo, hi FROM forecasts WHERE made_at = ?1 AND model_version = ?2 AND horizon = ?3")?;
|
||||
let it = st_.query_map(params![made, VERSION, h as i64], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, Option<f64>>(2)?,
|
||||
r.get::<_, Option<f64>>(3)?, r.get::<_, Option<f64>>(4)?, r.get::<_, Option<f64>>(5)?)))?;
|
||||
for r in it {
|
||||
let (key, target, prob, mean, lo, hi) = r?;
|
||||
let Some((id, ch)) = key.split_once(':') else { continue };
|
||||
let e = rows.entry(id.to_string()).or_insert_with(|| {
|
||||
let (n, dk, group) = names.get(id).cloned().unwrap_or_default();
|
||||
json!({"id": id, "name": n, "danish": dk, "group": group, "p": {}, "plays": {}})
|
||||
});
|
||||
if target == "any" {
|
||||
e["p_any"] = json!(prob); e["mean"] = json!(mean); e["lo"] = json!(lo); e["hi"] = json!(hi);
|
||||
} else {
|
||||
e["p"][ch] = json!(prob.map(|p| (p * 1000.0).round() / 1000.0));
|
||||
e["plays"][ch] = json!(mean.map(|m| (m * 10.0).round() / 10.0));
|
||||
}
|
||||
}
|
||||
let mut out: Vec<Value> = rows.into_values().collect();
|
||||
out.sort_by(|a, b| b["mean"].as_f64().unwrap_or(0.0).total_cmp(&a["mean"].as_f64().unwrap_or(0.0)));
|
||||
let body = Arc::new(json!({"made_at": made, "h": h, "rows": out}).to_string());
|
||||
let mut cache = st.rank_cache.write().unwrap();
|
||||
cache.retain(|k, _| k.0 == made);
|
||||
cache.insert((made, h), body.clone());
|
||||
Ok((200, "application/json", body))
|
||||
}
|
||||
|
||||
fn artist(st: &State, id: &str) -> Result<Reply> {
|
||||
let c = ro(&st.db_path)?;
|
||||
let Some((name, raw)) = c.query_row("SELECT name, raw FROM dr_artists WHERE artist_id = ?1", [id], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?))).optional()?
|
||||
else { return Ok((404, "application/json", Arc::new(json!({"error": "unknown artist"}).to_string()))) };
|
||||
let v: Value = serde_json::from_str(&raw).unwrap_or(Value::Null);
|
||||
let a = &v["artist"];
|
||||
let image = a["images"][0]["sizes"].as_array().and_then(|s| s.iter().find(|x| x["size"] == "medium")).and_then(|x| x["source"].as_str()).map(str::to_string);
|
||||
let top: Vec<Value> = v["mostPlayed"].as_array().into_iter().flatten().take(8)
|
||||
.map(|t| json!({"title": t["title"], "total": t["total"]})).collect();
|
||||
|
||||
// Weekly plays per channel, in the panel's weeks when the models are loaded.
|
||||
// Complete weeks only (as in the panel): ended before the first artist page was read.
|
||||
let mut weeks: Vec<String> = c.prepare(
|
||||
"SELECT DISTINCT week_start FROM dr_artist_weeks
|
||||
WHERE date(week_start, '+7 days') <= (SELECT substr(min(fetched_at), 1, 10) FROM dr_artists) ORDER BY week_start")?
|
||||
.query_map([], |r| r.get(0))?.collect::<Result<_, _>>()?;
|
||||
if let Some(m) = st.models.read().unwrap().as_ref() { weeks = m.panel.weeks.iter().map(|d| d.to_string()).collect(); }
|
||||
let widx: HashMap<&str, usize> = weeks.iter().enumerate().map(|(i, w)| (w.as_str(), i)).collect();
|
||||
let mut plays: HashMap<&str, Vec<i64>> = CHANNELS.iter().map(|c| (*c, vec![0; weeks.len()])).collect();
|
||||
for r in c.prepare("SELECT week_start, channel, plays FROM dr_artist_weeks WHERE artist_id = ?1")?
|
||||
.query_map([id], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, i64>(2)?)))? {
|
||||
let (w, ch, n) = r?;
|
||||
if let (Some(&i), Some(v)) = (widx.get(w.as_str()), plays.get_mut(ch.as_str())) { v[i] = n; }
|
||||
}
|
||||
|
||||
let mut fc: Vec<Value> = Vec::new();
|
||||
if let Some(made) = latest_run(&c)? {
|
||||
let mut byh: HashMap<i64, Value> = HashMap::new();
|
||||
for r in c.prepare("SELECT horizon, target_week, key, prob, mean, lo, hi FROM forecasts WHERE made_at = ?1 AND model_version = ?2 AND key LIKE ?3")?
|
||||
.query_map(params![made, VERSION, format!("{id}:%")], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?,
|
||||
r.get::<_, Option<f64>>(3)?, r.get::<_, Option<f64>>(4)?, r.get::<_, Option<f64>>(5)?, r.get::<_, Option<f64>>(6)?)))? {
|
||||
let (h, week, key, prob, mean, lo, hi) = r?;
|
||||
let e = byh.entry(h).or_insert_with(|| json!({"h": h, "week": week, "p": {}, "plays": {}}));
|
||||
let ch = key.split_once(':').map(|x| x.1).unwrap_or("");
|
||||
if ch == "any" { e["p_any"] = json!(prob); e["mean"] = json!(mean); e["lo"] = json!(lo); e["hi"] = json!(hi); }
|
||||
else { e["p"][ch] = json!(prob); e["plays"][ch] = json!(mean); }
|
||||
}
|
||||
let mut v: Vec<(i64, Value)> = byh.into_iter().collect();
|
||||
v.sort_by_key(|x| x.0);
|
||||
fc = v.into_iter().map(|x| x.1).collect();
|
||||
}
|
||||
|
||||
// Chart runs: rows whose artist field names this artist (same matching as the panel, roughly).
|
||||
let key = crate::model::panel::key(&name);
|
||||
let mut charts: Vec<Value> = Vec::new();
|
||||
if key.chars().count() >= 3 {
|
||||
let like = format!("%{}%", name.to_uppercase());
|
||||
for r in c.prepare("SELECT chart, year, week, position, artist, title FROM hitlisten_charts
|
||||
WHERE chart IN ('t40', 'airplay') AND upper(artist) LIKE ?1 ORDER BY year DESC, week DESC LIMIT 400")?
|
||||
.query_map([like], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?, r.get::<_, i64>(2)?, r.get::<_, i64>(3)?, r.get::<_, String>(4)?, r.get::<_, String>(5)?)))? {
|
||||
let (chart, y, w, pos, art, title) = r?;
|
||||
let names: Vec<String> = art.to_lowercase().split(&[',', '&'][..]).flat_map(|p| p.split(" feat").map(str::to_string).collect::<Vec<_>>())
|
||||
.flat_map(|p| p.split(" x ").map(crate::model::panel::key).collect::<Vec<_>>()).collect();
|
||||
if !names.iter().any(|n| *n == key) && crate::model::panel::key(&art) != key { continue; }
|
||||
charts.push(json!({"chart": chart, "year": y, "week": w, "position": pos, "title": title}));
|
||||
}
|
||||
}
|
||||
|
||||
ok(json!({
|
||||
"id": id, "name": name, "country": a["country"], "gender": a["gender"], "description": a["description"],
|
||||
"debut": a["lifespan"]["beginYear"], "type": a["type"], "image": image, "top_tracks": top,
|
||||
"weeks": weeks, "plays": plays, "forecasts": fc, "charts": charts,
|
||||
}))
|
||||
}
|
||||
|
||||
/// One artist's forecast under a changed chart situation. Overrides (all optional):
|
||||
/// entry weeks since a new chart entry (a release that charted), "none" = never
|
||||
/// t40 position on the latest Track Top-40 (0 = not on it)
|
||||
/// air position on the latest Airplay Top-20 (0 = not on it)
|
||||
fn whatif(st: &State, q: &HashMap<String, String>) -> Result<Reply> {
|
||||
let guard = st.models.read().unwrap();
|
||||
let Some(m) = guard.as_ref() else { return Ok((503, "application/json", Arc::new(json!({"loading": true}).to_string()))) };
|
||||
let Some(&a) = m.index.get(q.get("id").map(String::as_str).unwrap_or("")) else {
|
||||
return Ok((404, "application/json", Arc::new(json!({"error": "artist not in the model"}).to_string())));
|
||||
};
|
||||
let h: usize = q.get("h").and_then(|v| v.parse().ok()).unwrap_or(4).clamp(1, MAX_H);
|
||||
let d = &m.direct[h - 1];
|
||||
let p = &m.panel;
|
||||
let t0 = p.w() - 1;
|
||||
let chart = p.chart(a, t0);
|
||||
let num = |k: &str| q.get(k).filter(|v| !v.is_empty()).and_then(|v| v.parse::<f64>().ok());
|
||||
let entry: Option<Option<f64>> = q.get("entry").filter(|v| !v.is_empty()).map(|v| if v == "none" { None } else { v.parse().ok() });
|
||||
let t40 = num("t40");
|
||||
let air = num("air");
|
||||
|
||||
let run = |overrides: bool| -> Value {
|
||||
let mut x = vec![0.0; P];
|
||||
let (mut mu, mut p_nb, mut p_lg) = ([0.0; C], [0.0; C], [0.0; C]);
|
||||
for c in 0..C {
|
||||
features::row(p, a, c, t0, t0 + h, &mut x);
|
||||
if overrides {
|
||||
if let Some(e) = entry {
|
||||
let (r6, r26) = match e { Some(w) => ((-w / 6.0).exp(), (-w / 26.0).exp()), None => (0.0, 0.0) };
|
||||
x[12] = r6; x[13] = r26;
|
||||
}
|
||||
if let Some(r) = t40 { x[10] = if r >= 1.0 && r <= 40.0 { (41.0 - r) / 40.0 } else { 0.0 }; }
|
||||
if let Some(r) = air { x[11] = if r >= 1.0 && r <= 20.0 { (21.0 - r) / 20.0 } else { 0.0 }; }
|
||||
}
|
||||
mu[c] = glm::dot(&d.nb, &x).clamp(-30.0, 12.0).exp();
|
||||
p_nb[c] = glm::nb_tail(mu[c], d.alpha, m.tau[c]);
|
||||
p_lg[c] = glm::sigmoid(glm::dot(&d.logit, &x));
|
||||
}
|
||||
let sims = 4000;
|
||||
let shape = 1.0 / d.alpha;
|
||||
let mut rng = Rng(0xa1 ^ a as u64);
|
||||
let (mut any, mut tot) = (0usize, Vec::with_capacity(sims));
|
||||
for _ in 0..sims {
|
||||
let g = rng.gamma(shape) / shape;
|
||||
let (mut s, mut hit) = (0.0, false);
|
||||
for c in 0..C { let y = rng.poisson(mu[c] * g); s += y; hit |= y >= m.tau[c] as f64; }
|
||||
any += hit as usize; tot.push(s);
|
||||
}
|
||||
tot.sort_by(f64::total_cmp);
|
||||
let ch = |v: &[f64; C]| CHANNELS.iter().zip(v).map(|(c, p)| (c.to_string(), json!(p))).collect::<serde_json::Map<_, _>>();
|
||||
json!({"p_any": any as f64 / sims as f64, "mean": mu.iter().sum::<f64>(), "lo": tot[sims / 10], "hi": tot[sims * 9 / 10],
|
||||
"p": ch(&p_nb), "p_logit": ch(&p_lg), "plays": ch(&mu)})
|
||||
};
|
||||
ok(json!({
|
||||
"h": h, "week": (p.weeks[t0] + chrono::Duration::weeks(h as i64)).to_string(),
|
||||
"current": {"t40": chart.t40_rank, "air": chart.air_rank,
|
||||
"entry": if chart.weeks_since_entry == u16::MAX { Value::Null } else { json!(chart.weeks_since_entry) },
|
||||
"t40_hits": chart.t40_hits, "air_hits": chart.air_hits},
|
||||
"base": run(false), "scenario": run(true),
|
||||
}))
|
||||
}
|
||||
Reference in New Issue
Block a user