//! The ranking, version 0 (doc: "The ranking calculation"). For every artist, horizon h = 1–12 //! weeks after the latest complete week and DR channel: P(in rotation), expected plays, and, //! across channels, P(in rotation on at least one) and total plays with an 80% interval. //! //! The artist's state is drawn once per simulation run and shared by all channels (a common //! gamma frailty, so each channel's marginal stays the fitted NB), so channels aren't treated as //! independent. Not yet conditioned on a release at t + h: that needs the release-effect curve //! from release dates (Deezer backfill). Until then this ranks expected airplay given current form. use super::adds::{AddModel, Rot, Scenario}; use super::common::{thresholds, Direct, Rng}; use super::features::P; use super::panel::{Panel, C, CHANNELS}; use crate::db::Db; use anyhow::Result; use chrono::{Duration, Utc}; use rusqlite::params; pub const VERSION: &str = "dr-nb-v0"; pub struct Line { pub artist: usize, pub p_channel: [f64; C], pub mean_channel: [f64; C], pub p_any: f64, pub mean: f64, pub lo: f64, pub hi: f64, } pub fn create_tables(db: &Db) -> Result<()> { db.conn.execute_batch( "CREATE TABLE IF NOT EXISTS forecasts ( made_at TEXT NOT NULL, model_version TEXT NOT NULL, target TEXT NOT NULL, -- 'rotation' (prob, mean plays) per channel, 'any' across channels; -- 'add' / 'add_any': P(new add in the 8 weeks from target_week), blended -- by the artist's chance of releasing ('p_release'); 'add_rel' / -- 'add_any_rel' / 'add_rel': if the artist releases a NEW song (release-event model), -- 'add_any_nmf': ...placed on NMF Denmark; 'recent_release': released in the last 6 weeks; -- 'now': in rotation in the origin week (prob 1/0, mean = plays) key TEXT NOT NULL, -- '{dr artist id}:{channel}' or '{dr artist id}:any' horizon INTEGER NOT NULL, -- weeks after origin_week origin_week TEXT NOT NULL, target_week TEXT NOT NULL, prob REAL, mean REAL, lo REAL, hi REAL, PRIMARY KEY (made_at, model_version, target, key, horizon) ); CREATE TABLE IF NOT EXISTS backtest_scores ( run_at TEXT NOT NULL, model TEXT NOT NULL, horizon INTEGER NOT NULL, metric TEXT NOT NULL, value REAL, PRIMARY KEY (run_at, model, horizon, metric) );", )?; Ok(()) } pub fn forecast(p: &Panel, h: usize, sims: usize) -> Vec { let t0 = p.w() - 1; let tau = thresholds(p, p.w()); let d = Direct::fit(p, h, t0, &tau, false); let mut x = vec![0.0; P]; let mut rng = Rng(0x5eed ^ h as u64); let shape = 1.0 / d.alpha; (0..p.a()).map(|a| { let mut mean_channel = [0.0; C]; let mut p_channel = [0.0; C]; for c in 0..C { let (mu, pr, _) = d.predict(p, a, c, t0, tau[c], &mut x); mean_channel[c] = mu; p_channel[c] = pr; } let mut totals = Vec::with_capacity(sims); let mut any = 0usize; for _ in 0..sims { let g = rng.gamma(shape) / shape; let (mut tot, mut hit) = (0.0, false); for c in 0..C { let y = rng.poisson(mean_channel[c] * g); tot += y; hit |= y >= tau[c] as f64; } any += hit as usize; totals.push(tot); } totals.sort_by(f64::total_cmp); let q = |f: f64| totals[((sims as f64 - 1.0) * f).round() as usize]; Line { artist: a, p_channel, mean_channel, p_any: any as f64 / sims as f64, mean: mean_channel.iter().sum(), lo: q(0.1), hi: q(0.9) } }).collect() } /// Forecasts for h = 1..=max_h, stored in `forecasts`; returns the lines of `show_h`. pub fn run(db: &Db, p: &Panel, max_h: usize, show_h: usize) -> Result> { create_tables(db)?; let made = Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string(); let origin = p.weeks[p.w() - 1]; let mut shown = Vec::new(); for h in 1..=max_h { let lines = forecast(p, h, 2000); let target = (origin + Duration::weeks(h as i64)).to_string(); let tx = db.conn.unchecked_transaction()?; { let mut ins = tx.prepare("INSERT OR REPLACE INTO forecasts VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)")?; for l in &lines { let id = &p.artists[l.artist].id; ins.execute(params![made, VERSION, "any", format!("{id}:any"), h as i64, origin.to_string(), target, l.p_any, l.mean, l.lo, l.hi])?; for c in 0..C { ins.execute(params![made, VERSION, "rotation", format!("{id}:{}", CHANNELS[c]), h as i64, origin.to_string(), target, l.p_channel[c], l.mean_channel[c], Option::::None, Option::::None])?; } } } tx.commit()?; eprintln!("rank: h={h} stored ({} artists)", lines.len()); if h == show_h { shown = lines; } } // New adds: windows of W weeks starting h = 0..=max_h weeks after the origin. let tau = thresholds(p, p.w()); let rot = Rot::new(p, &tau); // The release-event model ("if they release a new song, will DR add it?"), the same for every // window start: the artist's state now, the new song's NMF placement blended from history. // The site's default: added within 2 weeks of the release. let ev = AddModel::fit_events_pooled(p, &rot, p.w() - 1, 2); let t0 = p.w() - 1; // A new song released in week t0+h+1: time-since features moved forward to then; none if too // soon after the artist's last release. let event = |a: usize, h: usize| -> Option<([f64; C], f64, f64, f64)> { if super::adds::too_soon(p, a, t0, h) { return None; } let q = super::adds::p_nmf(p, a, t0); let (pcn, pan) = ev.predict_at(p, &rot, a, t0, h, Scenario::new(true, true).at(h), None); let (pcr, par) = ev.predict_at(p, &rot, a, t0, h, Scenario::new(true, false).at(h), None); Some((std::array::from_fn(|c| q * pcn[c] + (1.0 - q) * pcr[c]), q * pan + (1.0 - q) * par, pan, q)) }; for h in 0..=max_h { let m = AddModel::fit(p, &rot, h, p.w() - 1); let start = (origin + Duration::weeks(h as i64 + 1)).to_string(); let tx = db.conn.unchecked_transaction()?; { let mut ins = tx.prepare("INSERT OR REPLACE INTO forecasts VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)")?; for a in 0..p.a() { let id = &p.artists[a].id; // Any add in the window (current songs included): no new release, or one at the usual rate. let (pc1, pa1) = m.predict(p, &rot, a, t0, Scenario::new(true, false), None); let (pc0, pa0) = m.predict(p, &rot, a, t0, Scenario::new(false, false), None); let pr = super::adds::p_release(p, a, t0); let mix = |p1: f64, p0: f64| match pr { Some(r) => r * p1 + (1.0 - r) * p0, None => p0 }; let ev_a = event(a, h); let none = Option::::None; let mut put = |target: &str, ch: &str, prob: f64| ins.execute(params![made, VERSION, target, format!("{id}:{ch}"), h as i64, origin.to_string(), start, prob, none, none, none]); put("add_any", "any", mix(pa1, pa0))?; match ev_a { Some((_, ea, ean, q)) => { put("add_any_rel", "any", ea)?; put("add_any_nmf", "any", ean)?; put("p_nmf", "any", q)?; } None => { put("too_soon", "any", 1.0)?; } } if let Some(r) = pr { put("p_release", "any", r)?; } for c in 0..C { put("add", CHANNELS[c], mix(pc1[c], pc0[c]))?; if let Some((ec, ..)) = ev_a { put("add_rel", CHANNELS[c], ec[c])?; } } } } tx.commit()?; eprintln!("rank: adds h={h} stored"); } // Where each artist stands in the origin week: prob = 1 if in rotation, mean = plays. let tx = db.conn.unchecked_transaction()?; { let mut ins = tx.prepare("INSERT OR REPLACE INTO forecasts VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)")?; let t0 = p.w() - 1; for a in 0..p.a() { let id = &p.artists[a].id; for c in 0..C { let y = p.y(a, c, t0); if y == 0.0 { continue; } ins.execute(params![made, VERSION, "now", format!("{id}:{}", CHANNELS[c]), 0i64, origin.to_string(), origin.to_string(), rot.on(a, c, t0) as i32 as f64, y, Option::::None, Option::::None])?; } } } tx.commit()?; Ok(shown) }