181 lines
8.9 KiB
Rust
181 lines
8.9 KiB
Rust
//! The ranking, version 0 (doc: "The ranking calculation"). For every artist, horizon h = 1–12
|
||
//! weeks after the latest complete week and DR channel: P(in rotation), expected plays, and,
|
||
//! across channels, P(in rotation on at least one) and total plays with an 80% interval.
|
||
//!
|
||
//! The artist's state is drawn once per simulation run and shared by all channels (a common
|
||
//! gamma frailty, so each channel's marginal stays the fitted NB), so channels aren't treated as
|
||
//! independent. Not yet conditioned on a release at t + h: that needs the release-effect curve
|
||
//! from release dates (Deezer backfill). Until then this ranks expected airplay given current form.
|
||
|
||
use super::adds::{AddModel, Rot, Scenario};
|
||
use super::common::{thresholds, Direct, Rng};
|
||
use super::features::P;
|
||
use super::panel::{Panel, C, CHANNELS};
|
||
use crate::db::Db;
|
||
use anyhow::Result;
|
||
use chrono::{Duration, Utc};
|
||
use rusqlite::params;
|
||
|
||
pub const VERSION: &str = "dr-nb-v0";
|
||
|
||
pub struct Line {
|
||
pub artist: usize,
|
||
pub p_channel: [f64; C],
|
||
pub mean_channel: [f64; C],
|
||
pub p_any: f64,
|
||
pub mean: f64,
|
||
pub lo: f64,
|
||
pub hi: f64,
|
||
}
|
||
|
||
pub fn create_tables(db: &Db) -> Result<()> {
|
||
db.conn.execute_batch(
|
||
"CREATE TABLE IF NOT EXISTS forecasts (
|
||
made_at TEXT NOT NULL, model_version TEXT NOT NULL,
|
||
target TEXT NOT NULL, -- 'rotation' (prob, mean plays) per channel, 'any' across channels;
|
||
-- 'add' / 'add_any': P(new add in the 8 weeks from target_week), blended
|
||
-- by the artist's chance of releasing ('p_release'); 'add_rel' /
|
||
-- 'add_any_rel' / 'add_rel': if the artist releases a NEW song (release-event model),
|
||
-- 'add_any_nmf': ...placed on NMF Denmark; 'recent_release': released in the last 6 weeks;
|
||
-- 'now': in rotation in the origin week (prob 1/0, mean = plays)
|
||
key TEXT NOT NULL, -- '{dr artist id}:{channel}' or '{dr artist id}:any'
|
||
horizon INTEGER NOT NULL, -- weeks after origin_week
|
||
origin_week TEXT NOT NULL, target_week TEXT NOT NULL,
|
||
prob REAL, mean REAL, lo REAL, hi REAL,
|
||
PRIMARY KEY (made_at, model_version, target, key, horizon)
|
||
);
|
||
CREATE TABLE IF NOT EXISTS backtest_scores (
|
||
run_at TEXT NOT NULL, model TEXT NOT NULL, horizon INTEGER NOT NULL, metric TEXT NOT NULL, value REAL,
|
||
PRIMARY KEY (run_at, model, horizon, metric)
|
||
);",
|
||
)?;
|
||
Ok(())
|
||
}
|
||
|
||
pub fn forecast(p: &Panel, h: usize, sims: usize) -> Vec<Line> {
|
||
let t0 = p.w() - 1;
|
||
let tau = thresholds(p, p.w());
|
||
let d = Direct::fit(p, h, t0, &tau, false);
|
||
let mut x = vec![0.0; P];
|
||
let mut rng = Rng(0x5eed ^ h as u64);
|
||
let shape = 1.0 / d.alpha;
|
||
(0..p.a()).map(|a| {
|
||
let mut mean_channel = [0.0; C];
|
||
let mut p_channel = [0.0; C];
|
||
for c in 0..C {
|
||
let (mu, pr, _) = d.predict(p, a, c, t0, tau[c], &mut x);
|
||
mean_channel[c] = mu;
|
||
p_channel[c] = pr;
|
||
}
|
||
let mut totals = Vec::with_capacity(sims);
|
||
let mut any = 0usize;
|
||
for _ in 0..sims {
|
||
let g = rng.gamma(shape) / shape;
|
||
let (mut tot, mut hit) = (0.0, false);
|
||
for c in 0..C {
|
||
let y = rng.poisson(mean_channel[c] * g);
|
||
tot += y;
|
||
hit |= y >= tau[c] as f64;
|
||
}
|
||
any += hit as usize;
|
||
totals.push(tot);
|
||
}
|
||
totals.sort_by(f64::total_cmp);
|
||
let q = |f: f64| totals[((sims as f64 - 1.0) * f).round() as usize];
|
||
Line { artist: a, p_channel, mean_channel, p_any: any as f64 / sims as f64, mean: mean_channel.iter().sum(), lo: q(0.1), hi: q(0.9) }
|
||
}).collect()
|
||
}
|
||
|
||
/// Forecasts for h = 1..=max_h, stored in `forecasts`; returns the lines of `show_h`.
|
||
pub fn run(db: &Db, p: &Panel, max_h: usize, show_h: usize) -> Result<Vec<Line>> {
|
||
create_tables(db)?;
|
||
let made = Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||
let origin = p.weeks[p.w() - 1];
|
||
let mut shown = Vec::new();
|
||
for h in 1..=max_h {
|
||
let lines = forecast(p, h, 2000);
|
||
let target = (origin + Duration::weeks(h as i64)).to_string();
|
||
let tx = db.conn.unchecked_transaction()?;
|
||
{
|
||
let mut ins = tx.prepare("INSERT OR REPLACE INTO forecasts VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)")?;
|
||
for l in &lines {
|
||
let id = &p.artists[l.artist].id;
|
||
ins.execute(params![made, VERSION, "any", format!("{id}:any"), h as i64, origin.to_string(), target, l.p_any, l.mean, l.lo, l.hi])?;
|
||
for c in 0..C {
|
||
ins.execute(params![made, VERSION, "rotation", format!("{id}:{}", CHANNELS[c]), h as i64, origin.to_string(), target,
|
||
l.p_channel[c], l.mean_channel[c], Option::<f64>::None, Option::<f64>::None])?;
|
||
}
|
||
}
|
||
}
|
||
tx.commit()?;
|
||
eprintln!("rank: h={h} stored ({} artists)", lines.len());
|
||
if h == show_h { shown = lines; }
|
||
}
|
||
// New adds: windows of W weeks starting h = 0..=max_h weeks after the origin.
|
||
let tau = thresholds(p, p.w());
|
||
let rot = Rot::new(p, &tau);
|
||
// The release-event model ("if they release a new song, will DR add it?"), the same for every
|
||
// window start: the artist's state now, the new song's NMF placement blended from history.
|
||
// The site's default: added within 2 weeks of the release.
|
||
let ev = AddModel::fit_events_pooled(p, &rot, p.w() - 1, 2);
|
||
let t0 = p.w() - 1;
|
||
// A new song released in week t0+h+1: time-since features moved forward to then; none if too
|
||
// soon after the artist's last release.
|
||
let event = |a: usize, h: usize| -> Option<([f64; C], f64, f64, f64)> {
|
||
if super::adds::too_soon(p, a, t0, h) { return None; }
|
||
let q = super::adds::p_nmf(p, a, t0);
|
||
let (pcn, pan) = ev.predict_at(p, &rot, a, t0, h, Scenario::new(true, true).at(h), None);
|
||
let (pcr, par) = ev.predict_at(p, &rot, a, t0, h, Scenario::new(true, false).at(h), None);
|
||
Some((std::array::from_fn(|c| q * pcn[c] + (1.0 - q) * pcr[c]), q * pan + (1.0 - q) * par, pan, q))
|
||
};
|
||
for h in 0..=max_h {
|
||
let m = AddModel::fit(p, &rot, h, p.w() - 1);
|
||
let start = (origin + Duration::weeks(h as i64 + 1)).to_string();
|
||
let tx = db.conn.unchecked_transaction()?;
|
||
{
|
||
let mut ins = tx.prepare("INSERT OR REPLACE INTO forecasts VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)")?;
|
||
for a in 0..p.a() {
|
||
let id = &p.artists[a].id;
|
||
// Any add in the window (current songs included): no new release, or one at the usual rate.
|
||
let (pc1, pa1) = m.predict(p, &rot, a, t0, Scenario::new(true, false), None);
|
||
let (pc0, pa0) = m.predict(p, &rot, a, t0, Scenario::new(false, false), None);
|
||
let pr = super::adds::p_release(p, a, t0);
|
||
let mix = |p1: f64, p0: f64| match pr { Some(r) => r * p1 + (1.0 - r) * p0, None => p0 };
|
||
let ev_a = event(a, h);
|
||
let none = Option::<f64>::None;
|
||
let mut put = |target: &str, ch: &str, prob: f64| ins.execute(params![made, VERSION, target, format!("{id}:{ch}"), h as i64,
|
||
origin.to_string(), start, prob, none, none, none]);
|
||
put("add_any", "any", mix(pa1, pa0))?;
|
||
match ev_a {
|
||
Some((_, ea, ean, q)) => { put("add_any_rel", "any", ea)?; put("add_any_nmf", "any", ean)?; put("p_nmf", "any", q)?; }
|
||
None => { put("too_soon", "any", 1.0)?; }
|
||
}
|
||
if let Some(r) = pr { put("p_release", "any", r)?; }
|
||
for c in 0..C {
|
||
put("add", CHANNELS[c], mix(pc1[c], pc0[c]))?;
|
||
if let Some((ec, ..)) = ev_a { put("add_rel", CHANNELS[c], ec[c])?; }
|
||
}
|
||
}
|
||
}
|
||
tx.commit()?;
|
||
eprintln!("rank: adds h={h} stored");
|
||
}
|
||
// Where each artist stands in the origin week: prob = 1 if in rotation, mean = plays.
|
||
let tx = db.conn.unchecked_transaction()?;
|
||
{
|
||
let mut ins = tx.prepare("INSERT OR REPLACE INTO forecasts VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)")?;
|
||
let t0 = p.w() - 1;
|
||
for a in 0..p.a() {
|
||
let id = &p.artists[a].id;
|
||
for c in 0..C {
|
||
let y = p.y(a, c, t0);
|
||
if y == 0.0 { continue; }
|
||
ins.execute(params![made, VERSION, "now", format!("{id}:{}", CHANNELS[c]), 0i64, origin.to_string(), origin.to_string(),
|
||
rot.on(a, c, t0) as i32 as f64, y, Option::<f64>::None, Option::<f64>::None])?;
|
||
}
|
||
}
|
||
}
|
||
tx.commit()?;
|
||
Ok(shown)
|
||
}
|