Deezer discographies for every DR artist; release features in the add model (recency, releases in the window); rankings 'if they release in the window' and blended by release rate; site switch and what-if release toggle

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-15 03:21:59 +00:00
co-authored by Claude Opus 5.5
parent 7815ed417c
commit 5b7e544b1c
7 changed files with 189 additions and 34 deletions
+61
View File
@@ -163,3 +163,64 @@ fn urlencode(s: &str) -> String {
_ => format!("%{b:02X}"), _ => format!("%{b:02X}"),
}).collect() }).collect()
} }
// ------------------------------------------------------------------ discographies
/// Every DR artist's releases from Deezer, for release dates and the historical candidate pool:
/// GET /search/artist?q={name} the Deezer artist: exact name match, most fans
/// GET /artist/{id}/albums every release (album, EP, single) with its date
pub fn create_discography_tables(db: &Db) -> Result<()> {
db.conn.execute_batch(
"CREATE TABLE IF NOT EXISTS deezer_artists (
dr_artist_id TEXT PRIMARY KEY, deezer_artist_id INTEGER, name TEXT, fans INTEGER,
candidates INTEGER, fetched_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS deezer_releases (
deezer_artist_id INTEGER NOT NULL, album_id INTEGER NOT NULL, title TEXT, release_date TEXT,
record_type TEXT, explicit INTEGER, fetched_at TEXT NOT NULL,
PRIMARY KEY (deezer_artist_id, album_id)
);
CREATE INDEX IF NOT EXISTS deezer_releases_date ON deezer_releases (release_date);",
)?;
Ok(())
}
pub fn discographies(db: &Db, http: &Http, limit: usize, log: &mut Vec<String>) -> Result<usize> {
create_discography_tables(db)?;
let todo: Vec<(String, String)> = db.conn.prepare(
"SELECT artist_id, name FROM dr_artists WHERE artist_id NOT IN (SELECT dr_artist_id FROM deezer_artists) ORDER BY artist_id")?
.query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?.collect::<Result<_, _>>()?;
let now = Utc::now().to_rfc3339();
let mut done = 0;
for (dr_id, name) in todo.into_iter().take(limit) {
let found = match fetch_json(http, &format!("{BASE}/search/artist?q={}&limit=10", urlencode(&name))) {
Ok(v) => v.unwrap_or(Value::Null),
Err(e) => { log.push(format!("deezer artist '{name}': {e}")); continue; }
};
let hits = found["data"].as_array().cloned().unwrap_or_default();
let best = hits.iter().filter(|h| key(h["name"].as_str().unwrap_or_default()) == key(&name))
.max_by_key(|h| h["nb_fan"].as_i64().unwrap_or(0));
let (did, fans) = (best.and_then(|h| h["id"].as_i64()), best.and_then(|h| h["nb_fan"].as_i64()));
if let Some(id) = did {
let have: i64 = db.conn.query_row("SELECT count(*) FROM deezer_releases WHERE deezer_artist_id = ?1", [id], |r| r.get(0))?;
if have == 0 {
let mut url = Some(format!("{BASE}/artist/{id}/albums?limit=100"));
let mut pages = 0;
while let Some(u) = url.take() {
let v = match fetch_json(http, &u) { Ok(Some(v)) => v, Ok(None) => break, Err(e) => { log.push(format!("deezer albums {id}: {e}")); break; } };
for a in v["data"].as_array().into_iter().flatten() {
db.conn.execute("INSERT OR REPLACE INTO deezer_releases VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
params![id, a["id"].as_i64(), a["title"].as_str(), a["release_date"].as_str(), a["record_type"].as_str(),
a["explicit_lyrics"].as_bool().map(|b| b as i64), now])?;
}
pages += 1;
url = v["next"].as_str().filter(|_| pages < 5).map(str::to_string);
}
}
}
db.conn.execute("INSERT OR REPLACE INTO deezer_artists VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
params![dr_id, did, name, fans, hits.len() as i64, now])?;
done += 1;
}
Ok(done)
}
+9 -2
View File
@@ -1,6 +1,6 @@
//! airplay: collects what Danish radio stations play, for modelling when artists get played again. //! airplay: collects what Danish radio stations play, for modelling when artists get played again.
//! //!
//! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|musicbrainz|spotify|playlists|nmf] [--backfill] //! airplay collect [--db data/airplay.db] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf] [--backfill]
//! airplay stats [--db data/airplay.db] //! airplay stats [--db data/airplay.db]
//! airplay check [--db data/airplay.db] gaps, duplicates, DR cross-check, chart completeness //! airplay check [--db data/airplay.db] gaps, duplicates, DR cross-check, chart completeness
//! airplay model panel | backtest | rank the first models (src/model) //! airplay model panel | backtest | rank the first models (src/model)
@@ -57,7 +57,7 @@ fn main() -> Result<()> {
rotations::store_all(&db, &dir, args.iter().any(|a| a == "--reparse")) rotations::store_all(&db, &dir, args.iter().any(|a| a == "--reparse"))
} }
_ => { _ => {
eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|musicbrainz|spotify|playlists|nmf] [--backfill]\n airplay stats [--db FILE]\n airplay check [--db FILE]\n airplay model panel|backtest|rank\n airplay serve [--addr HOST:PORT] [--prefix /airplay]"); eprintln!("usage: airplay collect [--db FILE] [--days 7] [--all-bauer] [--only dr|bauer|hitlisten|deezer|discographies|musicbrainz|spotify|playlists|nmf] [--backfill]\n airplay stats [--db FILE]\n airplay check [--db FILE]\n airplay model panel|backtest|rank\n airplay serve [--addr HOST:PORT] [--prefix /airplay]");
std::process::exit(2); std::process::exit(2);
} }
} }
@@ -106,6 +106,13 @@ fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> {
let deezer_http = http::Http::new(150)?; let deezer_http = http::Http::new(150)?;
let r = deezer::collect(db, &deezer_http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 3000 }), &mut log); let r = deezer::collect(db, &deezer_http, arg(args, "--limit").and_then(|l| l.parse().ok()).unwrap_or(if backfill { usize::MAX } else { 3000 }), &mut log);
step("deezer song metadata", r.map(|n| format!("{n} songs looked up")), &mut log); step("deezer song metadata", r.map(|n| format!("{n} songs looked up")), &mut log);
let r = deezer::discographies(db, &deezer_http, if backfill { usize::MAX } else { 1500 }, &mut log);
step("deezer discographies", r.map(|n| format!("{n} artists looked up")), &mut log);
}
if wants("discographies") {
// On its own (next to a running song backfill): slower, to stay under Deezer's limit together.
let r = deezer::discographies(db, &http::Http::new(500)?, usize::MAX, &mut log);
step("deezer discographies", r.map(|n| format!("{n} artists looked up")), &mut log);
} }
if wants("musicbrainz") { if wants("musicbrainz") {
// MusicBrainz allows one request per second. // MusicBrainz allows one request per second.
+48 -13
View File
@@ -27,11 +27,42 @@ pub const GAP: usize = 4;
pub const W: usize = 8; pub const W: usize = 8;
/// Per-channel rows: the panel features plus these. /// Per-channel rows: the panel features plus these.
/// Release covariates (Deezer discography), appended to both row kinds.
pub const REL: usize = 6;
pub const REL_NAMES: &[&str] = &["discography known", "release recency 4w", "release recency 12w", "log releases last 52w",
"releases in the window", "releases in window x not in rotation now"];
/// Weeks since the latest release, releases in the past year, and whether the artist releases in
/// the window t+h+1 ..= t+h+W (the actual one when `rel` is None: training and backtests; else the
/// scenario "releases / doesn't release in the window").
pub fn rel_feats(p: &Panel, a: usize, t: usize, h: usize, rel: Option<bool>, in_rot_now: bool) -> [f64; REL] {
let Some(dates) = p.releases[a].as_ref() else { return [0.0; REL] };
let end_t = p.weeks[t] + chrono::Duration::days(6);
let since = dates.iter().rev().find(|d| **d <= end_t).map(|d| (end_t - *d).num_days() as f64 / 7.0);
let last_year = dates.iter().filter(|d| **d <= end_t && (end_t - **d).num_days() <= 364).count() as f64;
let w0 = p.weeks[t] + chrono::Duration::weeks(h as i64 + 1);
let w1 = w0 + chrono::Duration::weeks(W as i64);
let in_window = rel.unwrap_or_else(|| dates.iter().any(|d| *d >= w0 && *d < w1));
let f = |b: bool| b as i32 as f64;
[1.0, since.map(|s| (-s / 4.0).exp()).unwrap_or(0.0), since.map(|s| (-s / 12.0).exp()).unwrap_or(0.0), last_year.ln_1p(),
f(in_window), f(in_window && !in_rot_now)]
}
/// P(the artist releases something in an 8-week window), from their release rate over two years
/// (shrunk towards one release a year for artists with little history).
pub fn p_release(p: &Panel, a: usize, t: usize) -> Option<f64> {
let dates = p.releases[a].as_ref()?;
let end_t = p.weeks[t] + chrono::Duration::days(6);
let n = dates.iter().filter(|d| **d <= end_t && (end_t - **d).num_days() <= 730).count() as f64;
let per_year = (n + 1.0) / 3.0;
Some(1.0 - (-per_year * W as f64 / 52.0).exp())
}
pub const EXTRA: &[&str] = &["in rotation on channel now", "in rotation anywhere now", "channels in rotation now (/14)", pub const EXTRA: &[&str] = &["in rotation on channel now", "in rotation anywhere now", "channels in rotation now (/14)",
"run on channel = 1 week", "run = 2 weeks", "run = 3-5 weeks", "run >= 6 weeks", "run on channel = 1 week", "run = 2 weeks", "run = 3-5 weeks", "run >= 6 weeks",
"near threshold (>= half of tau, not in rotation)", "plays / tau (capped 3, /3)", "near threshold (>= half of tau, not in rotation)", "plays / tau (capped 3, /3)",
"weeks since in rotation on channel (/12)"]; "weeks since in rotation on channel (/12)"];
pub const PC: usize = P + 10; pub const PC: usize = P + 10 + REL;
/// Artist-level ("any channel") rows. /// Artist-level ("any channel") rows.
pub const ANY_NAMES: &[&str] = &[ pub const ANY_NAMES: &[&str] = &[
@@ -41,7 +72,7 @@ pub const ANY_NAMES: &[&str] = &[
"log Track Top-40 hits", "log Airplay Top-20 hits", "Danish", "group", "female", "career years (/50)", "debut unknown", "log Track Top-40 hits", "log Airplay Top-20 hits", "Danish", "group", "female", "career years (/50)", "debut unknown",
"Christmas in window", "Christmas in window",
]; ];
pub const PA: usize = 21; pub const PA: usize = 21 + REL;
/// Positions of the chart covariates (for the what-if): (t40 score, air score, recency 6w, recency 26w). /// Positions of the chart covariates (for the what-if): (t40 score, air score, recency 6w, recency 26w).
pub const CHART_IDX_CHANNEL: [usize; 4] = [10, 11, 12, 13]; pub const CHART_IDX_CHANNEL: [usize; 4] = [10, 11, 12, 13];
pub const CHART_IDX_ANY: [usize; 4] = [9, 10, 11, 12]; pub const CHART_IDX_ANY: [usize; 4] = [9, 10, 11, 12];
@@ -84,7 +115,7 @@ fn run_and_since(flag: impl Fn(usize) -> bool, t: usize) -> (f64, f64) {
fn christmas_in_window(p: &Panel, t: usize, h: usize) -> bool { (t + h + 1..=t + h + W).any(|w| features::christmas(p, w)) } fn christmas_in_window(p: &Panel, t: usize, h: usize) -> bool { (t + h + 1..=t + h + W).any(|w| features::christmas(p, w)) }
pub fn row_channel(p: &Panel, r: &Rot, a: usize, c: usize, t: usize, h: usize, out: &mut [f64]) { pub fn row_channel(p: &Panel, r: &Rot, a: usize, c: usize, t: usize, h: usize, rel: Option<bool>, out: &mut [f64]) {
features::row(p, a, c, t, t + h + 1, &mut out[..P]); features::row(p, a, c, t, t + h + 1, &mut out[..P]);
out[21] = christmas_in_window(p, t, h) as i32 as f64; out[21] = christmas_in_window(p, t, h) as i32 as f64;
let n_on = (0..C).filter(|&k| r.on(a, k, t)).count() as f64; let n_on = (0..C).filter(|&k| r.on(a, k, t)).count() as f64;
@@ -96,9 +127,10 @@ pub fn row_channel(p: &Panel, r: &Rot, a: usize, c: usize, t: usize, h: usize, o
out[P..PC].copy_from_slice(&[f(on), f(r.any(a, t)), n_on / C as f64, out[P..PC].copy_from_slice(&[f(on), f(r.any(a, t)), n_on / C as f64,
f(weeks == 1), f(weeks == 2), f((3..=5).contains(&weeks)), f(weeks >= 6), f(weeks == 1), f(weeks == 2), f((3..=5).contains(&weeks)), f(weeks >= 6),
f(!on && ratio >= 0.5), ratio.min(3.0) / 3.0, since]); f(!on && ratio >= 0.5), ratio.min(3.0) / 3.0, since]);
out[P + 10..PC].copy_from_slice(&rel_feats(p, a, t, h, rel, on));
} }
pub fn row_any(p: &Panel, r: &Rot, a: usize, t: usize, h: usize, out: &mut [f64]) { pub fn row_any(p: &Panel, r: &Rot, a: usize, t: usize, h: usize, rel: Option<bool>, out: &mut [f64]) {
let tot = |w: usize| (0..C).map(|c| p.y(a, c, w)).sum::<f64>(); let tot = |w: usize| (0..C).map(|c| p.y(a, c, w)).sum::<f64>();
let mean = |n: usize| (0..n).map(|i| tot(t - i)).sum::<f64>() / n as f64; let mean = |n: usize| (0..n).map(|i| tot(t - i)).sum::<f64>() / n as f64;
let max12 = (0..LAGS).map(|i| tot(t - i)).fold(0.0, f64::max); let max12 = (0..LAGS).map(|i| tot(t - i)).fold(0.0, f64::max);
@@ -118,7 +150,8 @@ pub fn row_any(p: &Panel, r: &Rot, a: usize, t: usize, h: usize, out: &mut [f64]
art.debut.map(|d| ((year - d).clamp(0, 50)) as f64 / 50.0).unwrap_or(0.0), art.debut.is_none() as i32 as f64, art.debut.map(|d| ((year - d).clamp(0, 50)) as f64 / 50.0).unwrap_or(0.0), art.debut.is_none() as i32 as f64,
christmas_in_window(p, t, h) as i32 as f64, christmas_in_window(p, t, h) as i32 as f64,
]; ];
out.copy_from_slice(&v); out[..21].copy_from_slice(&v);
out[21..PA].copy_from_slice(&rel_feats(p, a, t, h, rel, r.any(a, t)));
} }
/// Logit models for one h, trained on origins whose whole window is observed by `last_week`. /// Logit models for one h, trained on origins whose whole window is observed by `last_week`.
@@ -158,7 +191,7 @@ impl AddModel {
let (mut dc, mut da) = (Design::new(2), Design::new(2)); let (mut dc, mut da) = (Design::new(2), Design::new(2));
for t in c0..=hi { for t in c0..=hi {
for a in 0..p.a() { for a in 0..p.a() {
let (pc, pa) = inner.predict(p, r, a, t, None); let (pc, pa) = inner.predict(p, r, a, t, None, None);
for c in 0..C { for c in 0..C {
if pc[c] < 1e-4 && !r.add_in_window(a, c, t, h) { continue; } // certain zeros carry no information if pc[c] < 1e-4 && !r.add_in_window(a, c, t, h) { continue; } // certain zeros carry no information
dc.push(&[1.0, logit(pc[c])], r.add_in_window(a, c, t, h) as i32 as f64, 1.0); dc.push(&[1.0, logit(pc[c])], r.add_in_window(a, c, t, h) as i32 as f64, 1.0);
@@ -184,10 +217,10 @@ impl AddModel {
// Channels where the artist hasn't been heard for 12 weeks: keep a quarter, weighted 4. // Channels where the artist hasn't been heard for 12 weeks: keep a quarter, weighted 4.
let quiet = (0..LAGS).all(|i| p.y(a, c, t - i) == 0.0); let quiet = (0..LAGS).all(|i| p.y(a, c, t - i) == 0.0);
let wgt = if quiet { if mix((a as u64) << 32 ^ (c as u64) << 20 ^ t as u64 ^ 0xadd) % 4 != 0 { continue; } 4.0 } else { 1.0 }; let wgt = if quiet { if mix((a as u64) << 32 ^ (c as u64) << 20 ^ t as u64 ^ 0xadd) % 4 != 0 { continue; } 4.0 } else { 1.0 };
row_channel(p, r, a, c, t, h, &mut x); row_channel(p, r, a, c, t, h, None, &mut x);
dc.push(&x, r.add_in_window(a, c, t, h) as i32 as f64, wgt); dc.push(&x, r.add_in_window(a, c, t, h) as i32 as f64, wgt);
} }
row_any(p, r, a, t, h, &mut xa); row_any(p, r, a, t, h, None, &mut xa);
da.push(&xa, r.add_any_in_window(a, t, h) as i32 as f64, 1.0); da.push(&xa, r.add_any_in_window(a, t, h) as i32 as f64, 1.0);
} }
} }
@@ -195,16 +228,17 @@ impl AddModel {
} }
/// (P(add) per channel, P(add on any channel)) from origin t. /// (P(add) per channel, P(add on any channel)) from origin t.
pub fn predict(&self, p: &Panel, r: &Rot, a: usize, t: usize, chart: Option<&dyn Fn(&mut [f64], [usize; 4])>) -> ([f64; C], f64) { /// `rel`: None = whether the artist actually released in the window (backtests), Some = scenario.
pub fn predict(&self, p: &Panel, r: &Rot, a: usize, t: usize, rel: Option<bool>, chart: Option<&dyn Fn(&mut [f64], [usize; 4])>) -> ([f64; C], f64) {
let mut x = vec![0.0; PC]; let mut x = vec![0.0; PC];
let mut pc = [0.0; C]; let mut pc = [0.0; C];
for c in 0..C { for c in 0..C {
row_channel(p, r, a, c, t, self.h, &mut x); row_channel(p, r, a, c, t, self.h, rel, &mut x);
if let Some(f) = chart { f(&mut x, CHART_IDX_CHANNEL); } if let Some(f) = chart { f(&mut x, CHART_IDX_CHANNEL); }
pc[c] = platt(self.platt_channel, glm::sigmoid(glm::dot(&self.channel, &x))); pc[c] = platt(self.platt_channel, glm::sigmoid(glm::dot(&self.channel, &x)));
} }
let mut xa = vec![0.0; PA]; let mut xa = vec![0.0; PA];
row_any(p, r, a, t, self.h, &mut xa); row_any(p, r, a, t, self.h, rel, &mut xa);
if let Some(f) = chart { f(&mut xa, CHART_IDX_ANY); } if let Some(f) = chart { f(&mut xa, CHART_IDX_ANY); }
(pc, platt(self.platt_any, glm::sigmoid(glm::dot(&self.any, &xa)))) (pc, platt(self.platt_any, glm::sigmoid(glm::dot(&self.any, &xa))))
} }
@@ -289,7 +323,7 @@ pub fn backtest(p: &Panel, tau: &[u32; C], horizons: &[usize], start: usize, md:
} }
let (mut lc, mut la) = ([0.0; 3], [0.0; 3]); let (mut lc, mut la) = ([0.0; 3], [0.0; 3]);
for a in 0..p.a() { for a in 0..p.a() {
let (pc, pa) = m.predict(p, &r, a, t0, None); let (pc, pa) = m.predict(p, &r, a, t0, None, None);
let mut tot = 0.0; let mut tot = 0.0;
for c in 0..C { for c in 0..C {
let y = r.add_in_window(a, c, t0, h); let y = r.add_in_window(a, c, t0, h);
@@ -342,8 +376,9 @@ pub fn backtest(p: &Panel, tau: &[u32; C], horizons: &[usize], start: usize, md:
let m = AddModel::fit(p, &r, 0, p.w() - 1); let m = AddModel::fit(p, &r, 0, p.w() - 1);
writeln!(md, "## Coefficients, window starting now (h = 0), all closed windows\n\n| Any-channel feature | Coefficient |\n|---|---|")?; writeln!(md, "## Coefficients, window starting now (h = 0), all closed windows\n\n| Any-channel feature | Coefficient |\n|---|---|")?;
for (n, b) in ANY_NAMES.iter().zip(&m.any) { writeln!(md, "| {n} | {b:+.3} |")?; } for (n, b) in ANY_NAMES.iter().zip(&m.any) { writeln!(md, "| {n} | {b:+.3} |")?; }
for (n, b) in REL_NAMES.iter().zip(&m.any[21..]) { writeln!(md, "| {n} | {b:+.3} |")?; }
writeln!(md, "\n| Per-channel feature | Coefficient |\n|---|---|")?; writeln!(md, "\n| Per-channel feature | Coefficient |\n|---|---|")?;
for (n, b) in features::NAMES.iter().chain(EXTRA).zip(&m.channel) { writeln!(md, "| {n} | {b:+.3} |")?; } for (n, b) in features::NAMES.iter().chain(EXTRA).chain(REL_NAMES).zip(&m.channel) { writeln!(md, "| {n} | {b:+.3} |")?; }
writeln!(md)?; writeln!(md)?;
Ok(rows) Ok(rows)
} }
+24 -1
View File
@@ -51,6 +51,8 @@ pub struct Panel {
chart: Vec<Chart>, chart: Vec<Chart>,
/// Rotation thresholds fitted to DR's rotation lists, if calibrated. /// Rotation thresholds fitted to DR's rotation lists, if calibrated.
pub list_tau: Option<[u32; C]>, pub list_tau: Option<[u32; C]>,
/// Release dates per artist (Deezer discography: singles, EPs, albums), sorted; None = not looked up / not found.
pub releases: Vec<Option<Vec<NaiveDate>>>,
} }
impl Panel { impl Panel {
@@ -114,7 +116,8 @@ impl Panel {
let chart = chart_states(db, &weeks, &names, artists.len())?; let chart = chart_states(db, &weeks, &names, artists.len())?;
let list_tau = super::calibrate::stored(db)?; let list_tau = super::calibrate::stored(db)?;
Ok(Panel { weeks, artists, y, chart, list_tau }) let releases = load_releases(db, &id_of, artists.len())?;
Ok(Panel { weeks, artists, y, chart, list_tau, releases })
} }
} }
@@ -186,3 +189,23 @@ fn chart_states(db: &Db, weeks: &[NaiveDate], names: &HashMap<String, Vec<usize>
} }
Ok(out) Ok(out)
} }
fn load_releases(db: &Db, id_of: &HashMap<String, usize>, n: usize) -> Result<Vec<Option<Vec<NaiveDate>>>> {
let mut out: Vec<Option<Vec<NaiveDate>>> = vec![None; n];
let exists: i64 = db.conn.query_row("SELECT count(*) FROM sqlite_master WHERE name = 'deezer_artists'", [], |r| r.get(0))?;
if exists == 0 { return Ok(out); }
for r in db.conn.prepare("SELECT dr_artist_id FROM deezer_artists WHERE deezer_artist_id IS NOT NULL")?.query_map([], |r| r.get::<_, String>(0))? {
if let Some(&a) = id_of.get(&r?) { out[a] = Some(Vec::new()); }
}
for r in db.conn.prepare(
"SELECT d.dr_artist_id, r.release_date FROM deezer_artists d JOIN deezer_releases r ON r.deezer_artist_id = d.deezer_artist_id
WHERE coalesce(r.record_type, '') != 'compile' AND r.release_date > '1900'")?
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? {
let (id, d) = r?;
if let (Some(&a), Ok(d)) = (id_of.get(&id), NaiveDate::parse_from_str(&d, "%Y-%m-%d")) {
if let Some(v) = out[a].as_mut() { v.push(d); }
}
}
for v in out.iter_mut().flatten() { v.sort(); v.dedup(); }
Ok(out)
}
+17 -6
View File
@@ -33,7 +33,9 @@ pub fn create_tables(db: &Db) -> Result<()> {
"CREATE TABLE IF NOT EXISTS forecasts ( "CREATE TABLE IF NOT EXISTS forecasts (
made_at TEXT NOT NULL, model_version TEXT NOT NULL, made_at TEXT NOT NULL, model_version TEXT NOT NULL,
target TEXT NOT NULL, -- 'rotation' (prob, mean plays) per channel, 'any' across channels; target TEXT NOT NULL, -- 'rotation' (prob, mean plays) per channel, 'any' across channels;
-- 'add' / 'add_any': P(new add in the 8 weeks from target_week); -- 'add' / 'add_any': P(new add in the 8 weeks from target_week), blended
-- by the artist's chance of releasing ('p_release'); 'add_rel' /
-- 'add_any_rel': the same if the artist releases in the window;
-- 'now': in rotation in the origin week (prob 1/0, mean = plays) -- 'now': in rotation in the origin week (prob 1/0, mean = plays)
key TEXT NOT NULL, -- '{dr artist id}:{channel}' or '{dr artist id}:any' key TEXT NOT NULL, -- '{dr artist id}:{channel}' or '{dr artist id}:any'
horizon INTEGER NOT NULL, -- weeks after origin_week horizon INTEGER NOT NULL, -- weeks after origin_week
@@ -119,12 +121,21 @@ pub fn run(db: &Db, p: &Panel, max_h: usize, show_h: usize) -> Result<Vec<Line>>
let mut ins = tx.prepare("INSERT OR REPLACE INTO forecasts VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)")?; let mut ins = tx.prepare("INSERT OR REPLACE INTO forecasts VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)")?;
for a in 0..p.a() { for a in 0..p.a() {
let id = &p.artists[a].id; let id = &p.artists[a].id;
let (pc, pa) = m.predict(p, &rot, a, p.w() - 1, None); // If they release in the window, if they don't, and blended by their release rate.
ins.execute(params![made, VERSION, "add_any", format!("{id}:any"), h as i64, origin.to_string(), start, pa, let t0 = p.w() - 1;
Option::<f64>::None, Option::<f64>::None, Option::<f64>::None])?; let (pc1, pa1) = m.predict(p, &rot, a, t0, Some(true), None);
let (pc0, pa0) = m.predict(p, &rot, a, t0, Some(false), None);
let pr = super::adds::p_release(p, a, t0);
let mix = |p1: f64, p0: f64| match pr { Some(r) => r * p1 + (1.0 - r) * p0, None => p0 };
let none = Option::<f64>::None;
let mut put = |target: &str, ch: &str, prob: f64| ins.execute(params![made, VERSION, target, format!("{id}:{ch}"), h as i64,
origin.to_string(), start, prob, none, none, none]);
put("add_any", "any", mix(pa1, pa0))?;
put("add_any_rel", "any", pa1)?;
if let Some(r) = pr { put("p_release", "any", r)?; }
for c in 0..C { for c in 0..C {
ins.execute(params![made, VERSION, "add", format!("{id}:{}", CHANNELS[c]), h as i64, origin.to_string(), start, pc[c], put("add", CHANNELS[c], mix(pc1[c], pc0[c]))?;
Option::<f64>::None, Option::<f64>::None, Option::<f64>::None])?; put("add_rel", CHANNELS[c], pc1[c])?;
} }
} }
} }
+10 -4
View File
@@ -186,6 +186,9 @@ fn fold(e: &mut Value, target: &str, ch: &str, prob: Option<f64>, mean: Option<f
"rotation" => { e["p"][ch] = r3(prob); e["plays"][ch] = r1(mean); } "rotation" => { e["p"][ch] = r3(prob); e["plays"][ch] = r1(mean); }
"add_any" => { e["add_any"] = r3(prob); } "add_any" => { e["add_any"] = r3(prob); }
"add" => { e["add"][ch] = r3(prob); } "add" => { e["add"][ch] = r3(prob); }
"add_any_rel" => { e["add_any_rel"] = r3(prob); }
"add_rel" => { if !e["add_rel"].is_object() { e["add_rel"] = json!({}); } e["add_rel"][ch] = r3(prob); }
"p_release" => { e["p_release"] = r3(prob); }
"now" => { e["now"][ch] = json!({"on": prob.unwrap_or(0.0) > 0.5, "plays": mean}); } "now" => { e["now"][ch] = json!({"on": prob.unwrap_or(0.0) > 0.5, "plays": mean}); }
_ => {} _ => {}
} }
@@ -231,7 +234,7 @@ fn rank(st: &State, h: usize) -> Result<Reply> {
} }
let mut rows: HashMap<String, Value> = HashMap::new(); let mut rows: HashMap<String, Value> = HashMap::new();
let mut st_ = c.prepare("SELECT key, target, prob, mean, lo, hi, target_week FROM forecasts WHERE made_at = ?1 AND model_version = ?2 let mut st_ = c.prepare("SELECT key, target, prob, mean, lo, hi, target_week FROM forecasts WHERE made_at = ?1 AND model_version = ?2
AND ((target IN ('rotation', 'any') AND horizon = ?3) OR (target IN ('add', 'add_any') AND horizon = ?4) OR target = 'now')")?; AND ((target IN ('rotation', 'any') AND horizon = ?3) OR (target IN ('add', 'add_any', 'add_rel', 'add_any_rel', 'p_release') AND horizon = ?4) OR target = 'now')")?;
let it = st_.query_map(params![made, VERSION, h.max(1) as i64, h as i64], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, Option<f64>>(2)?, let it = st_.query_map(params![made, VERSION, h.max(1) as i64, h as i64], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, Option<f64>>(2)?,
r.get::<_, Option<f64>>(3)?, r.get::<_, Option<f64>>(4)?, r.get::<_, Option<f64>>(5)?, r.get::<_, String>(6)?)))?; r.get::<_, Option<f64>>(3)?, r.get::<_, Option<f64>>(4)?, r.get::<_, Option<f64>>(5)?, r.get::<_, String>(6)?)))?;
let mut window = String::new(); let mut window = String::new();
@@ -296,7 +299,7 @@ fn artist(st: &State, id: &str) -> Result<Reply> {
let ch = key.split_once(':').map(|x| x.1).unwrap_or(""); let ch = key.split_once(':').map(|x| x.1).unwrap_or("");
if target == "now" { fold(&mut now, "now", ch, prob, mean, lo, hi); continue; } if target == "now" { fold(&mut now, "now", ch, prob, mean, lo, hi); continue; }
let e = byh.entry(h).or_insert_with(|| json!({"h": h, "p": {}, "plays": {}, "add": {}})); let e = byh.entry(h).or_insert_with(|| json!({"h": h, "p": {}, "plays": {}, "add": {}}));
if target.starts_with("add") { e["window_start"] = json!(week); } else { e["week"] = json!(week); } if target.starts_with("add") || target == "p_release" { e["window_start"] = json!(week); } else { e["week"] = json!(week); }
fold(e, &target, ch, prob, mean, lo, hi); fold(e, &target, ch, prob, mean, lo, hi);
} }
now_out = now["now"].clone(); now_out = now["now"].clone();
@@ -352,11 +355,13 @@ fn whatif(st: &State, q: &HashMap<String, String>) -> Result<Reply> {
let num = |k: &str| q.get(k).filter(|v| !v.is_empty()).and_then(|v| v.parse::<f64>().ok()); let num = |k: &str| q.get(k).filter(|v| !v.is_empty()).and_then(|v| v.parse::<f64>().ok());
let entry: Option<Option<f64>> = q.get("entry").filter(|v| !v.is_empty()).map(|v| if v == "none" { None } else { v.parse().ok() }); let entry: Option<Option<f64>> = q.get("entry").filter(|v| !v.is_empty()).map(|v| if v == "none" { None } else { v.parse().ok() });
let (t40, air) = (num("t40"), num("air")); let (t40, air) = (num("t40"), num("air"));
// The scenario: a release in the window (default) or none.
let release = q.get("release").map(|v| v != "no").unwrap_or(true);
let over = move |x: &mut [f64], idx: [usize; 4]| set_chart(x, idx, entry, t40, air); let over = move |x: &mut [f64], idx: [usize; 4]| set_chart(x, idx, entry, t40, air);
let run = |overrides: bool| -> Value { let run = |overrides: bool| -> Value {
let f: Option<&dyn Fn(&mut [f64], [usize; 4])> = if overrides { Some(&over) } else { None }; let f: Option<&dyn Fn(&mut [f64], [usize; 4])> = if overrides { Some(&over) } else { None };
let (add_c, add_any) = am.predict(p, &m.rot, a, t0, f); let (add_c, add_any) = am.predict(p, &m.rot, a, t0, Some(release), f);
let mut x = vec![0.0; P]; let mut x = vec![0.0; P];
let mut mu = [0.0; C]; let mut mu = [0.0; C];
for c in 0..C { for c in 0..C {
@@ -378,7 +383,8 @@ fn whatif(st: &State, q: &HashMap<String, String>) -> Result<Reply> {
}; };
let now: serde_json::Map<String, Value> = (0..C).filter(|&c| m.rot.on(a, c, t0)).map(|c| (CHANNELS[c].to_string(), json!(p.y(a, c, t0)))).collect(); let now: serde_json::Map<String, Value> = (0..C).filter(|&c| m.rot.on(a, c, t0)).map(|c| (CHANNELS[c].to_string(), json!(p.y(a, c, t0)))).collect();
ok(json!({ ok(json!({
"h": h, "window_start": (p.weeks[t0] + chrono::Duration::weeks(h as i64 + 1)).to_string(), "h": h, "release": release, "p_release": crate::model::adds::p_release(p, a, t0),
"window_start": (p.weeks[t0] + chrono::Duration::weeks(h as i64 + 1)).to_string(),
"current": {"t40": chart.t40_rank, "air": chart.air_rank, "current": {"t40": chart.t40_rank, "air": chart.air_rank,
"entry": if chart.weeks_since_entry == u16::MAX { Value::Null } else { json!(chart.weeks_since_entry) }, "entry": if chart.weeks_since_entry == u16::MAX { Value::Null } else { json!(chart.weeks_since_entry) },
"t40_hits": chart.t40_hits, "air_hits": chart.air_hits, "in_rotation": now}, "t40_hits": chart.t40_hits, "air_hits": chart.air_hits, "in_rotation": now},
+20 -8
View File
@@ -118,6 +118,12 @@ svg text { fill: var(--muted); font-size: 11px; }
<option value="p8jazz">P8 Jazz</option> <option value="p8jazz">P8 Jazz</option>
</select> </select>
</div> </div>
<div class="ctl">Assume
<select id="mode">
<option value="rel">They release in the window</option>
<option value="mix">Their usual release rate</option>
</select>
</div>
<div class="ctl">Sort by <div class="ctl">Sort by
<select id="sort"> <select id="sort">
<option value="p">Chance of a new add</option> <option value="p">Chance of a new add</option>
@@ -139,7 +145,7 @@ svg text { fill: var(--muted); font-size: 11px; }
</table> </table>
</div> </div>
<button class="more" id="more" hidden>Show more</button> <button class="more" id="more" hidden>Show more</button>
<p class="note">"In rotation now" lists DR stations where the artist is on the list, then Bauer stations (grey) with 14+ plays in the last 7 days. <b>New add</b> = the chance that the artist gets a song into rotation within the 8-week window: entering rotation (at or above the station's weekly play threshold) after at least 4 weeks out of rotation. For "any DR station", out of rotation everywhere first. An artist with a song in rotation now usually has to drop out before a new add, which is why current hits rank lower here. "In rotation now" shows where the artist is in rotation in the latest week; plays per week are for the window's first week. Click an artist for history and a what-if.</p> <p class="note">"In rotation now" lists DR stations where the artist is on the list, then Bauer stations (grey) with 14+ plays in the last 7 days. "Assume they release" shows the chance if the artist releases something in the window (release dates from their Deezer discography); "usual release rate" weighs that by how often they release. <b>New add</b> = the chance that the artist gets a song into rotation within the 8-week window: entering rotation (at or above the station's weekly play threshold) after at least 4 weeks out of rotation. For "any DR station", out of rotation everywhere first. An artist with a song in rotation now usually has to drop out before a new add, which is why current hits rank lower here. "In rotation now" shows where the artist is in rotation in the latest week; plays per week are for the window's first week. Click an artist for history and a what-if.</p>
</section> </section>
<section id="tab-model" hidden> <section id="tab-model" hidden>
@@ -185,9 +191,12 @@ const fmtDate = s => new Date(s + 'T00:00:00Z').toLocaleDateString('en-GB', { da
// Chance of a new add in the window (station group g, or any DR station). // Chance of a new add in the window (station group g, or any DR station).
function chanP(r, g) { function chanP(r, g) {
if (g === 'any') return r.add_any; // "They release in the window" uses the conditional forecast where a discography is known.
if (g === 'p4') return Math.max(...P4S.map(c => r.add?.[c] ?? 0)); const rel = document.getElementById('mode')?.value === 'rel' && r.add_rel;
return r.add?.[g]; const any = rel ? r.add_any_rel : r.add_any, per = rel ? r.add_rel : r.add;
if (g === 'any') return any;
if (g === 'p4') return Math.max(...P4S.map(c => per?.[c] ?? 0));
return per?.[g];
} }
function nowPlays(r, g) { function nowPlays(r, g) {
const now = r.now || {}; const now = r.now || {};
@@ -321,11 +330,12 @@ async function showArtist(id) {
<button class="x" aria-label="Close" onclick="closeDrawer()">×</button> <button class="x" aria-label="Close" onclick="closeDrawer()">×</button>
</div> </div>
<div class="sec"> <div class="sec">
<h3>Chance of a new add in the 8 weeks from ${fc.window_start ? fmtDate(fc.window_start) : '—'}</h3> <h3>Chance of a new add in the 8 weeks from ${fc.window_start ? fmtDate(fc.window_start) : '—'}${$('mode').value === 'rel' && fc.add_rel ? ', if they release in it' : ''}</h3>
<div class="kpis"> <div class="kpis">
<div class="kpi"><div>Any DR station</div><b>${pct(fc.add_any)}</b></div> <div class="kpi"><div>Any DR station</div><b>${pct(fc.add_any)}</b></div>
${GROUPS.map(([g, l]) => `<div class="kpi"><div>${l}</div><b>${pct(chanP(fc, g))}</b></div>`).join('')} ${GROUPS.map(([g, l]) => `<div class="kpi"><div>${l}</div><b>${pct(chanP(fc, g))}</b></div>`).join('')}
</div> </div>
${fc.p_release != null ? `<p class="note">Chance they release something in the window (from their release rate): <b>${pct(fc.p_release)}</b>. If they release: ${pct(fc.add_any_rel)} · at their usual rate: ${pct(fc.add_any)}.</p>` : '<p class="note">No discography found on Deezer for this artist, so release timing isn\'t in the forecast.</p>'}
<p class="note">In rotation now: ${nowChips(a.now, Object.fromEntries(Object.entries(a.bauer || {}).map(([s, v]) => [s, v.plays])))} · plays in the week of ${fp.week ? fmtDate(fp.week) : '—'}: <b>${Math.round(fp.mean ?? 0)}</b> (${Math.round(fp.lo ?? 0)}–${Math.round(fp.hi ?? 0)})</p> <p class="note">In rotation now: ${nowChips(a.now, Object.fromEntries(Object.entries(a.bauer || {}).map(([s, v]) => [s, v.plays])))} · plays in the week of ${fp.week ? fmtDate(fp.week) : '—'}: <b>${Math.round(fp.mean ?? 0)}</b> (${Math.round(fp.lo ?? 0)}–${Math.round(fp.hi ?? 0)})</p>
</div> </div>
<div class="sec"><h3>Weekly plays, last 51 weeks and next 12</h3>${historyChart(a)}</div> <div class="sec"><h3>Weekly plays, last 51 weeks and next 12</h3>${historyChart(a)}</div>
@@ -341,11 +351,12 @@ function whatIfUI(a, h) {
const box = $('wi'); const box = $('wi');
box.innerHTML = `<div class="wi"> box.innerHTML = `<div class="wi">
<div class="row"><label for="wi-h">Window starts in (weeks)</label><input type="range" id="wi-h" min="0" max="${meta.horizons.length}" value="${h}"><b id="wi-hv"></b></div> <div class="row"><label for="wi-h">Window starts in (weeks)</label><input type="range" id="wi-h" min="0" max="${meta.horizons.length}" value="${h}"><b id="wi-hv"></b></div>
<div class="row"><label for="wi-r">Releases a song in the window</label><input type="checkbox" id="wi-r" checked><b></b></div>
<div class="row"><label for="wi-e">New chart entry, weeks ago</label><input type="range" id="wi-e" min="-1" max="26" value="-1"><b id="wi-ev"></b></div> <div class="row"><label for="wi-e">New chart entry, weeks ago</label><input type="range" id="wi-e" min="-1" max="26" value="-1"><b id="wi-ev"></b></div>
<div class="row"><label for="wi-t">Track Top-40 position now</label><input type="range" id="wi-t" min="0" max="40" value="0"><b id="wi-tv"></b></div> <div class="row"><label for="wi-t">Track Top-40 position now</label><input type="range" id="wi-t" min="0" max="40" value="0"><b id="wi-tv"></b></div>
<div class="row"><label for="wi-a">Airplay Top-20 position now</label><input type="range" id="wi-a" min="0" max="20" value="0"><b id="wi-av"></b></div> <div class="row"><label for="wi-a">Airplay Top-20 position now</label><input type="range" id="wi-a" min="0" max="20" value="0"><b id="wi-av"></b></div>
<div id="wi-out"></div> <div id="wi-out"></div>
<p class="note">Change the artist's chart situation (as if it were true this week) and see how the model's forecast moves. "New chart entry" stands in for a release that charted, until release dates are in the model.</p> <p class="note">Switch the release on or off and change the artist's chart situation (as if it were true this week) to see how the forecast moves.</p>
</div>`; </div>`;
let init = false, timer = null; let init = false, timer = null;
const label = () => { const label = () => {
@@ -357,6 +368,7 @@ function whatIfUI(a, h) {
const run = async () => { const run = async () => {
label(); label();
let qs = `id=${encodeURIComponent(a.id)}&h=${$('wi-h').value}`; let qs = `id=${encodeURIComponent(a.id)}&h=${$('wi-h').value}`;
qs += `&release=${$('wi-r').checked ? 'yes' : 'no'}`;
if (init) qs += `&entry=${$('wi-e').value < 0 ? 'none' : $('wi-e').value}&t40=${$('wi-t').value}&air=${$('wi-a').value}`; if (init) qs += `&entry=${$('wi-e').value < 0 ? 'none' : $('wi-e').value}&t40=${$('wi-t').value}&air=${$('wi-a').value}`;
const r = await fetch(BASE + '/api/whatif?' + qs); const r = await fetch(BASE + '/api/whatif?' + qs);
if (r.status === 503) { $('wi-out').innerHTML = '<p class="note">The model is still loading on the server; try again in a minute.</p>'; return; } if (r.status === 503) { $('wi-out').innerHTML = '<p class="note">The model is still loading on the server; try again in a minute.</p>'; return; }
@@ -375,7 +387,7 @@ function whatIfUI(a, h) {
<div class="kpis">${rows.map(([l, b, s]) => `<div class="kpi"><div>${l}</div><b>${pct(s)}</b>${d(b, s, 'p')}<div>was ${pct(b)}</div></div>`).join('')} <div class="kpis">${rows.map(([l, b, s]) => `<div class="kpi"><div>${l}</div><b>${pct(s)}</b>${d(b, s, 'p')}<div>was ${pct(b)}</div></div>`).join('')}
<div class="kpi"><div>Plays, first week</div><b>${Math.round(w.scenario.mean)}</b>${d(w.base.mean, w.scenario.mean, 'n')}<div>was ${Math.round(w.base.mean)}</div></div></div>`; <div class="kpi"><div>Plays, first week</div><b>${Math.round(w.scenario.mean)}</b>${d(w.base.mean, w.scenario.mean, 'n')}<div>was ${Math.round(w.base.mean)}</div></div></div>`;
}; };
['wi-h', 'wi-e', 'wi-t', 'wi-a'].forEach(id => $(id).addEventListener('input', () => { label(); clearTimeout(timer); timer = setTimeout(run, 120); })); ['wi-h', 'wi-r', 'wi-e', 'wi-t', 'wi-a'].forEach(id => $(id).addEventListener('input', () => { label(); clearTimeout(timer); timer = setTimeout(run, 120); }));
run(); run();
} }
@@ -402,7 +414,7 @@ document.querySelectorAll('nav button').forEach(b => b.addEventListener('click',
$('tab-rank').hidden = b.dataset.tab !== 'rank'; $('tab-model').hidden = b.dataset.tab !== 'model'; $('tab-rank').hidden = b.dataset.tab !== 'rank'; $('tab-model').hidden = b.dataset.tab !== 'model';
})); }));
$('h').addEventListener('input', () => { shown = 100; loadRank(); }); $('h').addEventListener('input', () => { shown = 100; loadRank(); });
['ch', 'sort', 'dk'].forEach(id => $(id).addEventListener('change', () => { shown = 100; render(); })); ['ch', 'sort', 'dk', 'mode'].forEach(id => $(id).addEventListener('change', () => { shown = 100; render(); }));
$('q').addEventListener('input', () => { shown = 100; render(); }); $('q').addEventListener('input', () => { shown = 100; render(); });
$('more').addEventListener('click', () => { shown += 100; render(); }); $('more').addEventListener('click', () => { shown += 100; render(); });
$('rows').addEventListener('click', e => { const tr = e.target.closest('tr[data-id]'); if (tr) showArtist(tr.dataset.id); }); $('rows').addEventListener('click', e => { const tr = e.target.closest('tr[data-id]'); if (tr) showArtist(tr.dataset.id); });