Wider artist pool (artists credited on DR-played tracks, groups; weekly refresh, oldest first); Platt recalibration of the add model on recent closed windows; Bauer plays per artist on the site

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-15 03:16:28 +00:00
co-authored by Claude Opus 5.5
parent 33a3c3f2e5
commit 7815ed417c
5 changed files with 130 additions and 18 deletions
+37 -8
View File
@@ -92,18 +92,47 @@ pub fn collect_plays(db: &Db, http: &Http, days: i64, log: &mut Vec<String>) ->
/// Artist pages: 52 weeks of weekly plays per channel. Fetched for every artist seen in the
/// plays, again once the page is `max_age_days` old. `limit` caps the number per run.
pub fn collect_artists(db: &Db, http: &Http, max_age_days: i64, limit: usize, log: &mut Vec<String>) -> Result<usize> {
let cutoff = (Utc::now() - Duration::days(max_age_days)).to_rfc3339();
let mut ids: Vec<String> = Vec::new();
// Where artist ids come from: DR's play logs, and (from stored artist pages) artists credited
// on tracks DR has played and groups an artist belongs to ("Medlem af"). Band members alone
// aren't release candidates and are left out.
db.conn.execute_batch(
"CREATE TABLE IF NOT EXISTS dr_artist_sources (
artist_id TEXT NOT NULL, source TEXT NOT NULL, first_seen TEXT NOT NULL,
PRIMARY KEY (artist_id, source)
);",
)?;
let now = Utc::now().to_rfc3339();
{
let mut add = db.conn.prepare("INSERT OR IGNORE INTO dr_artist_sources VALUES (?1, ?2, ?3)")?;
let mut st = db.conn.prepare("SELECT DISTINCT artist_ids FROM plays WHERE source = 'dr' AND artist_ids != ''")?;
let rows = st.query_map([], |r| r.get::<_, String>(0))?;
let mut seen = std::collections::BTreeSet::new();
for r in rows { for id in r?.split(',') { if !id.is_empty() { seen.insert(id.to_string()); } } }
let mut fresh = db.conn.prepare("SELECT count(*) FROM dr_artists WHERE artist_id = ?1 AND fetched_at > ?2")?;
for id in seen {
if fresh.query_row(params![id, cutoff], |r| r.get::<_, i64>(0))? == 0 { ids.push(id); }
let rows: Vec<String> = st.query_map([], |r| r.get(0))?.collect::<Result<_, _>>()?;
for r in rows { for id in r.split(',') { if !id.is_empty() { add.execute(params![id, "plays", now])?; } } }
let mut st = db.conn.prepare("SELECT raw FROM dr_artists")?;
let raws: Vec<String> = st.query_map([], |r| r.get(0))?.collect::<Result<_, _>>()?;
for raw in raws {
let v: Value = serde_json::from_str(&raw).unwrap_or(Value::Null);
for rel in v["artist"]["relationships"].as_array().into_iter().flatten() {
if rel["type"].as_str() == Some("Medlem af") {
if let Some(id) = rel["artist"]["id"].as_str() { add.execute(params![id, "group", now])?; }
}
}
for list in ["mostPlayed", "latestPlayed"] {
for t in v[list].as_array().into_iter().flatten() {
for role in t["roles"].as_array().into_iter().flatten() {
if let Some(id) = role["artist"]["id"].as_str() { add.execute(params![id, "credited", now])?; }
}
}
}
}
}
// Never read first, then the stalest; everything older than max_age_days.
let cutoff = (Utc::now() - Duration::days(max_age_days)).to_rfc3339();
let ids: Vec<String> = db.conn.prepare(
"SELECT s.artist_id FROM (SELECT DISTINCT artist_id FROM dr_artist_sources) s
LEFT JOIN dr_artists a ON a.artist_id = s.artist_id
WHERE a.fetched_at IS NULL OR a.fetched_at < ?1
ORDER BY a.fetched_at IS NOT NULL, a.fetched_at")?
.query_map([cutoff], |r| r.get(0))?.collect::<Result<_, _>>()?;
let mut done = 0;
for id in ids.into_iter().take(limit) {
let props = match page_props(http, &format!("/lyd/musik/kunstner/{id}")) {
+1 -1
View File
@@ -86,7 +86,7 @@ fn collect(db: &db::Db, args: &[String], db_path: &str) -> Result<()> {
let dir = std::path::Path::new(db_path).parent().unwrap_or(std::path::Path::new(".")).join("rotations");
let r = dr::collect_rotation_pdf(db, &http, &dir);
step("dr rotation list", r.map(|f| f.map(|f| format!("saved {f}")).unwrap_or_else(|| "unchanged".into())), &mut log);
let r = dr::collect_artists(db, &http, 6, if backfill { usize::MAX } else { 400 }, &mut log);
let r = dr::collect_artists(db, &http, 6, if backfill { usize::MAX } else { 1500 }, &mut log);
step("dr artist pages", r.map(|n| format!("{n} fetched")), &mut log);
}
if wants("bauer") {
+44 -3
View File
@@ -126,15 +126,56 @@ pub struct AddModel {
pub h: usize,
pub channel: Vec<f64>,
pub any: Vec<f64>,
/// Platt recalibration (a, b): p' = sigmoid(a + b * logit(p)); identity when there wasn't
/// enough history to fit it honestly.
pub platt_channel: (f64, f64),
pub platt_any: (f64, f64),
}
/// Recent origins used to fit the recalibration.
const CAL_ORIGINS: usize = 6;
fn logit(p: f64) -> f64 { let p = p.clamp(1e-6, 1.0 - 1e-6); (p / (1.0 - p)).ln() }
fn platt(ab: (f64, f64), p: f64) -> f64 { glm::sigmoid(ab.0 + ab.1 * logit(p)).clamp(1e-6, 1.0 - 1e-6) }
fn origins(h: usize, last_week: usize) -> std::ops::RangeInclusive<usize> {
let hi = last_week.saturating_sub(h + W);
LAGS - 1..=hi
}
impl AddModel {
/// The logits on every window closed by `last_week`, then a recalibration: an inner model fitted
/// on windows closed before the last CAL_ORIGINS origins predicts those origins, and a Platt fit
/// maps its probabilities to what happened. (The raw fits were overconfident above ~0.4.)
pub fn fit(p: &Panel, r: &Rot, h: usize, last_week: usize) -> AddModel {
let mut m = Self::fit_raw(p, r, h, last_week);
let hi = last_week.saturating_sub(h + W);
if hi + 1 >= CAL_ORIGINS + LAGS {
let c0 = hi + 1 - CAL_ORIGINS;
// the inner model may only use windows that closed before the first calibration origin
if c0 >= LAGS - 1 + h + W + 4 {
let inner = Self::fit_raw(p, r, h, c0);
let (mut dc, mut da) = (Design::new(2), Design::new(2));
for t in c0..=hi {
for a in 0..p.a() {
let (pc, pa) = inner.predict(p, r, a, t, None);
for c in 0..C {
if pc[c] < 1e-4 && !r.add_in_window(a, c, t, h) { continue; } // certain zeros carry no information
dc.push(&[1.0, logit(pc[c])], r.add_in_window(a, c, t, h) as i32 as f64, 1.0);
}
da.push(&[1.0, logit(pa)], r.add_any_in_window(a, t, h) as i32 as f64, 1.0);
}
}
let (bc, ba) = (glm::logit(&dc, 0.0), glm::logit(&da, 0.0));
// keep the map monotone and sane
if bc[1] > 0.2 { m.platt_channel = (bc[0], bc[1]); }
if ba[1] > 0.2 { m.platt_any = (ba[0], ba[1]); }
}
}
m
}
fn fit_raw(p: &Panel, r: &Rot, h: usize, last_week: usize) -> AddModel {
let (mut dc, mut da) = (Design::new(PC), Design::new(PA));
let (mut x, mut xa) = (vec![0.0; PC], vec![0.0; PA]);
for t in origins(h, last_week) {
@@ -150,7 +191,7 @@ impl AddModel {
da.push(&xa, r.add_any_in_window(a, t, h) as i32 as f64, 1.0);
}
}
AddModel { h, channel: glm::logit(&dc, 1.0), any: glm::logit(&da, 1.0) }
AddModel { h, channel: glm::logit(&dc, 1.0), any: glm::logit(&da, 1.0), platt_channel: (0.0, 1.0), platt_any: (0.0, 1.0) }
}
/// (P(add) per channel, P(add on any channel)) from origin t.
@@ -160,12 +201,12 @@ impl AddModel {
for c in 0..C {
row_channel(p, r, a, c, t, self.h, &mut x);
if let Some(f) = chart { f(&mut x, CHART_IDX_CHANNEL); }
pc[c] = glm::sigmoid(glm::dot(&self.channel, &x)).clamp(1e-6, 1.0 - 1e-6);
pc[c] = platt(self.platt_channel, glm::sigmoid(glm::dot(&self.channel, &x)));
}
let mut xa = vec![0.0; PA];
row_any(p, r, a, t, self.h, &mut xa);
if let Some(f) = chart { f(&mut xa, CHART_IDX_ANY); }
(pc, glm::sigmoid(glm::dot(&self.any, &xa)).clamp(1e-6, 1.0 - 1e-6))
(pc, platt(self.platt_any, glm::sigmoid(glm::dot(&self.any, &xa))))
}
}
+38 -1
View File
@@ -191,6 +191,34 @@ fn fold(e: &mut Value, target: &str, ch: &str, prob: Option<f64>, mean: Option<f
}
}
/// Bauer plays (NOVA, Radio Soft, The Voice, Pop FM, Radio 100) in the 7 days up to the latest
/// one, per artist name key (each credited artist) and station: (plays, songs).
fn bauer_week(c: &Connection) -> Result<HashMap<String, HashMap<String, (i64, Vec<(String, i64)>)>>> {
let mut out: HashMap<String, HashMap<String, (i64, Vec<(String, i64)>)>> = HashMap::new();
let mut st = c.prepare(
"SELECT station, artist, title, count(*) FROM plays WHERE source = 'bauer'
AND played_at > (SELECT strftime('%Y-%m-%dT%H:%M:%SZ', max(played_at), '-7 days') FROM plays WHERE source = 'bauer')
GROUP BY station, artist, title")?;
let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, i64>(3)?)))?;
for r in rows {
let (station, artist, title, n) = r?;
let mut parts = vec![artist.to_lowercase()];
for sep in [" feat. ", " feat ", " ft. ", " ft ", " featuring ", " & ", ", ", " x ", " og ", " vs "] {
parts = parts.iter().flat_map(|p| p.split(sep).map(str::to_string).collect::<Vec<_>>()).collect();
}
let mut keys: Vec<String> = parts.iter().map(|p| crate::model::panel::key(p)).collect();
keys.push(crate::model::panel::key(&artist));
keys.sort(); keys.dedup();
for k in keys.into_iter().filter(|k| !k.is_empty()) {
let e = out.entry(k).or_default().entry(station.clone()).or_insert((0, Vec::new()));
e.0 += n;
e.1.push((title.clone(), n));
}
}
Ok(out)
}
fn rank(st: &State, h: usize) -> Result<Reply> {
let c = ro(&st.db_path)?;
let Some(made) = latest_run(&c)? else { return ok(json!({"rows": []})) };
@@ -217,6 +245,11 @@ fn rank(st: &State, h: usize) -> Result<Reply> {
if target == "add_any" && window.is_empty() { window = week; }
fold(e, &target, ch, prob, mean, lo, hi);
}
let bauer = bauer_week(&c)?;
for e in rows.values_mut() {
let k = crate::model::panel::key(e["name"].as_str().unwrap_or(""));
if let Some(b) = bauer.get(&k) { e["bauer"] = json!(b.iter().map(|(s, v)| (s.clone(), v.0)).collect::<HashMap<_, _>>()); }
}
let mut out: Vec<Value> = rows.into_values().collect();
out.sort_by(|a, b| b["add_any"].as_f64().unwrap_or(0.0).total_cmp(&a["add_any"].as_f64().unwrap_or(0.0)));
let body = Arc::new(json!({"made_at": made, "h": h, "window_start": window, "rows": out}).to_string());
@@ -288,8 +321,12 @@ fn artist(st: &State, id: &str) -> Result<Reply> {
}
}
let bauer: Value = bauer_week(&c)?.remove(&key).map(|b| json!(b.into_iter().map(|(s, (n, mut songs))| {
songs.sort_by(|x, y| y.1.cmp(&x.1));
(s, json!({"plays": n, "songs": songs.into_iter().take(5).map(|(t, n)| json!({"title": t, "plays": n})).collect::<Vec<_>>()}))
}).collect::<HashMap<_, _>>())).unwrap_or(Value::Null);
ok(json!({
"id": id, "name": name, "country": a["country"], "gender": a["gender"], "description": a["description"],
"id": id, "name": name, "bauer": bauer, "country": a["country"], "gender": a["gender"], "description": a["description"],
"debut": a["lifespan"]["beginYear"], "type": a["type"], "image": image, "top_tracks": top,
"weeks": weeks, "plays": plays, "forecasts": fc, "now": now_out, "charts": charts,
}))
+10 -5
View File
@@ -139,7 +139,7 @@ svg text { fill: var(--muted); font-size: 11px; }
</table>
</div>
<button class="more" id="more" hidden>Show more</button>
<p class="note"><b>New add</b> = the chance that the artist gets a song into rotation within the 8-week window: entering rotation (at or above the station's weekly play threshold) after at least 4 weeks out of rotation. For "any DR station", out of rotation everywhere first. An artist with a song in rotation now usually has to drop out before a new add, which is why current hits rank lower here. "In rotation now" shows where the artist is in rotation in the latest week; plays per week are for the window's first week. Click an artist for history and a what-if.</p>
<p class="note">"In rotation now" lists DR stations where the artist is on the list, then Bauer stations (grey) with 14+ plays in the last 7 days. <b>New add</b> = the chance that the artist gets a song into rotation within the 8-week window: entering rotation (at or above the station's weekly play threshold) after at least 4 weeks out of rotation. For "any DR station", out of rotation everywhere first. An artist with a song in rotation now usually has to drop out before a new add, which is why current hits rank lower here. "In rotation now" shows where the artist is in rotation in the latest week; plays per week are for the window's first week. Click an artist for history and a what-if.</p>
</section>
<section id="tab-model" hidden>
@@ -175,6 +175,7 @@ const BASE = location.pathname.replace(/\/(index\.html)?$/, '');
const API = p => fetch(BASE + '/api/' + p).then(r => r.json());
const GROUPS = [['p3', 'P3'], ['p4', 'P4'], ['p5', 'P5'], ['p6beat', 'P6'], ['p8jazz', 'P8']];
const P4S = ['p4kbh','p4sjaelland','p4fyn','p4syd','p4trekanten','p4vest','p4aarhus','p4nord','p4bornholm','p4esbjerg'];
const BAUER = { nov: 'NOVA', dso: 'Radio Soft', the: 'The Voice', pop: 'Pop FM', rhu: 'Radio 100' };
const COLORS = { p3: 'var(--p3)', p4: 'var(--p4)', p5: 'var(--p5)', p6beat: 'var(--p6)', p8jazz: 'var(--p8)' };
let meta = null, data = { 1: null }, shown = 100, current = [];
const $ = id => document.getElementById(id);
@@ -195,10 +196,13 @@ function nowPlays(r, g) {
if (g === 'p4') return P4S.reduce((s, c) => s + v(c), 0) / P4S.length;
return v(g);
}
function nowChips(now) {
function nowChips(now, bauer) {
now = now || {};
const on = GROUPS.filter(([g]) => g === 'p4' ? P4S.some(c => now[c]?.on) : now[g]?.on).map(([, l]) => l);
return on.length ? on.join(' · ') : '<span class="note">—</span>';
// Bauer: no published lists; 14+ plays in the last 7 days (two a day) shown as heavy rotation.
const b = Object.entries(bauer || {}).filter(([, n]) => n >= 14).sort((x, y) => y[1] - x[1]).map(([s, n]) => `${BAUER[s] || s} ${n}`);
const all = on.concat(b.map(x => `<span class="note">${x}</span>`));
return all.length ? all.join(' · ') : '<span class="note">—</span>';
}
function chanPlays(r, g) {
if (g === 'any') return r.mean;
@@ -248,7 +252,7 @@ function render() {
<td class="l name">${esc(r.name)}${r.danish ? '<small>DK</small>' : ''}</td>
<td><span class="bar"><i style="width:${Math.round((p ?? 0) * 70)}px"></i>${pct(p)}</span></td>
${GROUPS.map(([c]) => `<td class="hide-sm"><span class="cell" style="${heat(chanP(r, c))}">${pct(chanP(r, c))}</span></td>`).join('')}
<td class="l hide-sm">${nowChips(r.now)}</td>
<td class="l hide-sm">${nowChips(r.now, r.bauer)}</td>
<td>${pl == null ? '—' : pl.toFixed(pl < 10 ? 1 : 0)}${range}</td>
</tr>`;
}).join('');
@@ -322,10 +326,11 @@ async function showArtist(id) {
<div class="kpi"><div>Any DR station</div><b>${pct(fc.add_any)}</b></div>
${GROUPS.map(([g, l]) => `<div class="kpi"><div>${l}</div><b>${pct(chanP(fc, g))}</b></div>`).join('')}
</div>
<p class="note">In rotation now: ${nowChips(a.now)} · plays in the week of ${fp.week ? fmtDate(fp.week) : '—'}: <b>${Math.round(fp.mean ?? 0)}</b> (${Math.round(fp.lo ?? 0)}–${Math.round(fp.hi ?? 0)})</p>
<p class="note">In rotation now: ${nowChips(a.now, Object.fromEntries(Object.entries(a.bauer || {}).map(([s, v]) => [s, v.plays])))} · plays in the week of ${fp.week ? fmtDate(fp.week) : '—'}: <b>${Math.round(fp.mean ?? 0)}</b> (${Math.round(fp.lo ?? 0)}–${Math.round(fp.hi ?? 0)})</p>
</div>
<div class="sec"><h3>Weekly plays, last 51 weeks and next 12</h3>${historyChart(a)}</div>
<div class="sec"><h3>What if…</h3><div id="wi" class="note">Loading the model…</div></div>
${a.bauer ? `<div class="sec"><h3>Bauer radio, last 7 days</h3><div class="chips">${Object.entries(a.bauer).sort((x, y) => y[1].plays - x[1].plays).map(([s, v]) => `<span class="chip"><b>${BAUER[s] || s}</b> ${v.plays} plays: ${v.songs.map(t => `${esc(t.title)} (${t.plays})`).join(', ')}</span>`).join('')}</div><p class="note">Bauer publishes no rotation lists or history; plays are collected daily from now on, and Bauer adds join the model once there are a few weeks of logs.</p></div>` : ''}
${a.top_tracks.length ? `<div class="sec"><h3>Most played on DR (all time)</h3><div class="tracks">${a.top_tracks.map(t => `<div>${esc(t.title)} <span class="note">${t.total}</span></div>`).join('')}</div></div>` : ''}
${runList.length ? `<div class="sec"><h3>Chart runs (IFPI)</h3><div class="chips">${runList.map(r => `<span class="chip">${r.chart === 't40' ? 'Track Top-40' : 'Airplay Top-20'}: ${esc(r.title)}, peak #${r.best}, ${r.weeks} wk, ${r.first.year}</span>`).join('')}</div></div>` : ''}
`);